From db8dbb6c7ac546677f4ddb0946f2d518b12812ec Mon Sep 17 00:00:00 2001 From: Jeltz Date: Thu, 11 Mar 2021 20:08:41 +0100 Subject: [PATCH 01/44] Add borgbackup_server role --- roles/borgbackup_server/defaults/main.yml | 5 +++ roles/borgbackup_server/tasks/main.yml | 47 +++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 roles/borgbackup_server/defaults/main.yml create mode 100644 roles/borgbackup_server/tasks/main.yml diff --git a/roles/borgbackup_server/defaults/main.yml b/roles/borgbackup_server/defaults/main.yml new file mode 100644 index 0000000..bedb950 --- /dev/null +++ b/roles/borgbackup_server/defaults/main.yml @@ -0,0 +1,5 @@ +--- +borg_server_user: borgbackup +borg_server_group: borgbackup +borg_home_dir: /var/lib/borgbackup +... diff --git a/roles/borgbackup_server/tasks/main.yml b/roles/borgbackup_server/tasks/main.yml new file mode 100644 index 0000000..eb92431 --- /dev/null +++ b/roles/borgbackup_server/tasks/main.yml @@ -0,0 +1,47 @@ +--- +- name: Install borg + apt: + name: borgbackup + state: latest + update_cache: true + register: apt_result + retries: 3 + until: apt_result is succeeded + +- name: Create a local group for borg + become: true + group: + name: "{{ borg_server_group }}" + system: true + +- name: Create a local user for borg + become: true + user: + home: "{{ borg_home_dir }}" + create_home: true + comment: Local user for borgbackup server + name: "{{ borg_server_user }}" + group: "{{ borg_server_group }}" + system: true + # Does not constitute a valid hash, preventing from login via password + password: "*" + update_password: always + +- name: Create backup directory + become: true + file: + path: "{{ borg_server_backups_dir }}" + state: directory + owner: "{{ borg_server_user }}" + group: "{{ borg_server_group }}" + mode: u=rw + +- include_role: + name: update_motd + vars: + motd_messages: + - key: 10-borg-server + message: >- + Les sauvegardes (borg) sont stockées dans + {{ borg_server_backups_dir }}. +... From 2ea7f6f9f7ff5ed58c17f33bbaa52d9858fd4541 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Thu, 11 Mar 2021 20:09:09 +0100 Subject: [PATCH 02/44] Add host_vars for perceval --- host_vars/perceval.adm.auro.re.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 host_vars/perceval.adm.auro.re.yml diff --git a/host_vars/perceval.adm.auro.re.yml b/host_vars/perceval.adm.auro.re.yml new file mode 100644 index 0000000..a534d22 --- /dev/null +++ b/host_vars/perceval.adm.auro.re.yml @@ -0,0 +1,3 @@ +--- +borg_server_backups_dir: /borg +... From 7cf616f6c739c3cb03b78d5c1f2a3d5c76b8827f Mon Sep 17 00:00:00 2001 From: Jeltz Date: Thu, 11 Mar 2021 20:09:26 +0100 Subject: [PATCH 03/44] Add playbook for backups --- backups.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 backups.yml diff --git a/backups.yml b/backups.yml new file mode 100644 index 0000000..36f09be --- /dev/null +++ b/backups.yml @@ -0,0 +1,5 @@ +--- +- hosts: perceval.adm.auro.re + roles: + - borgbackup_server +... From f0f56ecd3f61b54ceab14ca95ada39e58dd6b417 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Thu, 11 Mar 2021 20:15:35 +0100 Subject: [PATCH 04/44] Fix linter-related issues --- .ansible-lint | 4 +++- roles/update_motd/handlers/main.yml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.ansible-lint b/.ansible-lint index d98efd4..329f921 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -2,9 +2,11 @@ skip_list: - no-changed-when - load-failure - document-start + - package-latest warn_list: - experimental # all rules tagged as experimental exclude_paths: -- group_vars/all/vault.yml + - group_vars/all/vault.yml + - utils/ diff --git a/roles/update_motd/handlers/main.yml b/roles/update_motd/handlers/main.yml index 41dcc84..3736f9d 100644 --- a/roles/update_motd/handlers/main.yml +++ b/roles/update_motd/handlers/main.yml @@ -2,7 +2,7 @@ - name: Remove cached motd become: true file: - path: "{{ item }}" + path: "{{ item }}" state: absent loop: - /var/run/motd.dynamic From 46423953301e65601c265643361d2e70d72fdc38 Mon Sep 17 00:00:00 2001 From: Otthorn Date: Thu, 11 Mar 2021 23:29:57 +0100 Subject: [PATCH 05/44] [borgbackup_client] Add initial role defintion --- roles/borgbackup_client/defaults/main.yml | 5 ++ roles/borgbackup_client/handlers/main.yml | 17 ++++ roles/borgbackup_client/tasks/main.yml | 85 +++++++++++++++++++ .../templates/borgmatic.service.j2 | 29 +++++++ .../templates/borgmatic.timer.j2 | 10 +++ .../templates/config.yaml.j2 | 25 ++++++ 6 files changed, 171 insertions(+) create mode 100644 roles/borgbackup_client/defaults/main.yml create mode 100644 roles/borgbackup_client/handlers/main.yml create mode 100644 roles/borgbackup_client/tasks/main.yml create mode 100644 roles/borgbackup_client/templates/borgmatic.service.j2 create mode 100644 roles/borgbackup_client/templates/borgmatic.timer.j2 create mode 100644 roles/borgbackup_client/templates/config.yaml.j2 diff --git a/roles/borgbackup_client/defaults/main.yml b/roles/borgbackup_client/defaults/main.yml new file mode 100644 index 0000000..28ba278 --- /dev/null +++ b/roles/borgbackup_client/defaults/main.yml @@ -0,0 +1,5 @@ +--- +borg_keep_daily: 7 +borg_keep_weekly: 4 +borg_keep_monthly: 12 +... diff --git a/roles/borgbackup_client/handlers/main.yml b/roles/borgbackup_client/handlers/main.yml new file mode 100644 index 0000000..426cc4e --- /dev/null +++ b/roles/borgbackup_client/handlers/main.yml @@ -0,0 +1,17 @@ +--- +- name: Run systemd daemon-reload + systemd: + daemon_reload: true + +- name: Start and enable borgmatic timer + systemd: + name: borgmatic.timer + state: started + enabled: true + +- name: Start and enable borgmatic service + systemd: + name: borgmatic.service + state: started + enabled: true +... diff --git a/roles/borgbackup_client/tasks/main.yml b/roles/borgbackup_client/tasks/main.yml new file mode 100644 index 0000000..a337fee --- /dev/null +++ b/roles/borgbackup_client/tasks/main.yml @@ -0,0 +1,85 @@ +--- +- name: Pin borgmatic + template: + src: "apt/{{ item }}.j2" + dest: "/etc/apt/{{ item }}" + loop: + - sources.list.d/bullseye.list + - preferences.d/borgmatic-bullseye + when: + - ansible_distribution == "Debian" + - ansible_distribution_major_version | int <= 10 + - ansible_distribution_release != "bullseye" + +- name: Don't pin borgmatic if we are on bullseye + file: + path: "/etc/apt/{{ item }}" + state: absent + loop: + - sources.list.d/bullseye.list + - preferences.d/borgmatic-bullseye + when: + - ansible_distribution == "Debian" + - ansible_distribution_release == "bullseye" + +- name: Install borgmatic + apt: + name: borgmatic + register: apt_result + retries: 3 + until: apt_result is succeeded + +- name: Create SSH key + openssh_keypair: + path: /etc/borgbackup/id_remote + type: ed25519 + owner: root + group: root + mode: u=r,g=r,o= + register: ssh_key + +- name: Add public key to remote + delegate_to: "{{ borg_server_host }}" + become: "{{ borg_server_user }}" + authorized_key: + exclusive: false + key: "{{ ssh_key.public_key }}" + key_options: >- + command="borg serve --restrict-to-path {{ borg_host_dir }}, + no-agent-forwarding, + no-port-forwarding, + no-pty, + no-user-rc, + no-X11-forwarding + vars: + borg_host_dir: "{{ borg_server_base_dir }}/{{ ansible_fqdn }}" + +- name: Add borgmatic configuration file + become: true + template: + src: borgmatic.conf.j2 + dest: /etc/borgmatic/config.yaml + owner: root + group: root + mode: u=rw,g=r,o= + +- name: Install timer and service for borgmatic + template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + owner: root + group: root + mode: u=rw,g=r,o= + loop: + - borgmatic.timer + - borgmatic.service + notify: + - Run systemd daemon-reload + - Start and enable borgmatic timer + - Start and enable borgmatic service + +- include_role: update_motd + vars: + key: 10-borgmatic + message: Borgmatic (client) est installé dans /etc/borgmatic/config.yaml. +... diff --git a/roles/borgbackup_client/templates/borgmatic.service.j2 b/roles/borgbackup_client/templates/borgmatic.service.j2 new file mode 100644 index 0000000..f689b66 --- /dev/null +++ b/roles/borgbackup_client/templates/borgmatic.service.j2 @@ -0,0 +1,29 @@ +{{ ansible_managed | comment }} + +[Unit] +Description=Service for borgmatic backup +Wants=network-online.target +After=network-online.target +ConditionACPower=true + +[Service] +Type=oneshot +ProtectSystem=full +CapabilityBoundingSet=CAP_DAC_READ_SEARCH CAP_NET_RAW + +# Lower CPU and I/O priority. +Nice=19 +CPUSchedulingPolicy=batch +IOSchedulingClass=best-effort +IOSchedulingPriority=7 +IOWeight=100 + +Restart=no +# Prevent rate limiting of borgmatic log events. If you are using an older version of systemd that +# doesn't support this (pre-240 or so), you may have to remove this option. +LogRateLimitIntervalSec=0 + +# Delay start to prevent backups running during boot. Note that systemd-inhibit requires dbus and +# dbus-user-session to be installed. +ExecStartPre=sleep 1m +ExecStart=systemd-inhibit --who="borgmatic" --why="Prevent interrupting scheduled backup" /usr/bin/borgmatic --syslog-verbosity 1 diff --git a/roles/borgbackup_client/templates/borgmatic.timer.j2 b/roles/borgbackup_client/templates/borgmatic.timer.j2 new file mode 100644 index 0000000..22f1bf9 --- /dev/null +++ b/roles/borgbackup_client/templates/borgmatic.timer.j2 @@ -0,0 +1,10 @@ +{{ ansible_managed | comment }} + +[Unit] +Description=Timer for borgmatic backup + +[Timer] +OnCalendar=daily + +[Install] +WantedBy=timers.target diff --git a/roles/borgbackup_client/templates/config.yaml.j2 b/roles/borgbackup_client/templates/config.yaml.j2 new file mode 100644 index 0000000..ead247e --- /dev/null +++ b/roles/borgbackup_client/templates/config.yaml.j2 @@ -0,0 +1,25 @@ +--- +{{ ansible_managed | comment }} + +location: + source_directories: {{ borg_backup_directories | to_nice_yaml }} + repositories: + - {{ borg_server_user }}@{{ borg_server_host }}:{{ borg_host_dir }} + +retention: + keep_daily: "{{ borg_keep_daily }}" + keep_weekly: "{{ borg_keep_weekly }}" + keep_monthly: "{{ borg_keep_yearly }}" + +consistency: + checks: + - repository + - archives + +{% if postgresql_databases is defined %} +hooks: + postgresql_databases: + - name: all +{% endif %} +... + From 313314a6749d6f4b3ed54d91d474242425a81c6a Mon Sep 17 00:00:00 2001 From: Otthorn Date: Thu, 11 Mar 2021 23:36:27 +0100 Subject: [PATCH 06/44] [borgbackup_client] fix risky file permission on apt config for pinning version --- roles/borgbackup_client/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/borgbackup_client/tasks/main.yml b/roles/borgbackup_client/tasks/main.yml index a337fee..799e36d 100644 --- a/roles/borgbackup_client/tasks/main.yml +++ b/roles/borgbackup_client/tasks/main.yml @@ -3,6 +3,9 @@ template: src: "apt/{{ item }}.j2" dest: "/etc/apt/{{ item }}" + owner: root + group: root + mode: u=rw,g=r,o= loop: - sources.list.d/bullseye.list - preferences.d/borgmatic-bullseye From 531f7593d2af3857f9d3bf19bd287003e6682a99 Mon Sep 17 00:00:00 2001 From: Otthorn Date: Thu, 11 Mar 2021 23:37:55 +0100 Subject: [PATCH 07/44] [borgbackup_client] fix identation --- roles/borgbackup_client/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/borgbackup_client/tasks/main.yml b/roles/borgbackup_client/tasks/main.yml index 799e36d..b396017 100644 --- a/roles/borgbackup_client/tasks/main.yml +++ b/roles/borgbackup_client/tasks/main.yml @@ -55,7 +55,7 @@ no-user-rc, no-X11-forwarding vars: - borg_host_dir: "{{ borg_server_base_dir }}/{{ ansible_fqdn }}" + borg_host_dir: "{{ borg_server_base_dir }}/{{ ansible_fqdn }}" - name: Add borgmatic configuration file become: true From 4123af6c01d8934da2014c231db436a78098d1dd Mon Sep 17 00:00:00 2001 From: Otthorn Date: Thu, 11 Mar 2021 23:56:07 +0100 Subject: [PATCH 08/44] [borgbackup_client] Install client on all machines (servers, vms and lxc containers) --- backups.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backups.yml b/backups.yml index 36f09be..b87c1f9 100644 --- a/backups.yml +++ b/backups.yml @@ -2,4 +2,8 @@ - hosts: perceval.adm.auro.re roles: - borgbackup_server + +- host: all, !unifi + roles: + - borgbackup_client ... From 3f8ffbe1644c6377b7716596ce29af1d8bde0148 Mon Sep 17 00:00:00 2001 From: Otthorn Date: Fri, 12 Mar 2021 00:01:11 +0100 Subject: [PATCH 09/44] [borgbackup_client] Add borg username and group defaults --- roles/borgbackup_client/defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/borgbackup_client/defaults/main.yml b/roles/borgbackup_client/defaults/main.yml index 28ba278..2550cdd 100644 --- a/roles/borgbackup_client/defaults/main.yml +++ b/roles/borgbackup_client/defaults/main.yml @@ -2,4 +2,6 @@ borg_keep_daily: 7 borg_keep_weekly: 4 borg_keep_monthly: 12 +borg_server_user: borgbackup +borg_server_group: borgbackup ... From 454bc66ae0848872e121f48cb56587e62d39c22d Mon Sep 17 00:00:00 2001 From: Otthorn Date: Fri, 12 Mar 2021 00:30:00 +0100 Subject: [PATCH 10/44] [borgbackup_client] backup /etc/ everywhere --- group_vars/all/vars.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/group_vars/all/vars.yml b/group_vars/all/vars.yml index 55125de..ba4f9e4 100644 --- a/group_vars/all/vars.yml +++ b/group_vars/all/vars.yml @@ -93,3 +93,13 @@ apartment_block_dhcp: "{{ apartment_block }}" ipv6_base_prefix: "2a09:6840" is_aurore_host: "{{ 'aurore_vm' in group_names }}" + +# Borgbackup +borg_keep_daily: 7 +borg_keep_weekly: 4 +borg_keep_monthly: 12 +borg_host_dir: "/borg/{{ ansible_fqdn }}/" +borg_backup_directories: + - "/etc/" + + From 29f2823960523144ee250fd6005e992f1fd09187 Mon Sep 17 00:00:00 2001 From: Otthorn Date: Fri, 12 Mar 2021 00:32:08 +0100 Subject: [PATCH 11/44] [borgbackup_client] tell borg that a postgresql database lies on re2o-bdd --- host_vars/re2o-bdd.adm.auro.re.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 host_vars/re2o-bdd.adm.auro.re.yml diff --git a/host_vars/re2o-bdd.adm.auro.re.yml b/host_vars/re2o-bdd.adm.auro.re.yml new file mode 100644 index 0000000..3b7266a --- /dev/null +++ b/host_vars/re2o-bdd.adm.auro.re.yml @@ -0,0 +1 @@ +postgresql_databases: true From cdcfad7ac2e86959cd92dde5a53d9f53f0712816 Mon Sep 17 00:00:00 2001 From: Otthorn Date: Fri, 12 Mar 2021 01:05:19 +0100 Subject: [PATCH 12/44] [borgbackup_client] backup /var/ also (for now) --- group_vars/all/vars.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/group_vars/all/vars.yml b/group_vars/all/vars.yml index ba4f9e4..e2c834d 100644 --- a/group_vars/all/vars.yml +++ b/group_vars/all/vars.yml @@ -101,5 +101,4 @@ borg_keep_monthly: 12 borg_host_dir: "/borg/{{ ansible_fqdn }}/" borg_backup_directories: - "/etc/" - - + - "/var/" From d16676bfb6ff9d45cfa2b4dad8c5fd9b0112f55a Mon Sep 17 00:00:00 2001 From: Otthorn Date: Fri, 12 Mar 2021 01:05:58 +0100 Subject: [PATCH 13/44] [borgbackup_client] hostname should also be unique, but more legible than fqdn --- group_vars/all/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/all/vars.yml b/group_vars/all/vars.yml index e2c834d..d4e2972 100644 --- a/group_vars/all/vars.yml +++ b/group_vars/all/vars.yml @@ -98,7 +98,7 @@ is_aurore_host: "{{ 'aurore_vm' in group_names }}" borg_keep_daily: 7 borg_keep_weekly: 4 borg_keep_monthly: 12 -borg_host_dir: "/borg/{{ ansible_fqdn }}/" +borg_host_dir: "/borg/{{ ansible_hostname }}/" borg_backup_directories: - "/etc/" - "/var/" From d7a4995496990944ad4b927a7b129e1e76aeb1d2 Mon Sep 17 00:00:00 2001 From: Otthorn Date: Fri, 12 Mar 2021 01:06:48 +0100 Subject: [PATCH 14/44] [borgbackup_client] backup the ZFS datasets --- host_vars/caradoc.adm.auro.re.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 host_vars/caradoc.adm.auro.re.yml diff --git a/host_vars/caradoc.adm.auro.re.yml b/host_vars/caradoc.adm.auro.re.yml new file mode 100644 index 0000000..726532d --- /dev/null +++ b/host_vars/caradoc.adm.auro.re.yml @@ -0,0 +1,11 @@ +--- +borg_keep_hourly: 6 +borg_keep_daily: 7 +borg_keep_weekly: 4 +borg_keep_monthly: 12 +borg_backup_directories: + - "/etc/" + - "/data_nextcloud" + - "/data_gitea" + - "/data_mail" +... From 965bbe62a4d45526b793b6ab75fb69c391b6b7b3 Mon Sep 17 00:00:00 2001 From: Otthorn Date: Fri, 12 Mar 2021 01:46:35 +0100 Subject: [PATCH 15/44] [borgbackup_client] configure encryption passphrase and storage --- roles/borgbackup_client/templates/config.yaml.j2 | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/roles/borgbackup_client/templates/config.yaml.j2 b/roles/borgbackup_client/templates/config.yaml.j2 index ead247e..0d9b90d 100644 --- a/roles/borgbackup_client/templates/config.yaml.j2 +++ b/roles/borgbackup_client/templates/config.yaml.j2 @@ -6,6 +6,15 @@ location: repositories: - {{ borg_server_user }}@{{ borg_server_host }}:{{ borg_host_dir }} +storage: + encryption_passphrase: "{{ borg_encryption_passphrase }}" + ssh_command: ssh -i /etc/borgbackup/id_remote + # compression: 'lz4' + # umask: 0077 + # lock_wait: 5 + # archive_name_format: '{hostname}-{now}' + + retention: keep_daily: "{{ borg_keep_daily }}" keep_weekly: "{{ borg_keep_weekly }}" @@ -22,4 +31,3 @@ hooks: - name: all {% endif %} ... - From 904678d1ac55abc0a0b59c462f0181002fdbee47 Mon Sep 17 00:00:00 2001 From: Otthorn Date: Fri, 12 Mar 2021 01:49:04 +0100 Subject: [PATCH 16/44] [borgbackup_client] Add borg encryption passphrase to vault --- group_vars/all/vars.yml | 1 + group_vars/all/vault.yml | 390 ++++++++++++++++++++------------------- 2 files changed, 198 insertions(+), 193 deletions(-) diff --git a/group_vars/all/vars.yml b/group_vars/all/vars.yml index d4e2972..a351878 100644 --- a/group_vars/all/vars.yml +++ b/group_vars/all/vars.yml @@ -102,3 +102,4 @@ borg_host_dir: "/borg/{{ ansible_hostname }}/" borg_backup_directories: - "/etc/" - "/var/" +borg_encryption_passphrase: "{{ vault_borg_encryption_passphrase }}" diff --git a/group_vars/all/vault.yml b/group_vars/all/vault.yml index 88ebc9c..5421d13 100644 --- a/group_vars/all/vault.yml +++ b/group_vars/all/vault.yml @@ -1,194 +1,198 @@ $ANSIBLE_VAULT;1.1;AES256 -61343633663661363430326639663636383962313132646533626563366164323530386666353863 -3164643762313835303732653631363664663963613238340a646533343436623562356231383831 -30626339313739393633623830663263626236616566376662656534656563303866396433393039 -3264356462336533320a386464386334633738626135653864323363353538393263626661633439 -65343032396264343539323261353839613462613732623932303035653965626161646637333062 -39393762393661373633303762306361356339343933363936373837623733653130393937343630 -36393865626561646330613636383361373465366166363132613164333438333362613730373432 -32626530316533383639623265353039646466396663333765363531656161633732313033666663 -38623831613138386136636539366262373965393865396631306331366663333236383939336165 -64613730643964656564636535376637303832393164383462346133343432373236656365633636 -36356630343739636262626364386561623036366439333365336464643861663730613034613063 -61393666666432303366313835323639366266663933323739316463353339636465366363303139 -35313630353363393933373130653730353831393162326563316162323430306161616431656433 -32653739356236656233396531376539396462646365356336343663383563316438343862383239 -30623464343632386636343861326632346233376664393638366236303865303064653436626539 -30626539353332666261616634316239653764343366323434333265383133353466623330613935 -63346333336461346432613565303235356439616564376537363164343130353833663464346336 -35363930303637626134643936643336323366373136313032333264633934633333636138393261 -35653930373632343838386430343961663436356539363064316638313531303364353864613861 -31306331663734396266383539323033393563653435396334656665313931346365623636306463 -65346362373335363032323866633466356363643133336330626533316365623865613163376636 -35353838353363303234393430373433613130333661303833383965643739383465316163623434 -34636133626565663062303130313530306663666665646664613639366239306461376663356332 -34303536343765373865643466623230636330316365306231643566303831313238396533343161 -65373434333561643136646232653737623737396431393334626234333262333334393634343331 -39356238646536386632653731653038646166303364376363633064343665303530396630356561 -34373238656534356166316639623232346434646132663936363238396363313864636531313764 -33343364346334373237366634626665656561343934613262313530373536316337353036333230 -39303730343638383138386138303863396631333534366233383533346232653639396566363662 -31366533386565663734626230353438353730363966356635616261356237346131616234326233 -61623633613633393265353838346436396535633839386664356534323962643761316661626162 -63666337363534653632343661373865383532313837396538633235313435333164393333383264 -63656362323561653731303332626331393531383030633231373331333032336334373330366562 -35663431316432373438326265646263633639636164323361643062636535623765623566393237 -36303766323930336538623564373038636661366132383465323431666361626134303136636234 -34393538323830323033616638656236623332623364393463303332616233313837376335643434 -33346532613264396264393834643039666465366338616133346261373336633261666234326238 -30653032633932653930633566646332623232313334353537363939376336663264313535663635 -39656630653964643462343930306232636266306639616664363864346433313239313863313666 -30653834666133353036636466643165653165303064353664393239343666646366356236643738 -61313734346334366530323935396133646534323538336333376362356664633538393933363739 -31346262303365386232376537346366303762613735366561383165653737633738303432646531 -32386162633364383737663734346464653362656633313036303261663834636334333233303039 -62656531653336333436623863633139623531643837643461663038386430316663313264643738 -34393731633163396236373038333566346432313566316539616439663939626433653637366134 -37663936643031383834343163373166373362386531373462643638656333373136356331306662 -33323533616466343165323233323238613963373035343335306636353838656337653835663835 -66383438653238316532323535393638643333623931363561663539346535393464623064373032 -63323138353466386538326137316263363166633637343632623162306331313666376666636463 -38616161353463396666313562613165366634316664663030613664366265626235613230623861 -66613532656262636161656463373266346333346636663763646566643836373436653032303133 -37323662363338663533353035653134616562303633303533303132306562346435613561636536 -38303465303331666235663034363066663034373336346539396639383530383534373732616261 -39636535323635623630313139303730663962316530333938626664623238383732303737613330 -61373331656465373639653263343736653534336563636564336137386461636465396633666132 -34386365363830346535323135663664353761646635663236383332636362306666656439383736 -39366661323532393837363139626236633538343264346535323166656530616238343039663166 -34633663396437663930633966653934643739353832636666323337363861373162326162643137 -66666633663561396162613731323331363565346435376233336132663566623462306233653836 -64663733653636646432366430353762336234333738623062623235616131626538646462646430 -33383132636161613136313061613766336235393635366232613766306532366531636265303239 -34396537656335383864626330643133313563656334666430346365363939616533643038383136 -36376537373162303265356431343432643862346634386332313662303835306239656566383531 -63396666393437636564323664663164643631326539616437343737346635363530333863393761 -39346537623966626638613166383733623338383139323431306663363433653064323738323765 -39313366353532643634643733623337666632346561333262326563636462356463326432376265 -30376337346335356362623430353161366435373032643633306136343665656335373766363166 -31396330343134356533616134613261316138386533653661626364343939613635626161636565 -63343035303637306564323035353865623166316539363938353461663137636539663964346337 -33663966316466373466643462613834663734393466653830393761386535616531613839303435 -64353062313731386663353938646363643164366163623537383236623635333239343761626463 -36666237613564303736306133326266386362613436653461356131636666613936663131616434 -32383831386262643761316630623939303064393433663063316463376235643031623631313333 -36383635386630626537643438623065386232616634303564353538613366333062653365633632 -62633137323831353631623962633533623364343031373565666434346364653830383430643366 -31643935313334623838636131316135373663626132336365336435613732326361616664623633 -35656265333264326436323262633136313563376334623339386461346131636636383165353632 -62356135663230383534366465663537336365366162346464616530393937616637623466663661 -64653361386633383433326639643134666336616234356462383231376637643262646163313366 -34306666383437383633633931303035343732663661643139663361653834376663323034306535 -62343462653264633235313561326633613936303164323366313164303062366331346132376331 -32346137383665353839393530323638653364633262383234633933383038633763383731363036 -63306564643233653631353033623162393763313661366437396139643332633166663265636636 -32646136326138663364326664656466363330656365636463306139626137663436316238393865 -63326338356332633166636430313934626438346235303161323438633162343334383738313562 -35356162366134323336643038363039633662643932316630303663373962643235643539383165 -36633638383963363032373338633165353131393466623462646538373762613364666239636666 -33666563303565313639373862326163323238373434393265333231646337643631383065323464 -62326131663831343732333239316639336536383032323831343961373761383066323366323434 -34346136633164663333353935663530653532346464663134366563346635383966323434353762 -33303561396538303465386533376432623734666166313465353039613563376262303462396163 -34663630316230633165663939333239656635666136373363306335633835353533623366376165 -36623032643666383831336134653037333333366664383436306566333366343538333734393433 -64353866396239323561313863303966323266663139653738353530303939373239633639653831 -66663464323932643633396665393231633765623561373663336631353430616230626563323861 -66373665393764653864613263333961356462616163373933623739633931323432393162343138 -64373630646437306534363233623530633563353962653130656531393137346337613365636463 -34646531386236666163393537353863626266633961653032353533353036656433663734323935 -62356434613034653966623437633364663433666537363135396534346330333832343166333731 -38323664396665313439653637333037646461633033643635643935333263356633313761323137 -33643737323938373332626661363761336236303265353261383263333439633930356565656236 -36383830313961646363363234343165633537306562643961313139316337633433386465613535 -35333038333631336663613331393837386266346137386435313031666335386439353961613062 -38656362313435313735623931626362316634383264613664643235323933363935643464353634 -64353431623033636537343061396433613365376536623933363632626639626231323736323339 -65316131626336613931333765303562316562623962353530643533636164306135386331353734 -37316266313463343034393130363732333737633033356463393564323166383533353132336432 -34663162386335613233363764396233356661633732613236303564313061653337303833373133 -66383136656337396337393233313661386538663338663933303364636566363436663164623531 -36616463646238386463623133386337623936386266373164626439303335346563333363373038 -63366233643463646431306330383537623434303534306336366163393431393538303835616261 -65336134613331363035353162303435663863653039353561643432316631626261363935663062 -31646663653535653363336539633336346561653664653764373463363364343832396665663163 -35303464323432303266333765346130396636386236376336363466633064613363633431623733 -32313863353936303932653564376163323833636133356633343065633332303064343838313561 -36363966663938386263613436663435376431376534303133643631323734373363326461336365 -64613933393539373165383536313564666163653639356164366566663437656661373836643861 -34323830316165313336323538336430333564306264616462326338323762336138303963366537 -64623761303537616439363039316365623739633435386563623832306537323232306437353932 -34323531373366633830626132393135636661656637363961363836383232326534386530353438 -63343235303463303437353537646666343434626531336639396531633939346330663831313234 -66323830393739316633646538303163326438396565376139653663326133613937663531343466 -64653161646262663539653036303832333431323165656263623831323734616438633631633330 -65346139326532393662313331646135313063653839623539636361623037306239383034313733 -65393161623935306631366633643363646539326561666631313138303937616230653831616531 -61653639363763623266326164373638366565313164373539393364323031663533623561373431 -30353162343731363336373230346663393164626663336461663065613835633631613036623538 -65303632306435613430643634353135383865333934653034326237313139313162653136323939 -39623731666230316230333539303131363462633530366361656235366563396438643836383832 -36303430303934323166353735393933636162333439313863626135643433323032613333363665 -35333132303138323538386163633264386132323439346335333237343738366130326438353161 -65303036396662643033356461656564326566323261396333336366633066616630643637363730 -39643663643162366130323333376561343037656263623863653665663431343061363436613838 -39326235336439646533626334626131656230306635323433626364336337663438646465356132 -62346463336534303863616363353638623465636331353961346137373934323363356466383864 -66653839623835393136326636666233666630646339623035666339366330666533343335616431 -35656437356136343437666563326438373561386264396561306361636366326530373061353061 -35626662376130306464356662646436383265306434313638326165623563343539366532363166 -37383564316334363365356163376338356631363365306237303562316366623237643639646636 -32353663373564633030373639343639623661613364363361636234646137326264363236346234 -62383938353636346134636666386263386238326366343864346138383036656538363265643365 -66613431613438346535383861336435653536663765633439653461336333333962396237353839 -62343738656232643564333132323634653238386437383039383064346435646263663534356438 -38613839333134346233353930303030656338363337393261623731623564343135653966656462 -35313439393165343333623832626630316164376134616563383362646630663036316336373931 -39633135616463633036366163623063656563303635353165653631333532396364643465636236 -35656161656238366532383666373033373861323939386465613737616434643061333338353339 -32626431316133306631313733376137306537633237353037393664363635303738373130313635 -32646362643432353430306132383233633739316562633762623062356163356537346132636332 -32393031663631653036323233666331633764386165643363316232623337613635306465376534 -65643030343862343636616330343462323666323235656333373331623361383464363161653839 -37363665623061363637363764313831636532653039616164663566366135323831653635626538 -34386537643361633430343139613232636563336666643138633166393333383635626237626339 -35323539666338636262663461613362313736353835623861313365656633303533393832343661 -65366262636230383139363932616361326336363166353137633165343838663162343664333136 -35323162363337613861303133373637666664353233666133323363623261396135343131623361 -34326666316230366131336133303439373938656365343165336133383639316338373334313638 -33326438636630643064623237663037336337373864363262363034376331353738636166646566 -63343062346538396434613235643764623630346366373362366161343835303532343033356434 -34323362306265353662363164306361613137363763303338643433303665333030643738376333 -65653865323362383436633964366538356333343532663833653630373265366134303464656531 -34306363613033366664336133346162393339326265626530336136626430336365396130663161 -62393666656163323030616137646636623232626634623061633363393632353338323137656538 -61353861323662623064653434353430646265626263646563353263356536376238313331623566 -33333338393430306138346364626239306339653361353266653566346439333434653932336163 -66643763306463313062323932383863363635346531666232363938303534623261336137363066 -30383266313137313064383836356634666532313565623737346232313230376437643164666363 -31386466356237633030393530656536653834363734666235386131336663303938346562623866 -38313439653036633133663535616334356133303265636631623163333638646164303732323564 -34623764323833323932643234393235353265316535623430333336353231373764396336636362 -64326139633762306436646232313536306530653265363430353465653637643561656536653964 -37383332363234636534313734393230626132613461393632323861633039663764626431363931 -61373138333830336431313232323338343064353932663932623934656235653463326365636165 -62313835396539666434366232363639653663326132653033346265613739376131643764333363 -61386239653166316234353763376632623132383232356636366131316236613638636264336530 -33343734366338306336356565323166346633363237643637366565663837326461393333373135 -30306633653461636633626432343032613661326431376630646332356534633335346466353838 -36383431656538616331316236316463353666666166313465336534393234396438646335323562 -32336262353432663331363532623735373834666139626464356565323335633036623939643234 -61623363653536383931366661666534333561666662313564383362623934393664396465366561 -65333233316432666539303832363735393666373231646530366566306432656433343036666131 -35373636343631643730343235653234326638663565636237663930663364623335343064666265 -37323165333261373532313136326164663962363133363634333036383137616431346230313135 -37386133343062653831353165323136633738386439396139353033333664306262616361623762 -65643039646466303364363434356534323866393432616362636439633334353562363065336533 -63373335383965653365666539636161363361363336376336383363306337386230366664656330 -65313161343638366564336136636438393365366330636232306130313033653036666537653964 -32366133313263306466363266616234643039366638633463346362323839316133343461333464 -33643034626437363233353331653839646361346132386431636562646236353166633836383938 -62323364303338623133323931613466306537623563646232306134373566313865363335346266 -64636638316165343861316134663066386133316166346564613030326263643130386137333739 -62643664383834346133643965366538303965373132393334623161336264633834633663333465 -6234 +61316365323636633262643334366235633639633534646562353032666465623937323833313561 +6562643130663731633435646332663365373265346635330a636266303066356364353431393335 +32626431366331363338363237663862643464393937626530366332353735393866366261643465 +3036623765663362630a343439363134386236633236336332626366386461373965646266366234 +65643765343438666339633738303566343661636134386234363261306335393061316463373332 +32613630393438626337623837373765633462336430663561663433363531373635323365353566 +62333233313835366534386263623132656363623235393030363833633665646633633336373435 +31636437393137323236383330653265336333663635373331383866393435316139393635633339 +61303532626537346336653836636263373466386139343734623937383161333561656164363439 +37316631393536373635633239393939356437383633646261663536306239646434646664323866 +35363332306538613265326239613931343934303733653133613639363431666434343832616534 +34396362373939376335323766323633333733393638626231336432653262326437346562643761 +33646563393632326166363463373734653730316439343931376566363065643034393365653937 +64393231363839336531326164623637383162646163326532643132393163343365393135393930 +65323464643465616235323165373465373061386135323530323761373339653433666162383139 +66636136663563613964643761623332333438373632313832653430323066663537343165373061 +66303864653563386536383430343130383863623238643232636134633964346234353134333031 +37626232656130343863636463663839373161626364356561343434623832396632653064313963 +30616534633634313933616334333537646663636433636664313730303733636162323233353162 +35373264316166636535313735643135356437353332333864393733653462393438653262656662 +64616565326533353530366536636330643733346264306536616636313935633863613032666336 +66393338383933323166383862653635326666316132363036383032626162653664643761613562 +36653066623332336632306637376437623164643965353733633765646266323833376364346435 +32613538336465386465643035633630663164663136666638333333636638333164626237633439 +32636363393362656230356261363839326266376233326237393634393433323739386365363266 +61363132313137646630306131653339313833633739366338333335613966653563306331363361 +39396538373831356436313837343066363033306563636364303738666163336561323661303331 +34633739343465326335613039663537616233626138343730643234346333633666643264313566 +38616462336265373737356561376336626366323238313239333535376561303033376536643966 +33333563336262396433336433346132616530363039663164306234363231626331396662663334 +31306337313533343239383063613532303132346638663035656635393733323031303937356636 +37633334356235613437656439626261613565643538613435616364633334663034666464386565 +61666530633138346664313237393130326263623032346539646365656631316334353530633864 +65306264343233373564333064396461623232363462306531653838333164623239633134363464 +61346134643937333261633632396638396532363035373130333237303137353364653564643438 +62373736303636386637616436396565313364663738356466313032316264373563373531626561 +65653530376234393730623035643936393839313065626661386263346630373362323965653865 +66333964363738326334346338613934613435616331633566653965346138346135313837343861 +39376462343933383063336463396632616338646666313737666437346330346663313066306565 +65303130346664633336353964633234643233623764336563393537663035316162303437313061 +33383131333433613361663335623439313436306339616333346364353064306238353535636639 +35353861383738633330656565616335393432373632323630633038303261306263623466643633 +65623530373137363866356335363238306439303237353764346561353339373562336266363435 +65323663343565393832646539633037396463336136313066646132333731613561633763616337 +31393265636232313333393263653230333861363463653365643661363235653838646538373065 +34303337663837376231613636626361616337613433626435626238376139656234393633343636 +35653537653030613431363436663632356638333036333866336163313832396338303037373561 +35323363346139653933316461333965363764376535653638386361396337323132663366376533 +63666232303038323363303532623833666539356662383236323639313239666633396163346564 +30646438333034663630326539336631323237623936366265366138653564376264336439643262 +38386630376237323231376165666166386231313164333266353062613162643065643164346531 +31393665333035356230396562366231636265353166653162666464363130623962396164636537 +30333538633639326237383832393737616432643339363239353534363438396137356465393138 +37396562623532356131653135366237643734663130326235373236393866663664363337323138 +34646663623430613832623535343535393865653834633733326635373432666565653332343835 +62386631353530323832336566373464323034313161353161633832623263343061643930643638 +38393462633063316232303037653834616264323230313732346161363833396632653732346136 +38613835656361616462386435653766383638626132306663363864633264616235333365643430 +39373863636435323162393031663038643961653330643434316163336635376437373133646638 +33393964626437306235616237323439386338333566366631343966656436393438343563613835 +33386464336330613233613561356437313163323632323166656231376561323763643439653430 +32353634656263303336306233353231636665353734363964343361306337386464373539353235 +66393365643537633563613963663363303830323564656132333534383630663330366436646431 +64653634386662393564383135623036373039653132393664326163303164353065376566613966 +36343230393532303435656137333530363666303130333136333263383131663966323136363666 +36323034333532366161613334643835613737393932393939626234653134623138343630373436 +37303731626565303636373637396434303434346239336432663836303931356536396630323630 +32623631326137363137336663363963326665623632343235323534366235393134663566633638 +36373063616531663762323665366361363131396264643362346665383634393465396562313034 +35366634366364353534613139303239643738666665306435333435383465626437386539393432 +61396463386130313266633263386130306634376464653431646463653933663066633636643465 +37303435663437653562626464313134356339333462336265313535633636366636346232323736 +66386638653231363338656361366237353236376131383563616265633637663464643030356331 +65663835376265336537303364363033373535666137653130333636643666376164613137626637 +30663466363164623535616639646364346462646162383632356137333232393061336238373131 +66383533343631613536326532366639353439373537353732373161636631653838623632313131 +35646666346534666332303662636635353739373462306337323664313036643237333437313936 +33376635643333306538623663306337643965373534316235316561363432646234376132313236 +38373736636266343934643730356236363532656236386631343063616130653464616361633033 +34643565643532626561633664396335326633666563646436333937666537613961633061393832 +66333265326435316264646530323433616266356630643930623961643134353561636534633533 +65376461346161623831326633616163336331326134643162323438616534613564356531653434 +37316634383366376430333162623162636261343466613431613537636637303166336166376563 +61366235333635376264343561336666396535333162633236653462623933303933636530636234 +61316533373032656166303338353031363533353033346339383665333964396635326633373132 +39626235366363633333663366386630623233393030396635633339306535616130323465373030 +62646231613763636538396236343434376131616461313436663034333266353934373730643262 +61386339313364313237306336356330373265663131343066333435353038653036306435383764 +65636262353038646639383930323037346531663065666563663063353563363138613231613561 +64366139303835306432383066356661363836356231323561636364313734366331393737333136 +63633039336634326239303635353765303530343336386231633933653265616161646434653264 +61333865333735336438313434323237326133613734336430623230663834343033353064633533 +31303064623930376532333034363763383061633831633564343065373730303132336331363965 +34353661313239346263383364373663356435353837303164633330393965336466393632396663 +36316563313632333335666462303434633731616232343430383362363131613762393165343432 +36303033656139353261646561663930656163393639343938633365623161393638653330373736 +37313437626465663132343534663835303237396632336561306437366330356538666565653231 +38306638396139376239376164316661363863383137316363626662346264333731316136353832 +63666339303631313564396262653532653361636136626633346132333263366532396430323039 +64313761633265616364353264336235653235373437346139303239333839306334356436616330 +34666231343565383136373938326434653565656133366130663739633266646339613838343233 +62303438663461653434643132633033393061303839356163646666363036656365353330613831 +65613834353935393739646636633338316338313963643438643663613763393031363165376362 +63336132626362653334666161366565333563616363653565383035666234323839363634303631 +33666533376263396661616366313166376162653966613039356463303564626332353734326666 +31356430383032343463363830613337386164343233373162356162326238303633303266646338 +39376332663066343438343265633139653633663036356134323034613939663836383931626138 +38393164356538616431373934333966356330346365383635343333393737336666326639336631 +31323232356162333639643236393433666263353461633163323965663532363339653466313134 +61383838356237386165363230353563376130366437323237323935343135656562383131653365 +38643630616132366332616461333332343339363934313966326263306238356166313764636365 +36643439643939373533393462626231353964323062376433666538376637636637313431393932 +66656461343839386365303064366337393330366465653062353631333265313934343937643239 +61303063303164316538333330353366373263656365376338626531343761393961313762613737 +39323036343564646538616330643235623233666236393739336138353364623630303538336437 +37353033356634333636323561373166626130646562663331383331346439636131626238373334 +35663962386661303866366535366633373839323761326237663633373235663539323335653938 +64366530333964363233343161326138626231383737393130616466376463323263376365303762 +35303639393739666561613632353431333037663837366162303266363837613335396362343631 +39643464306338663832346336333663353738326462353038306631653165363835373864363662 +35626339623237653863643938653538646338333730633431623836636430376663653163303238 +35613965623439343162333132386536643062363137306435666631643261653038366131383935 +38613634666330383432656539353734336630396333393166636661636239333864643636336166 +34633335336534366430326139653030313735626265363030626337393935313866393539643863 +33666130306638373535333562656130626561633062666236613764376330306164396165346432 +36346638363530303262373334313339633930613363663265313937663866316634303132373936 +62633062363934666264633264646334616339666331363637386538633338636131333030633534 +37666239336366316330363961363566666361333436383866636161383030396130653965376636 +65373337323937353331613861366532313565353963333235616264376563323535346162336537 +61303032643566346231306232313062666164326133633536643963343063336438333761333932 +36636531653230616234386662333938353230376337656366386634306336343464366536643263 +37613234613730393039613835653935356363313363353837383236616335653031303231383963 +34396637346661363737646536666133376562636364663832346362356535333162663863386635 +35633836313232356235373938303761376539633437623235383062356536373533646630643238 +39303165643266333364323331363737636135656236633538383266376663366539363834343361 +37323230623038663765623665313264656336646239613165326231333830646637353262636332 +33643764303730663733353338623035343566386663666364653861656531653838643535653163 +33666639306165656231343830653735376661383762616532396632316135343239353165663534 +65633165343132653466613034343231363339396431326334363163633232313236626237663230 +64363635646463313061643365633935393038353761656461346262303366663363343163333532 +66393665386561323236663537636563636163613138666464633038313763376236636235386430 +35333837666663633035646630306666343631373366323564633436366562326135346563376437 +32363533396231613833653431326536663966363430396165663662623236353638383830366336 +66343836303930353866363036326461356166626131306230653665343564376364343662393261 +66383765656462386537633162613066616164353933336533396232373531346130363531663632 +36336236343233333530353732646334626239616164616237373262383230663034656462323336 +34666634636562636661383131373339633866376563383131326633353438333935643030666133 +64343661613737316563623461316430373930343137366132376235303963323339353738393331 +63353335363938613763383561616337623838363336303263346366666532653635643164353865 +39623666356663643238373337663564636230303766626135653534303637626261663930393932 +37656637323737326362623634356135316336383930373662633139343866353131373461656237 +38613133643338373966343131656538343564363530636162623739356165666265393664356131 +64376365336263316138323737383337326437643934396439383439333139386432646662623764 +62346235323865346339333565653363313231366138306365616161346335326633386234663937 +66626132353665663734373038303030646331613534623733646565353730356233613035626238 +35396366646161623037656437393264613962306639393466333039666637663930613939393464 +39316166323632316238653733626137343161343438396665653530633466666161626363396233 +34633732326166313066333536353834386634396233333061323963623234626230646565313762 +37653763333434663734646439383934653666346134343832353134353162623931373139626565 +36623063653265333863643265646338376332653139376661343334376663646438643563316432 +34346432333530643761643832626530353637303933656434306436343830633939663261333762 +35643037626533633039363835303432303532626234656437346338623862353131616661326561 +63636134663634303935613831306432343334376362656162366139393564613338653730353039 +39343832653764326663613531623337346133316566666161336532313430323030323266383437 +65346332346633373231336335363637663638643363653937333038613233653863346137626266 +62653131623263353262633230333064663736626138306266383964633533616530653861623639 +35633966623632396639343937633061626436366561303161656364303162323033633536323234 +34653830636363306236616135343538363337643362636430656637646236386233326634633033 +38623234653739326434336234626630613931363934383864666634396261396130663139623764 +36313734376564393665636664373231336263653933303663326534366462363732303331326431 +66666437666265313637633439653939356238613761373130396232313961643037363131643931 +63633261323863663335373166343137656662323736386133383634336232353334303634386163 +61616439626364623331323563633436323163396532323439323933383666396630343437353332 +32366133653837396361623432356539656337633333303134626466646364393439353439666363 +32613664303633356362313764373666393562646437636562363131376333323265666431643465 +65353765376661386461633561613934396331336363396563373932623938306463653239333236 +64326339323866333031303239313634666564353436316232633335623066376236646164316363 +38633435306238653234613665323134316562323463643762643032346232636361633430663933 +35343332656139663230316234613436643133326665396138343461383264393061653333663737 +30613633306439653964386233353533646436303063366639326465656436643431393232356536 +63353539613430376264386130643232616638306362373761623537643562663166356230653630 +66333136323836653038623562313832626662616432323434383463336366653562313666323565 +38306335343034653661383137326163643536353631646432396336656264353637636562316630 +66396238396536663736386130666461613664373062623833386433356363313934313963383932 +66323835303264393132613161323465636635393038356166353563376634343531623930663361 +37396464666265393430363965363631396332326166623130613734333531363535626433396561 +66366234366435316333323831653739306131633430363564313730303735653738386463346261 +62656330303335396236393632343038316266323439646433663631303564356237353138326230 +38623865656666626438383533366639343366376463343133353963646136323438313236303935 +34646635343734633436356265633663363133636632393138316334396265303363636432373264 +36646534323333616133343830623139343963626662656134396336643737313039363631626237 +36643637326136316461303136373739646665646337663339613364633534346234636534393736 +32313430653632373461613965326363356163396261613366336136616661393132633761333535 +33616662633766356239343262343263623138656137633061656234353938303936653634376635 +31323063353232313339323966313061623764613361316139336261616666376630356437313465 +37376263666534306662356230633262363039303631633435613234313633356436653539303564 +3131 From 1611f4a93cf01ce475ca1021b3c4a69bf8778abd Mon Sep 17 00:00:00 2001 From: Otthorn Date: Fri, 12 Mar 2021 01:59:03 +0100 Subject: [PATCH 17/44] [utils] move the sudo_upgrade playbook with other utils --- sudo_upgrade.yml => utils/sudo_upgrade.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sudo_upgrade.yml => utils/sudo_upgrade.yml (100%) diff --git a/sudo_upgrade.yml b/utils/sudo_upgrade.yml similarity index 100% rename from sudo_upgrade.yml rename to utils/sudo_upgrade.yml From 2f2f71422f13d98e12eef0901c278743e54725d2 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sun, 14 Mar 2021 13:16:08 +0100 Subject: [PATCH 18/44] [borgbackup_client] Move some handlers to tasks --- roles/borgbackup_client/handlers/main.yml | 12 ------------ roles/borgbackup_client/tasks/main.yml | 12 ++++++++++++ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/roles/borgbackup_client/handlers/main.yml b/roles/borgbackup_client/handlers/main.yml index 426cc4e..60f493a 100644 --- a/roles/borgbackup_client/handlers/main.yml +++ b/roles/borgbackup_client/handlers/main.yml @@ -2,16 +2,4 @@ - name: Run systemd daemon-reload systemd: daemon_reload: true - -- name: Start and enable borgmatic timer - systemd: - name: borgmatic.timer - state: started - enabled: true - -- name: Start and enable borgmatic service - systemd: - name: borgmatic.service - state: started - enabled: true ... diff --git a/roles/borgbackup_client/tasks/main.yml b/roles/borgbackup_client/tasks/main.yml index b396017..090bda3 100644 --- a/roles/borgbackup_client/tasks/main.yml +++ b/roles/borgbackup_client/tasks/main.yml @@ -81,6 +81,18 @@ - Start and enable borgmatic timer - Start and enable borgmatic service +- name: Start and enable borgmatic timer + systemd: + name: borgmatic.timer + state: started + enabled: true + +- name: Start and enable borgmatic service + systemd: + name: borgmatic.service + state: started + enabled: true + - include_role: update_motd vars: key: 10-borgmatic From 8112788396ceadc0477dbab47603118dfff2a8f2 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sun, 14 Mar 2021 13:18:30 +0100 Subject: [PATCH 19/44] [borgbackup_client] Add 'user:' in authorized_key --- roles/borgbackup_client/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/borgbackup_client/tasks/main.yml b/roles/borgbackup_client/tasks/main.yml index 090bda3..2038a2e 100644 --- a/roles/borgbackup_client/tasks/main.yml +++ b/roles/borgbackup_client/tasks/main.yml @@ -43,9 +43,10 @@ - name: Add public key to remote delegate_to: "{{ borg_server_host }}" - become: "{{ borg_server_user }}" + become: true authorized_key: exclusive: false + user: "{{ borg_server_user }}" key: "{{ ssh_key.public_key }}" key_options: >- command="borg serve --restrict-to-path {{ borg_host_dir }}, From c99b611b8f32a8f177b35848234aef650269d387 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sun, 14 Mar 2021 14:17:36 +0100 Subject: [PATCH 20/44] Various fixes --- backups.yml | 2 +- group_vars/all/vars.yml | 2 ++ roles/borgbackup_client/tasks/main.yml | 28 ++++++++++++------- roles/borgbackup_client/templates/apt/list.j2 | 3 ++ .../templates/apt/preferences.j2 | 9 ++++++ 5 files changed, 33 insertions(+), 11 deletions(-) create mode 100644 roles/borgbackup_client/templates/apt/list.j2 create mode 100644 roles/borgbackup_client/templates/apt/preferences.j2 diff --git a/backups.yml b/backups.yml index b87c1f9..b2820fa 100644 --- a/backups.yml +++ b/backups.yml @@ -3,7 +3,7 @@ roles: - borgbackup_server -- host: all, !unifi +- hosts: all,!unifi roles: - borgbackup_client ... diff --git a/group_vars/all/vars.yml b/group_vars/all/vars.yml index a351878..885dc41 100644 --- a/group_vars/all/vars.yml +++ b/group_vars/all/vars.yml @@ -103,3 +103,5 @@ borg_backup_directories: - "/etc/" - "/var/" borg_encryption_passphrase: "{{ vault_borg_encryption_passphrase }}" + +borg_server_host: 10.128.0.4 diff --git a/roles/borgbackup_client/tasks/main.yml b/roles/borgbackup_client/tasks/main.yml index 2038a2e..871234f 100644 --- a/roles/borgbackup_client/tasks/main.yml +++ b/roles/borgbackup_client/tasks/main.yml @@ -1,18 +1,19 @@ --- - name: Pin borgmatic template: - src: "apt/{{ item }}.j2" - dest: "/etc/apt/{{ item }}" + src: "{{ item.src }}" + dest: "{{ item.dest }}" owner: root group: root mode: u=rw,g=r,o= loop: - - sources.list.d/bullseye.list - - preferences.d/borgmatic-bullseye + - src: apt/list.j2 + dest: /etc/apt/sources.list.d/bullseye.list + - src: apt/preferences.j2 + dest: /etc/apt/preferences.d/borgmatic-bullseye when: - ansible_distribution == "Debian" - ansible_distribution_major_version | int <= 10 - - ansible_distribution_release != "bullseye" - name: Don't pin borgmatic if we are on bullseye file: @@ -32,10 +33,18 @@ retries: 3 until: apt_result is succeeded +- name: Create /etc/borgbackup + file: + path: /etc/borgbackup + state: directory + owner: root + group: root + mode: u=rwx,g=rx,o= + - name: Create SSH key openssh_keypair: - path: /etc/borgbackup/id_remote - type: ed25519 + path: "/etc/borgbackup/id_remote" + type: rsa owner: root group: root mode: u=r,g=r,o= @@ -79,8 +88,6 @@ - borgmatic.service notify: - Run systemd daemon-reload - - Start and enable borgmatic timer - - Start and enable borgmatic service - name: Start and enable borgmatic timer systemd: @@ -94,7 +101,8 @@ state: started enabled: true -- include_role: update_motd +- include_role: + name: update_motd vars: key: 10-borgmatic message: Borgmatic (client) est installé dans /etc/borgmatic/config.yaml. diff --git a/roles/borgbackup_client/templates/apt/list.j2 b/roles/borgbackup_client/templates/apt/list.j2 new file mode 100644 index 0000000..ac5140d --- /dev/null +++ b/roles/borgbackup_client/templates/apt/list.j2 @@ -0,0 +1,3 @@ +{{ ansible_managed | comment }} + +deb https://deb.debian.org/debian/ bullseye main diff --git a/roles/borgbackup_client/templates/apt/preferences.j2 b/roles/borgbackup_client/templates/apt/preferences.j2 new file mode 100644 index 0000000..c4d8784 --- /dev/null +++ b/roles/borgbackup_client/templates/apt/preferences.j2 @@ -0,0 +1,9 @@ +{{ ansible_managed | comment }} + +Package: * +Pin: release n=bullseye +Pin-Priority: 1 + +Package: borgmatic +Pin: release n=bullseye +Pin-Priority: 900 From 021a5ef1e8347a3b536f6d166151492fcc1ca7ec Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 16:11:18 +0100 Subject: [PATCH 21/44] [borgbackup_client] various fixes for ssh keys --- roles/borgbackup_client/tasks/main.yml | 28 +++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/roles/borgbackup_client/tasks/main.yml b/roles/borgbackup_client/tasks/main.yml index 871234f..e22177e 100644 --- a/roles/borgbackup_client/tasks/main.yml +++ b/roles/borgbackup_client/tasks/main.yml @@ -44,11 +44,33 @@ - name: Create SSH key openssh_keypair: path: "/etc/borgbackup/id_remote" - type: rsa + type: ed25519 + regenerate: full_idempotence owner: root group: root mode: u=r,g=r,o= register: ssh_key + +- name: Gather server facts + delegate_to: "{{ borg_server_host }}" + setup: + gather_subset: + - all + register: server_facts + +- name: debug + debug: + var: "server_facts['ansible_facts'].ansible_ssh_host_key_ecdsa_public_keytype" + +- name: Add server key to known hosts + known_hosts: + hash_host: true + host: "{{ borg_server_host }}" + key: "{{ borg_server_host }} {{ server_facts['ansible_facts']['ansible_ssh_host_key_' + item + '_public_keytype'] }} {{ server_facts['ansible_facts']['ansible_ssh_host_key_' + item + '_public'] }}" + loop: + - ecdsa + - ed25519 + - rsa - name: Add public key to remote delegate_to: "{{ borg_server_host }}" @@ -65,12 +87,12 @@ no-user-rc, no-X11-forwarding vars: - borg_host_dir: "{{ borg_server_base_dir }}/{{ ansible_fqdn }}" + borg_host_dir: "/borg/{{ ansible_fqdn }}" - name: Add borgmatic configuration file become: true template: - src: borgmatic.conf.j2 + src: config.yaml.j2 dest: /etc/borgmatic/config.yaml owner: root group: root From d928c7f7f00e17d882d7fc3a4f99aa069a707d67 Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 16:11:40 +0100 Subject: [PATCH 22/44] [borgbackup_client] rename variable correclty --- roles/borgbackup_client/templates/config.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/borgbackup_client/templates/config.yaml.j2 b/roles/borgbackup_client/templates/config.yaml.j2 index 0d9b90d..93335f8 100644 --- a/roles/borgbackup_client/templates/config.yaml.j2 +++ b/roles/borgbackup_client/templates/config.yaml.j2 @@ -18,7 +18,7 @@ storage: retention: keep_daily: "{{ borg_keep_daily }}" keep_weekly: "{{ borg_keep_weekly }}" - keep_monthly: "{{ borg_keep_yearly }}" + keep_monthly: "{{ borg_keep_monthly }}" consistency: checks: From c5afbdbde4fad19bb7f559f0a2cb91970d8e6f39 Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 18:22:17 +0100 Subject: [PATCH 23/44] remove borg_host_dir --- group_vars/all/vars.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/group_vars/all/vars.yml b/group_vars/all/vars.yml index 885dc41..cf979f7 100644 --- a/group_vars/all/vars.yml +++ b/group_vars/all/vars.yml @@ -98,7 +98,6 @@ is_aurore_host: "{{ 'aurore_vm' in group_names }}" borg_keep_daily: 7 borg_keep_weekly: 4 borg_keep_monthly: 12 -borg_host_dir: "/borg/{{ ansible_hostname }}/" borg_backup_directories: - "/etc/" - "/var/" From 26514325828807e6d8847abf185866959d2b8a36 Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 18:22:52 +0100 Subject: [PATCH 24/44] [WIP] various fixes --- roles/borgbackup_client/tasks/main.yml | 27 ++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/roles/borgbackup_client/tasks/main.yml b/roles/borgbackup_client/tasks/main.yml index e22177e..b3e5c62 100644 --- a/roles/borgbackup_client/tasks/main.yml +++ b/roles/borgbackup_client/tasks/main.yml @@ -48,7 +48,7 @@ regenerate: full_idempotence owner: root group: root - mode: u=r,g=r,o= + mode: u=rw,g=,o= register: ssh_key - name: Gather server facts @@ -79,16 +79,18 @@ exclusive: false user: "{{ borg_server_user }}" key: "{{ ssh_key.public_key }}" - key_options: >- - command="borg serve --restrict-to-path {{ borg_host_dir }}, - no-agent-forwarding, - no-port-forwarding, - no-pty, - no-user-rc, - no-X11-forwarding + key_options: command="borg serve --restrict-to-path {{ borg_host_dir }}",no-agent-forwarding,no-port-forwarding,no-pty,no-user-rc,no-X11-forwarding vars: borg_host_dir: "/borg/{{ ansible_fqdn }}" +- name: Create configuration direcotory for borgmatic + file: + state: directory + owner: root + group: root + mode: u=rw,g=r,o= + path: /etc/borgmatic + - name: Add borgmatic configuration file become: true template: @@ -97,6 +99,11 @@ owner: root group: root mode: u=rw,g=r,o= + vars: + borg_host_dir: "/borg/{{ ansible_fqdn }}" + +- name: Init repository + command: borgmatic init --encryption repokey - name: Install timer and service for borgmatic template: @@ -111,6 +118,10 @@ notify: - Run systemd daemon-reload +- name: Run systemd deamon-reload + systemd: + daemon_reload: true + - name: Start and enable borgmatic timer systemd: name: borgmatic.timer From ff750c5b635cf37504a66502fd2cfac7fbddf588 Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 18:23:44 +0100 Subject: [PATCH 25/44] [borgbackup_client] remove 1 minute sleep and fix verbosity --- roles/borgbackup_client/templates/borgmatic.service.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/borgbackup_client/templates/borgmatic.service.j2 b/roles/borgbackup_client/templates/borgmatic.service.j2 index f689b66..17e9752 100644 --- a/roles/borgbackup_client/templates/borgmatic.service.j2 +++ b/roles/borgbackup_client/templates/borgmatic.service.j2 @@ -25,5 +25,5 @@ LogRateLimitIntervalSec=0 # Delay start to prevent backups running during boot. Note that systemd-inhibit requires dbus and # dbus-user-session to be installed. -ExecStartPre=sleep 1m -ExecStart=systemd-inhibit --who="borgmatic" --why="Prevent interrupting scheduled backup" /usr/bin/borgmatic --syslog-verbosity 1 +# ExecStartPre=sleep 1m +ExecStart=systemd-inhibit --who="borgmatic" --why="Prevent interrupting scheduled backup" /usr/bin/borgmatic -v 1 From 46d10022ea632799b5a352bed47c72233ef91eff Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 18:24:36 +0100 Subject: [PATCH 26/44] [borgbackup_client] fix rentention date to int and list correctly source directories --- roles/borgbackup_client/templates/config.yaml.j2 | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/roles/borgbackup_client/templates/config.yaml.j2 b/roles/borgbackup_client/templates/config.yaml.j2 index 93335f8..497c2d4 100644 --- a/roles/borgbackup_client/templates/config.yaml.j2 +++ b/roles/borgbackup_client/templates/config.yaml.j2 @@ -2,7 +2,10 @@ {{ ansible_managed | comment }} location: - source_directories: {{ borg_backup_directories | to_nice_yaml }} + source_directories: +{% for source in borg_backup_directories %} + - {{ source }} +{% endfor %} repositories: - {{ borg_server_user }}@{{ borg_server_host }}:{{ borg_host_dir }} @@ -16,9 +19,9 @@ storage: retention: - keep_daily: "{{ borg_keep_daily }}" - keep_weekly: "{{ borg_keep_weekly }}" - keep_monthly: "{{ borg_keep_monthly }}" + keep_daily: {{ borg_keep_daily }} + keep_weekly: {{ borg_keep_weekly }} + keep_monthly: {{ borg_keep_monthly }} consistency: checks: From c6cae750312d1199c0de406d4516b733a0bb31c7 Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 18:29:33 +0100 Subject: [PATCH 27/44] [borgbackup_server] fix /borg permissions --- roles/borgbackup_server/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/borgbackup_server/tasks/main.yml b/roles/borgbackup_server/tasks/main.yml index eb92431..595abe7 100644 --- a/roles/borgbackup_server/tasks/main.yml +++ b/roles/borgbackup_server/tasks/main.yml @@ -34,7 +34,7 @@ state: directory owner: "{{ borg_server_user }}" group: "{{ borg_server_group }}" - mode: u=rw + mode: u=rwx,g=,o= - include_role: name: update_motd From d9f11043098bc90a845d9d85ddaabefa4b32bc05 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sun, 14 Mar 2021 18:42:26 +0100 Subject: [PATCH 28/44] Move id_remote to /etc/borgmatic --- roles/borgbackup_client/tasks/main.yml | 51 ++++++++++++-------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/roles/borgbackup_client/tasks/main.yml b/roles/borgbackup_client/tasks/main.yml index b3e5c62..7e330ae 100644 --- a/roles/borgbackup_client/tasks/main.yml +++ b/roles/borgbackup_client/tasks/main.yml @@ -33,17 +33,28 @@ retries: 3 until: apt_result is succeeded -- name: Create /etc/borgbackup +- name: Create configuration direcotory for borgmatic file: - path: /etc/borgbackup + path: /etc/borgmatic state: directory owner: root group: root - mode: u=rwx,g=rx,o= + mode: u=rw,g=r,o= + +- name: Add borgmatic configuration file + become: true + template: + src: config.yaml.j2 + dest: /etc/borgmatic/config.yaml + owner: root + group: root + mode: u=rw,g=r,o= + vars: + borg_host_dir: "/borg/{{ ansible_fqdn }}" - name: Create SSH key openssh_keypair: - path: "/etc/borgbackup/id_remote" + path: "/etc/borgmatic/id_remote" type: ed25519 regenerate: full_idempotence owner: root @@ -58,10 +69,6 @@ - all register: server_facts -- name: debug - debug: - var: "server_facts['ansible_facts'].ansible_ssh_host_key_ecdsa_public_keytype" - - name: Add server key to known hosts known_hosts: hash_host: true @@ -79,28 +86,16 @@ exclusive: false user: "{{ borg_server_user }}" key: "{{ ssh_key.public_key }}" - key_options: command="borg serve --restrict-to-path {{ borg_host_dir }}",no-agent-forwarding,no-port-forwarding,no-pty,no-user-rc,no-X11-forwarding - vars: - borg_host_dir: "/borg/{{ ansible_fqdn }}" - -- name: Create configuration direcotory for borgmatic - file: - state: directory - owner: root - group: root - mode: u=rw,g=r,o= - path: /etc/borgmatic - -- name: Add borgmatic configuration file - become: true - template: - src: config.yaml.j2 - dest: /etc/borgmatic/config.yaml - owner: root - group: root - mode: u=rw,g=r,o= + key_options: "{{ options | join(',') }}" vars: borg_host_dir: "/borg/{{ ansible_fqdn }}" + options: + - 'command="borg serve --restrict-to-path {{ borg_host_dir }}"' + - no-agent-forwarding + - no-port-forwarding + - no-pty + - no-user-rc + - no-X11-forwarding - name: Init repository command: borgmatic init --encryption repokey From 86f8b31159973d9a124e8072d98e32f888b52045 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Sun, 14 Mar 2021 18:44:13 +0100 Subject: [PATCH 29/44] Delegate facts for borgbackup_client --- roles/borgbackup_client/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/borgbackup_client/tasks/main.yml b/roles/borgbackup_client/tasks/main.yml index 7e330ae..cb1701d 100644 --- a/roles/borgbackup_client/tasks/main.yml +++ b/roles/borgbackup_client/tasks/main.yml @@ -64,6 +64,7 @@ - name: Gather server facts delegate_to: "{{ borg_server_host }}" + delegate_facts: true setup: gather_subset: - all From 0b5562f3f4e55c3825937359342c5709b042116b Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 18:57:44 +0100 Subject: [PATCH 30/44] Add litl.adm.auro.re --- hosts | 1 + 1 file changed, 1 insertion(+) diff --git a/hosts b/hosts index 4191852..7694175 100644 --- a/hosts +++ b/hosts @@ -38,6 +38,7 @@ portail.adm.auro.re jitsi-aurore.adm.auro.re bdd.adm.auro.re bdd-ovh.adm.auro.re +litl.adm.auro.re [aurore_testing_vm] pendragon.adm.auro.re From ebfc4f2a26e40b73a248d24bfa727b3bd8d482ac Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 19:03:44 +0100 Subject: [PATCH 31/44] [borgbackup_client] do update cache --- roles/borgbackup_client/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/borgbackup_client/tasks/main.yml b/roles/borgbackup_client/tasks/main.yml index cb1701d..bcb02a6 100644 --- a/roles/borgbackup_client/tasks/main.yml +++ b/roles/borgbackup_client/tasks/main.yml @@ -29,6 +29,7 @@ - name: Install borgmatic apt: name: borgmatic + update_cache: true register: apt_result retries: 3 until: apt_result is succeeded From 33a1ec02f3187c6e7faddf1a56d11985417b07fb Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 19:07:02 +0100 Subject: [PATCH 32/44] [borgbackup_client] update config directory to be homogeneous --- roles/borgbackup_client/templates/config.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/borgbackup_client/templates/config.yaml.j2 b/roles/borgbackup_client/templates/config.yaml.j2 index 497c2d4..0be06ac 100644 --- a/roles/borgbackup_client/templates/config.yaml.j2 +++ b/roles/borgbackup_client/templates/config.yaml.j2 @@ -11,7 +11,7 @@ location: storage: encryption_passphrase: "{{ borg_encryption_passphrase }}" - ssh_command: ssh -i /etc/borgbackup/id_remote + ssh_command: ssh -i /etc/borgmatic/id_remote # compression: 'lz4' # umask: 0077 # lock_wait: 5 From b14b359027eb8a563dd1b42fd7bd15632a572bfd Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 19:21:15 +0100 Subject: [PATCH 33/44] [borgbackup_client] add exlude path to conf --- roles/borgbackup_client/templates/config.yaml.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/borgbackup_client/templates/config.yaml.j2 b/roles/borgbackup_client/templates/config.yaml.j2 index 0be06ac..c152168 100644 --- a/roles/borgbackup_client/templates/config.yaml.j2 +++ b/roles/borgbackup_client/templates/config.yaml.j2 @@ -5,6 +5,10 @@ location: source_directories: {% for source in borg_backup_directories %} - {{ source }} +{% endfor %} + exclude_patterns: +{% for exclude in borg_backup_exclude %} + - {{ exclude }} {% endfor %} repositories: - {{ borg_server_user }}@{{ borg_server_host }}:{{ borg_host_dir }} From e570ce67b3f7a36c048e7c30fbb6e537af9cdc39 Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 19:23:04 +0100 Subject: [PATCH 34/44] [borgbackup_client] do not backup /var/log/ --- group_vars/all/vars.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/group_vars/all/vars.yml b/group_vars/all/vars.yml index cf979f7..43e5c59 100644 --- a/group_vars/all/vars.yml +++ b/group_vars/all/vars.yml @@ -101,6 +101,8 @@ borg_keep_monthly: 12 borg_backup_directories: - "/etc/" - "/var/" +borg_backup_exclude: + - "/var/log/" borg_encryption_passphrase: "{{ vault_borg_encryption_passphrase }}" borg_server_host: 10.128.0.4 From 59f2c94a61c54de693bd01a849a3052789f36921 Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 21:14:37 +0100 Subject: [PATCH 35/44] Custom borgbackup configuration for the logs --- host_vars/log.adm.auro.re.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 host_vars/log.adm.auro.re.yml diff --git a/host_vars/log.adm.auro.re.yml b/host_vars/log.adm.auro.re.yml new file mode 100644 index 0000000..1431ddd --- /dev/null +++ b/host_vars/log.adm.auro.re.yml @@ -0,0 +1,10 @@ +--- +borg_keep_hourly: 0 +borg_keep_daily: 7 +borg_keep_weekly: 4 +borg_keep_monthly: 12 +borg_backup_directories: + - "/etc/" + - "/var/" +borg_backup_exclude: +... From 7480a7c565cc09c421d5d9542df6bbba50621956 Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 22:02:34 +0100 Subject: [PATCH 36/44] [borgbackup_client] precedence rules and sain defaults for borg config --- bdd.yml | 2 +- group_vars/bdd.yml | 3 +++ host_vars/bdd.adm.auro.re.yml_bak | 8 ++++++++ host_vars/caradoc.adm.auro.re.yml | 1 + host_vars/log.adm.auro.re.yml | 4 ---- hosts | 4 +++- roles/borgbackup_client/defaults/main.yml | 7 ++++--- roles/borgbackup_client/templates/config.yaml.j2 | 15 ++++++++++++--- 8 files changed, 32 insertions(+), 12 deletions(-) create mode 100644 group_vars/bdd.yml create mode 100644 host_vars/bdd.adm.auro.re.yml_bak diff --git a/bdd.yml b/bdd.yml index 7a0bf6c..f3fa7d3 100644 --- a/bdd.yml +++ b/bdd.yml @@ -1,7 +1,7 @@ #!/usr/bin/env ansible-playbook --- # Install and configure bdd servers at Saclay and at OVH -- hosts: bdd +- hosts: bdd,!re2o-bdd.adm.auro.re,!services-bdd-local.adm.auro.re roles: - postgresql_server ... diff --git a/group_vars/bdd.yml b/group_vars/bdd.yml new file mode 100644 index 0000000..8d71e6a --- /dev/null +++ b/group_vars/bdd.yml @@ -0,0 +1,3 @@ +--- +borg_keep_hourly: 6 +... diff --git a/host_vars/bdd.adm.auro.re.yml_bak b/host_vars/bdd.adm.auro.re.yml_bak new file mode 100644 index 0000000..206a504 --- /dev/null +++ b/host_vars/bdd.adm.auro.re.yml_bak @@ -0,0 +1,8 @@ +postgresql: + version: 13 + hosts: # dbname, username, CIDR ip addr, auth method + - [ "nextcloud", "nextcloud", "10.128.0.58", "md5" ] + - [ "gitea", "gitea", "10.128.0.60", "md5" ] + - [ "drone", "drone", "10.128.0.64", "md5" ] + - [ "wikijs", "wikijs", "10.128.0.66", "md5" ] + - [ "vote", "vote", "10.128.0.81", "md5" ] diff --git a/host_vars/caradoc.adm.auro.re.yml b/host_vars/caradoc.adm.auro.re.yml index 726532d..f6a66c2 100644 --- a/host_vars/caradoc.adm.auro.re.yml +++ b/host_vars/caradoc.adm.auro.re.yml @@ -5,6 +5,7 @@ borg_keep_weekly: 4 borg_keep_monthly: 12 borg_backup_directories: - "/etc/" + - "/var"/ - "/data_nextcloud" - "/data_gitea" - "/data_mail" diff --git a/host_vars/log.adm.auro.re.yml b/host_vars/log.adm.auro.re.yml index 1431ddd..42422ad 100644 --- a/host_vars/log.adm.auro.re.yml +++ b/host_vars/log.adm.auro.re.yml @@ -1,8 +1,4 @@ --- -borg_keep_hourly: 0 -borg_keep_daily: 7 -borg_keep_weekly: 4 -borg_keep_monthly: 12 borg_backup_directories: - "/etc/" - "/var/" diff --git a/hosts b/hosts index 7694175..9255734 100644 --- a/hosts +++ b/hosts @@ -29,7 +29,6 @@ stream.adm.auro.re re2o-server.adm.auro.re re2o-ldap.adm.auro.re re2o-db.adm.auro.re -#services-bdd-local.adm.auro.re backup.adm.auro.re mail.adm.auro.re wikijs.adm.auro.re @@ -39,6 +38,7 @@ jitsi-aurore.adm.auro.re bdd.adm.auro.re bdd-ovh.adm.auro.re litl.adm.auro.re +services-bdd-local.adm.auro.re [aurore_testing_vm] pendragon.adm.auro.re @@ -514,3 +514,5 @@ proxy.adm.auro.re [bdd] bdd.adm.auro.re bdd-ovh.adm.auro.re +services-bdd-local.adm.auro.re +re2o-bdd.adm.auro.re diff --git a/roles/borgbackup_client/defaults/main.yml b/roles/borgbackup_client/defaults/main.yml index 2550cdd..0ffec29 100644 --- a/roles/borgbackup_client/defaults/main.yml +++ b/roles/borgbackup_client/defaults/main.yml @@ -1,7 +1,8 @@ --- -borg_keep_daily: 7 -borg_keep_weekly: 4 -borg_keep_monthly: 12 +borg_keep_hourly: 0 +borg_keep_daily: 0 +borg_keep_weekly: 0 +borg_keep_monthly: 0 borg_server_user: borgbackup borg_server_group: borgbackup ... diff --git a/roles/borgbackup_client/templates/config.yaml.j2 b/roles/borgbackup_client/templates/config.yaml.j2 index c152168..396d1f0 100644 --- a/roles/borgbackup_client/templates/config.yaml.j2 +++ b/roles/borgbackup_client/templates/config.yaml.j2 @@ -23,9 +23,18 @@ storage: retention: - keep_daily: {{ borg_keep_daily }} - keep_weekly: {{ borg_keep_weekly }} - keep_monthly: {{ borg_keep_monthly }} + {% if borg_keep_hourly > 0 %} + - keep_hourly: {{ borg_keep_hourly }} + {% endif %} + {% if borg_keep_daily > 0 %} + - keep_hourly: {{ borg_keep_daily }} + {% endif %} + {% if borg_keep_weekly > 0 %} + - keep_hourly: {{ borg_keep_weekly }} + {% endif %} + {% if borg_keep_monthly > 0 %} + - keep_hourly: {{ borg_keep_monthly }} + {% endif %} consistency: checks: From 41eb4461147f6fc8b4d0ccc3a08454d894784415 Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 22:08:29 +0100 Subject: [PATCH 37/44] small fix --- host_vars/caradoc.adm.auro.re.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/host_vars/caradoc.adm.auro.re.yml b/host_vars/caradoc.adm.auro.re.yml index f6a66c2..2416ab3 100644 --- a/host_vars/caradoc.adm.auro.re.yml +++ b/host_vars/caradoc.adm.auro.re.yml @@ -4,8 +4,8 @@ borg_keep_daily: 7 borg_keep_weekly: 4 borg_keep_monthly: 12 borg_backup_directories: - - "/etc/" - - "/var"/ + - "/etc" + - "/var" - "/data_nextcloud" - "/data_gitea" - "/data_mail" From 6f36506a9899ad31f71cefed3511d98b2a68e90f Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 22:09:27 +0100 Subject: [PATCH 38/44] rm a _bak file --- host_vars/bdd.adm.auro.re.yml_bak | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 host_vars/bdd.adm.auro.re.yml_bak diff --git a/host_vars/bdd.adm.auro.re.yml_bak b/host_vars/bdd.adm.auro.re.yml_bak deleted file mode 100644 index 206a504..0000000 --- a/host_vars/bdd.adm.auro.re.yml_bak +++ /dev/null @@ -1,8 +0,0 @@ -postgresql: - version: 13 - hosts: # dbname, username, CIDR ip addr, auth method - - [ "nextcloud", "nextcloud", "10.128.0.58", "md5" ] - - [ "gitea", "gitea", "10.128.0.60", "md5" ] - - [ "drone", "drone", "10.128.0.64", "md5" ] - - [ "wikijs", "wikijs", "10.128.0.66", "md5" ] - - [ "vote", "vote", "10.128.0.81", "md5" ] From a54006c9d4e1406266ccb8160698fc349799d26d Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 23:39:12 +0100 Subject: [PATCH 39/44] Various fixes --- host_vars/log.adm.auro.re.yml | 2 +- hosts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/host_vars/log.adm.auro.re.yml b/host_vars/log.adm.auro.re.yml index 42422ad..675fe4e 100644 --- a/host_vars/log.adm.auro.re.yml +++ b/host_vars/log.adm.auro.re.yml @@ -2,5 +2,5 @@ borg_backup_directories: - "/etc/" - "/var/" -borg_backup_exclude: +borg_backup_exclude: [] ... diff --git a/hosts b/hosts index 9255734..8b71a89 100644 --- a/hosts +++ b/hosts @@ -39,6 +39,7 @@ bdd.adm.auro.re bdd-ovh.adm.auro.re litl.adm.auro.re services-bdd-local.adm.auro.re +log.adm.auro.re [aurore_testing_vm] pendragon.adm.auro.re From f8e5f0cc767fdc33c3f809c0fdf8835d37c96cd0 Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Sun, 14 Mar 2021 23:56:07 +0100 Subject: [PATCH 40/44] Manually cherrypicked from 65c94d8e84 --- host_vars/bdd.adm.auro.re | 8 ------ host_vars/bdd.adm.auro.re.yml | 47 +++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 8 deletions(-) delete mode 100644 host_vars/bdd.adm.auro.re create mode 100644 host_vars/bdd.adm.auro.re.yml diff --git a/host_vars/bdd.adm.auro.re b/host_vars/bdd.adm.auro.re deleted file mode 100644 index d1e9353..0000000 --- a/host_vars/bdd.adm.auro.re +++ /dev/null @@ -1,8 +0,0 @@ -postgresql: - version: 11 - hosts: # dbname, username, CIDR ip addr, auth method - - [ "nextcloud", "nextcloud", "10.128.0.58", "md5" ] - - [ "gitea", "gitea", "10.128.0.60", "md5" ] - - [ "drone", "drone", "10.128.0.64", "md5" ] - - [ "wikijs", "wikijs", "10.128.0.66", "md5" ] - - [ "vote", "vote", "10.128.0.81", "md5" ] diff --git a/host_vars/bdd.adm.auro.re.yml b/host_vars/bdd.adm.auro.re.yml new file mode 100644 index 0000000..e844a5a --- /dev/null +++ b/host_vars/bdd.adm.auro.re.yml @@ -0,0 +1,47 @@ +--- +postgresql: + version: 13 +postgresql_hosts: + - database: nextcloud + user: nextcloud + net: 10.128.0.58/32 + method: md5 + - database: gitea + user: gitea + net: 10.128.0.60/32 + method: md5 + - database: wikijs + user: wikijs + net: 10.128.0.66/32 + method: md5 + - database: drone + user: drone + net: 10.128.0.64/32 + method: md5 +postgresql_databases: + - nextcloud + - gitea + - wikijs + - drone +postgresql_users: + - name: nextcloud + database: nextcloud + password: "{{ postgresql_nextcloud_passwd }}" + privs: + - ALL + - name: gitea + database: gitea + password: "{{ postgresql_gitea_passwd }}" + privs: + - ALL + - name: wikijs + database: wikijs + password: "{{ postgresql_wikijs_passwd }}" + privs: + - ALL + - name: drone + database: drone + password: "{{ postgresql_drone_passwd }}" + privs: + - ALL +... From e12f67c920d3b684c8b5b55abb3bc3da2ae312d5 Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Mon, 15 Mar 2021 01:03:34 +0100 Subject: [PATCH 41/44] [borgbackup_client] ignore some Strech machines --- backups.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backups.yml b/backups.yml index b2820fa..4886548 100644 --- a/backups.yml +++ b/backups.yml @@ -3,7 +3,7 @@ roles: - borgbackup_server -- hosts: all,!unifi +- hosts: all,!unifi,!unifi-*,!wiki.adm.auro.re roles: - borgbackup_client ... From 243ec1fe9d6915516e7ce1818a0484f1031926b4 Mon Sep 17 00:00:00 2001 From: Solal Nathan Date: Mon, 15 Mar 2021 01:04:42 +0100 Subject: [PATCH 42/44] [borgbackup_client] VaRi0u5 f1X3s --- group_vars/bdd.yml | 2 + roles/borgbackup_client/tasks/main.yml | 47 +++++++++++++------ .../templates/borgmatic.service.j2 | 2 +- .../templates/config.yaml.j2 | 26 +++++----- 4 files changed, 50 insertions(+), 27 deletions(-) diff --git a/group_vars/bdd.yml b/group_vars/bdd.yml index 8d71e6a..f70b5eb 100644 --- a/group_vars/bdd.yml +++ b/group_vars/bdd.yml @@ -1,3 +1,5 @@ --- borg_keep_hourly: 6 +borg_backup_exclude: + - "/var/lib/postgresql/" ... diff --git a/roles/borgbackup_client/tasks/main.yml b/roles/borgbackup_client/tasks/main.yml index bcb02a6..defe34f 100644 --- a/roles/borgbackup_client/tasks/main.yml +++ b/roles/borgbackup_client/tasks/main.yml @@ -34,13 +34,13 @@ retries: 3 until: apt_result is succeeded -- name: Create configuration direcotory for borgmatic +- name: Create configuration directory for borgmatic file: path: /etc/borgmatic state: directory owner: root group: root - mode: u=rw,g=r,o= + mode: u=rwx,g=rx,o= - name: Add borgmatic configuration file become: true @@ -62,24 +62,43 @@ group: root mode: u=rw,g=,o= register: ssh_key - -- name: Gather server facts + +#- name: Gather server facts +# delegate_to: "{{ borg_server_host }}" +# delegate_facts: true +# setup: +# gather_subset: +# - all +# register: server_facts +# +#- name: Add server key to known hosts +# known_hosts: +# hash_host: true +# host: "{{ hostname }}" +# key: "{{ hostname }} {{ facts['ansible_ssh_host_key_' + item + '_public_keytype'] }} {{ facts['ansible_ssh_host_key_' + item + '_public'] }}" +# loop: +# - ecdsa +# - ed25519 +# - rsa +# vars: +# hostname: {{ borg_server_host }} +# facts: {{ server_facts['ansible_facts'] }} + +- name: Gather SSH host keys delegate_to: "{{ borg_server_host }}" - delegate_facts: true - setup: - gather_subset: - - all - register: server_facts + command: "ssh-keyscan {{ borg_server_host }}" + register: keys + +- name: DEBUG + debug: + var: keys.stdout_lines - name: Add server key to known hosts known_hosts: hash_host: true host: "{{ borg_server_host }}" - key: "{{ borg_server_host }} {{ server_facts['ansible_facts']['ansible_ssh_host_key_' + item + '_public_keytype'] }} {{ server_facts['ansible_facts']['ansible_ssh_host_key_' + item + '_public'] }}" - loop: - - ecdsa - - ed25519 - - rsa + key: "{{ item }}" + loop: "{{ keys.stdout_lines }}" - name: Add public key to remote delegate_to: "{{ borg_server_host }}" diff --git a/roles/borgbackup_client/templates/borgmatic.service.j2 b/roles/borgbackup_client/templates/borgmatic.service.j2 index 17e9752..0dc140b 100644 --- a/roles/borgbackup_client/templates/borgmatic.service.j2 +++ b/roles/borgbackup_client/templates/borgmatic.service.j2 @@ -7,7 +7,7 @@ After=network-online.target ConditionACPower=true [Service] -Type=oneshot +Type=simple ProtectSystem=full CapabilityBoundingSet=CAP_DAC_READ_SEARCH CAP_NET_RAW diff --git a/roles/borgbackup_client/templates/config.yaml.j2 b/roles/borgbackup_client/templates/config.yaml.j2 index 396d1f0..a8eb2cb 100644 --- a/roles/borgbackup_client/templates/config.yaml.j2 +++ b/roles/borgbackup_client/templates/config.yaml.j2 @@ -12,6 +12,7 @@ location: {% endfor %} repositories: - {{ borg_server_user }}@{{ borg_server_host }}:{{ borg_host_dir }} + borgmatic_source_directory: /var/backups storage: encryption_passphrase: "{{ borg_encryption_passphrase }}" @@ -23,18 +24,18 @@ storage: retention: - {% if borg_keep_hourly > 0 %} - - keep_hourly: {{ borg_keep_hourly }} - {% endif %} - {% if borg_keep_daily > 0 %} - - keep_hourly: {{ borg_keep_daily }} - {% endif %} - {% if borg_keep_weekly > 0 %} - - keep_hourly: {{ borg_keep_weekly }} - {% endif %} - {% if borg_keep_monthly > 0 %} - - keep_hourly: {{ borg_keep_monthly }} - {% endif %} +{% if borg_keep_hourly > 0 %} + keep_hourly: {{ borg_keep_hourly }} +{% endif %} +{% if borg_keep_daily > 0 %} + keep_daily: {{ borg_keep_daily }} +{% endif %} +{% if borg_keep_weekly > 0 %} + keep_weekly: {{ borg_keep_weekly }} +{% endif %} +{% if borg_keep_monthly > 0 %} + keep_monthly: {{ borg_keep_monthly }} +{% endif %} consistency: checks: @@ -45,5 +46,6 @@ consistency: hooks: postgresql_databases: - name: all + username: postgres {% endif %} ... From cb3ec07121f683f65ba5dc1a10487139a0b935aa Mon Sep 17 00:00:00 2001 From: Jeltz Date: Mon, 15 Mar 2021 07:25:09 +0100 Subject: [PATCH 43/44] Use 'inventory_hostname' instead of 'ansible_fqdn' While 'ansible_fdqn' can be changed by a compromised host, 'inventory_hostname' can't (hopefully). It should therefore no longer be possible for the said host to access the backups of another host. --- roles/borgbackup_client/tasks/main.yml | 29 ++------------------------ 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/roles/borgbackup_client/tasks/main.yml b/roles/borgbackup_client/tasks/main.yml index defe34f..f96b9a0 100644 --- a/roles/borgbackup_client/tasks/main.yml +++ b/roles/borgbackup_client/tasks/main.yml @@ -51,7 +51,7 @@ group: root mode: u=rw,g=r,o= vars: - borg_host_dir: "/borg/{{ ansible_fqdn }}" + borg_host_dir: "/borg/{{ inventory_hostname }}" - name: Create SSH key openssh_keypair: @@ -63,36 +63,11 @@ mode: u=rw,g=,o= register: ssh_key -#- name: Gather server facts -# delegate_to: "{{ borg_server_host }}" -# delegate_facts: true -# setup: -# gather_subset: -# - all -# register: server_facts -# -#- name: Add server key to known hosts -# known_hosts: -# hash_host: true -# host: "{{ hostname }}" -# key: "{{ hostname }} {{ facts['ansible_ssh_host_key_' + item + '_public_keytype'] }} {{ facts['ansible_ssh_host_key_' + item + '_public'] }}" -# loop: -# - ecdsa -# - ed25519 -# - rsa -# vars: -# hostname: {{ borg_server_host }} -# facts: {{ server_facts['ansible_facts'] }} - - name: Gather SSH host keys delegate_to: "{{ borg_server_host }}" command: "ssh-keyscan {{ borg_server_host }}" register: keys -- name: DEBUG - debug: - var: keys.stdout_lines - - name: Add server key to known hosts known_hosts: hash_host: true @@ -109,7 +84,7 @@ key: "{{ ssh_key.public_key }}" key_options: "{{ options | join(',') }}" vars: - borg_host_dir: "/borg/{{ ansible_fqdn }}" + borg_host_dir: "/borg/{{ inventory_hostname }}" options: - 'command="borg serve --restrict-to-path {{ borg_host_dir }}"' - no-agent-forwarding From 3f3f688da42a1e0882232e2be6533f8f5efa9ef7 Mon Sep 17 00:00:00 2001 From: Jeltz Date: Mon, 15 Mar 2021 07:51:48 +0100 Subject: [PATCH 44/44] Use 'present' instead of 'latest' (ansible-lint) --- roles/borgbackup_server/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/borgbackup_server/tasks/main.yml b/roles/borgbackup_server/tasks/main.yml index 595abe7..9c77196 100644 --- a/roles/borgbackup_server/tasks/main.yml +++ b/roles/borgbackup_server/tasks/main.yml @@ -2,7 +2,7 @@ - name: Install borg apt: name: borgbackup - state: latest + state: present update_cache: true register: apt_result retries: 3