Merge branch 'master' into 'change_default_soft'

# Conflicts:
#   roles/baseconfig/tasks/main.yml

.gitlab-ci.yml

@@ -1,6 +1,18 @@
 ---
-image: quay.io/ansible/molecule:2.19
+image: python:3.6
+
+stages:
+  - lint
 
 yamllint:
-  script: yamllint -c .yamllint.yml .
+  stage: lint
+  script:
+    - pip install yamllint==1.15.0
+    - yamllint -c .yamllint.yml .
+
+ansible-lint:
+  stage: lint
+  script:
+    - pip install ansible-lint==4.0.0
+    - ansible-lint *.yml
 ...

README.md

@@ -1,5 +1,12 @@
 # Playbook et rôles Ansible d'Aurore
 
+Ces politiques de déployement nécessite Ansible 2.7 ou plus récent.
+Le paquet dans Debian Buster est suffisamment à jour, sinon vous pouvez l'obtenir de la façon suivante :
+
+```bash
+pip3 install --user ansible
+```
+
 ## Exécution d'un playbook
 
 Pour appliquer le playbook `base.yml` :

matrix.yml

@@ -2,8 +2,8 @@
 # Install Matrix Synapse on corresponding containers
 - hosts: synapse.adm.auro.re
   vars:
-    mxisd_version: 1.3.1
+    mxisd_releases: https://github.com/kamax-matrix/mxisd/releases
-    synapse_rest_auth_url: https://raw.githubusercontent.com/kamax-matrix/matrix-synapse-rest-auth/master/rest_auth_provider.py
+    mxisd_deb: "{{ mxisd_releases }}/download/v1.3.1/mxisd_1.3.1_all.deb"
   roles:
     - debian-backports
     - matrix-synapse

roles/baseconfig/tasks/apt-listchanges.yml

@@ -6,6 +6,9 @@
     name: apt-listchanges
     state: present
     update_cache: true
+  register: apt_result
+  retries: 3
+  until: apt_result is succeeded
 
 # Send email when there is something new
 - name: Configure apt-listchanges

roles/baseconfig/tasks/main.yml

@@ -3,11 +3,7 @@
 - name: Install basic tools
   when: ansible_os_family == "Debian"
   apt:
-    name: "{{ packages }}"
+    name:
-    state: present
-    update_cache: true
-  vars:
-    packages:
       - bash-completion   # for bash users
       - zsh               # alternative shell
       - sudo              # to gain root access
@@ -19,6 +15,10 @@
       - tree              # create a graphical tree of files
       - acl               # for Ansible become support
       - screen            # Vulcain asked for this
+    update_cache: true
+  register: apt_result
+  retries: 3
+  until: apt_result is succeeded
 
 # Pimp my server
 - name: Customize motd

roles/baseconfig/tasks/molly-guard.yml

@@ -6,6 +6,9 @@
     name: molly-guard
     state: present
     update_cache: true
+  register: apt_result
+  retries: 3
+  until: apt_result is succeeded
 
 # Always ask for hostname
 - name: Configure molly-guard

roles/basesecurity/tasks/main.yml

@@ -22,4 +22,4 @@
     - /usr/bin/chfn  # With re2o
     - /bin/mount  # Only root should mount
     - /bin/umount  # Only root should umount
-  ignore_errors: yes  # Sometimes file won't exist
+  ignore_errors: true  # Sometimes file won't exist

roles/codimd/handlers/main.yml

@@ -2,7 +2,7 @@
 # Build front-end bundle
 # This can take very long and requires > 2GB of RAM
 - name: Build front-end for CodiMD
-  command: NODE_ENV="production" yarn run build
+  command: yarn run build
   args:
     chdir: /var/local/codimd/codimd
   become: true
@@ -10,4 +10,5 @@
 
 # Reload systemd daemons when a service file changes
 - name: Reload systemd daemons
-  command: systemctl daemon-reload
+  systemd:
+    daemon_reload: true

roles/codimd/tasks/0_apt_dependencies.yml

@@ -9,20 +9,37 @@
     dest: /etc/apt/preferences.d/nodejs
     mode: 0644
 
-# TODO
+# For HTTPS apt
-# apt-transport-https
+- name: Install HTTPS apt
-# curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
+  apt:
-# echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
+    name: apt-transport-https
+    update_cache: true
+  register: apt_result
+  retries: 3
+  until: apt_result is succeeded
+
+- name: Configure the apt key
+  apt_key:
+    url: https://dl.yarnpkg.com/debian/pubkey.gpg
+    id: 1646B01B86E50310
+  register: apt_key_result
+  retries: 3
+  until: apt_key_result is succeeded
+
+- name: Configure Yarn repository
+  apt_repository:
+    repo: "deb https://dl.yarnpkg.com/debian/ stable main"
 
 # Install CodiMD dependencies
 - name: Install required packages
   apt:
-    name: "{{ item }}"
+    name:
-    state: present
+      - git
+      - nodejs
+      - npm
+      - build-essential
+      - yarn
     update_cache: true
-  with_items:
+  register: apt_result
-    - git
+  retries: 3
-    - nodejs
+  until: apt_result is succeeded
-    - npm
-    - build-essential
-    - yarn

roles/codimd/tasks/1_user_group.yml

@@ -3,7 +3,7 @@
 - name: Create CodiMD system group
   group:
     name: codimd
-    system: yes
+    system: true
     state: present
 
 # Security #2
@@ -13,7 +13,7 @@
     group: codimd
     home: /var/local/codimd
     comment: CodiMD
-    system: yes
+    system: true
     state: present
 
 # Security #3

roles/codimd/tasks/main.yml

@@ -16,12 +16,15 @@
   notify: Build front-end for CodiMD
 
 # Setup dependencies and configs
-- name: Install CodiMD depedencies
+- name: Install CodiMD dependencies
-  command: NODE_ENV="production" bin/setup
+  yarn:
-  args:
+    path: /var/local/codimd/codimd
-    chdir: /var/local/codimd/codimd
+    production: true
   become: true
   become_user: codimd
+  register: yarn_result
+  retries: 3
+  until: yarn_result is succeeded
 
 # Connection to database
 - name: Connect CodiMD to PostgreSQL db

roles/dokuwiki/tasks/main.yml

@@ -24,3 +24,6 @@
   apt:
     name: dokuwiki
     update_cache: true
+  register: apt_result
+  retries: 3
+  until: apt_result is succeeded

roles/etherpad/handlers/main.yml

@@ -1,4 +1,5 @@
 ---
 # Reload systemd daemons when a service file changes
 - name: Reload systemd daemons
-  command: systemctl daemon-reload
+  systemd:
+    daemon_reload: true

roles/etherpad/tasks/0_apt_dependencies.yml

@@ -12,12 +12,13 @@
 # Install EtherPad dependencies
 - name: Install required packages
   apt:
-    name: "{{ item }}"
+    name:
-    state: present
+      - build-essential
+      - curl
+      - git
+      - nodejs
+      - npm
     update_cache: true
-  with_items:
+  register: apt_result
-    - build-essential
+  retries: 3
-    - curl
+  until: apt_result is succeeded
-    - git
-    - nodejs
-    - npm

roles/etherpad/tasks/1_user_group.yml

@@ -3,7 +3,7 @@
 - name: Create EtherPad system group
   group:
     name: etherpad
-    system: yes
+    system: true
     state: present
 
 # Security #2
@@ -13,7 +13,7 @@
     group: etherpad
     home: /var/local/etherpad
     comment: EtherPad
-    system: yes
+    system: true
     state: present
 
 # Security #3

roles/etherpad/tasks/main.yml

@@ -10,19 +10,38 @@
   git:
     repo: https://github.com/ether/etherpad-lite.git
     dest: /var/local/etherpad/etherpad-lite
-    version: master
+    version: 1.7.5
   become: true
   become_user: etherpad
 
-# Installation script
+# Installation script bin/installDeps.sh (1)
-# TODO: move this in a handler
+- name: Create node_modules directory
-- name: Install Etherpad dependencies
+  file:
-  command: bin/installDeps.sh
+    path: /var/local/etherpad/etherpad-lite/node_modules
-  args:
+    state: directory
-    chdir: /var/local/etherpad/etherpad-lite
   become: true
   become_user: etherpad
 
+# Installation script bin/installDeps.sh (2)
+- name: Create symbolic link to EtherPad src
+  file:
+    src: /var/local/etherpad/etherpad-lite/src
+    dest: /var/local/etherpad/etherpad-lite/node_modules/ep_etherpad-lite
+    state: link
+  become: true
+  become_user: etherpad
+
+# Installation script bin/installDeps.sh (3)
+# TODO --no-save
+- name: Install Etherpad dependencies
+  npm:
+    path: /var/local/etherpad/etherpad-lite/node_modules/ep_etherpad-lite
+  become: true
+  become_user: etherpad
+  register: npm_result
+  retries: 3
+  until: npm_result is succeeded
+
 # Configuration
 - name: Configure EtherPad
   lineinfile:
@@ -32,7 +51,7 @@
   with_dict:
     title: "  \"title\": \"Etherpad Aurore\","
     dbType: "  \"dbType\" : \"postgres\","
-    defaultPadText: "  \"defaultPadText\" : \"Bienvenue sur l'EtherPad d'Aurore !\\n\\nCe pad est synchronisé avec les autres utilisateur·rice·s présent·e·s sur cette page.\\n\","
+    defaultPadText: "  \"defaultPadText\" : \"Bienvenue sur l'EtherPad d'Aurore !\\n\\nCe pad est public.\\n\","
     lang: "    \"lang\": \"fr-fr\""
 
 # Service file
@@ -50,10 +69,8 @@
   service:
     name: etherpad-lite
     state: started
-    enabled: True
+    enabled: true
-
-# La configuration de la clé `dbSettings` n'est pas encore automatisé !
 
 # TODO-list
-#  * Configure admin user, logs
+# La configuration de la clé `dbSettings` n'est pas encore automatisé !
-# Plugins : https://framacloud.org/fr/cultiver-son-jardin/etherpad.html#concernant-framapad
+#  * Configure logs

roles/ldap-client/tasks/0_install_ldap.yml

@@ -2,13 +2,14 @@
 # Install LDAP client packages
 - name: Install LDAP client packages
   apt:
-    name: "{{ item }}"
+    name:
-    state: present
+      - nslcd
+      - libnss-ldapd
+      - libpam-ldapd
     update_cache: true
-  with_items:
+  register: apt_result
-    - nslcd
+  retries: 3
-    - libnss-ldapd
+  until: apt_result is succeeded
-    - libpam-ldapd
 
 # Reduce LDAP load
 # For the moment it is broken on Stretch when using PHP7.3

roles/ldap-client/tasks/1_group_security.yml

@@ -11,10 +11,13 @@
   package:
     name: sudo
     state: present
+  register: package_result
+  retries: 3
+  until: package_result is succeeded
 
 # Set sudo group
 - name: Configure sudoers
   lineinfile:
     dest: /etc/sudoers
     regexp: "^%{{ sudo_group }}"
-    line: "%{{ sudo_group }}	ALL=(ALL:ALL) ALL"
+    line: "%{{ sudo_group }} ALL=(ALL:ALL) ALL"

roles/ldap-replica/tasks/main.yml

@@ -5,6 +5,9 @@
     name: slapd
     state: present
     update_cache: true
+  register: apt_result
+  retries: 3
+  until: apt_result is succeeded
 
 # What is written after is really not a nice way to install a schema
 # because the LDAP is being flushed away always...
@@ -48,7 +51,7 @@
     path: "{{ item }}"
     owner: openldap
     group: openldap
-    recurse: yes
+    recurse: true
   with_items:
     - '/var/lib/ldap'
     - '/etc/ldap/slapd.d'

roles/matrix-appservice-discord/handlers/main.yml

@@ -1,4 +1,5 @@
 ---
 # Reload systemd daemons when a service file changes
 - name: Reload systemd daemons
-  command: systemctl daemon-reload
+  systemd:
+    daemon_reload: true

roles/matrix-appservice-discord/tasks/0_apt_dependencies.yml

@@ -10,11 +10,12 @@
 
 - name: Install required packages
   apt:
-    name: "{{ item }}"
+    name:
-    state: present
+      - git
+      - nodejs
+      - npm
+      - build-essential
     update_cache: true
-  with_items:
+  register: apt_result
-    - git
+  retries: 3
-    - nodejs
+  until: apt_result is succeeded
-    - npm
-    - build-essential

roles/matrix-appservice-discord/tasks/1_user_group.yml

@@ -3,7 +3,7 @@
 - name: Create matrix-appservice-discord system group
   group:
     name: matrix-appservice-discord
-    system: yes
+    system: true
     state: present
 
 # Security #2
@@ -13,7 +13,7 @@
     group: matrix-appservice-discord
     home: /var/local/matrix-appservice-discord
     comment: Matrix Appservice Discord
-    system: yes
+    system: true
     state: present
 
 # Security #3

roles/matrix-appservice-discord/tasks/main.yml

@@ -15,18 +15,23 @@
   become_user: matrix-appservice-discord
 
 # Setup dependencies
-- name: Install matrix-appservice-discord depedencies
+# May create issues with package-lock.json not in gitignore
-  command: npm ci
+- name: Install matrix-appservice-discord dependencies
-  args:
+  npm:
-    chdir: /var/local/matrix-appservice-discord/matrix-appservice-discord
+    path: /var/local/matrix-appservice-discord/matrix-appservice-discord
   become: true
   become_user: matrix-appservice-discord
+  register: npm_result
+  retries: 3
+  until: npm_result is succeeded
 
 # Typescript into javascript
 - name: Compile matrix-appservice-discord
-  command: npm run build
+  command: ./node_modules/.bin/tsc
   args:
     chdir: /var/local/matrix-appservice-discord/matrix-appservice-discord
+  register: npm_build_result
+  changed_when: npm_build_result
   become: true
   become_user: matrix-appservice-discord
 

roles/matrix-appservice-irc/handlers/main.yml

@@ -1,4 +1,5 @@
 ---
 # Reload systemd daemons when a service file changes
 - name: Reload systemd daemons
-  command: systemctl daemon-reload
+  systemd:
+    daemon_reload: true

roles/matrix-appservice-irc/tasks/0_apt_dependencies.yml

@@ -10,11 +10,12 @@
 
 - name: Install required packages
   apt:
-    name: "{{ item }}"
+    name:
-    state: present
+      - git
+      - nodejs
+      - npm
+      - build-essential
     update_cache: true
-  with_items:
+  register: apt_result
-    - git
+  retries: 3
-    - nodejs
+  until: apt_result is succeeded
-    - npm
-    - build-essential

roles/matrix-appservice-irc/tasks/1_user_group.yml

@@ -3,7 +3,7 @@
 - name: Create matrix-appservice-irc system group
   group:
     name: matrix-appservice-irc
-    system: yes
+    system: true
     state: present
 
 # Security #2
@@ -13,7 +13,7 @@
     group: matrix-appservice-irc
     home: /var/local/matrix-appservice-irc
     comment: Matrix Appservice IRC
-    system: yes
+    system: true
     state: present
 
 # Security #3

roles/matrix-appservice-irc/tasks/main.yml

@@ -14,12 +14,15 @@
   become_user: matrix-appservice-irc
 
 # Setup dependencies
-- name: Install matrix-appservice-irc depedencies
+- name: Install matrix-appservice-irc dependencies
-  command: npm install
+  npm:
-  args:
+    path: /var/local/matrix-appservice-irc/matrix-appservice-irc
-    chdir: /var/local/matrix-appservice-irc/matrix-appservice-irc
+    production: true
   become: true
   become_user: matrix-appservice-irc
+  register: npm_result
+  retries: 3
+  until: npm_result is succeeded
 
 # Configure
 - name: Configure matrix-appservice-irc

roles/matrix-mxisd/tasks/main.yml

@@ -1,7 +1,10 @@
 ---
 - name: Install mxisd
   apt:
-    deb: https://github.com/kamax-matrix/mxisd/releases/download/v{{ mxisd_version }}/mxisd_{{ mxisd_version }}_all.deb
+    deb: "{{ mxisd_deb }}"
+  register: apt_result
+  retries: 3
+  until: apt_result is succeeded
 
 - name: Configure mxisd
   template:

roles/matrix-riot/tasks/main.yml

@@ -4,6 +4,9 @@
   apt:
     name: apt-transport-https
     update_cache: true
+  register: apt_result
+  retries: 3
+  until: apt_result is succeeded
 
 # Add the repository into source list
 - name: Configure riot repository
@@ -15,17 +18,26 @@
   apt_key:
     url: https://riot.im/packages/debian/repo-key.asc
     id: E019645248E8F4A1
+  register: apt_key_result
+  retries: 3
+  until: apt_key_result is succeeded
 
 # Install riot
 - name: Install riot-web
   apt:
     name: riot-web
     update_cache: true
+  register: apt_result
+  retries: 3
+  until: apt_result is succeeded
 
 # Install nginx
 - name: Install nginx
   apt:
     name: nginx
+  register: apt_result
+  retries: 3
+  until: apt_result is succeeded
 
 # Configure nginx
 - name: Configure nginx

roles/matrix-synapse/files/rest_auth_provider.py

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+#
+# REST endpoint Authentication module for Matrix synapse
+# Copyright (C) 2017 Maxime Dor
+#
+# https://max.kamax.io/
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import logging
+from twisted.internet import defer
+import requests
+import json
+
+logger = logging.getLogger(__name__)
+
+class RestAuthProvider(object):
+
+    def __init__(self, config, account_handler):
+        self.account_handler = account_handler
+
+        if not config.endpoint:
+            raise RuntimeError('Missing endpoint config')
+
+        self.endpoint = config.endpoint
+        self.regLower = config.regLower
+        self.config = config
+        
+        logger.info('Endpoint: %s', self.endpoint)
+        logger.info('Enforce lowercase username during registration: %s', self.regLower)
+
+    @defer.inlineCallbacks
+    def check_password(self, user_id, password):
+        logger.info("Got password check for " + user_id)
+        data = {'user':{'id':user_id, 'password':password}}
+        r = requests.post(self.endpoint + '/_matrix-internal/identity/v1/check_credentials', json = data)
+        r.raise_for_status()
+        r = r.json()
+        if not r["auth"]:
+            reason = "Invalid JSON data returned from REST endpoint"
+            logger.warning(reason)
+            raise RuntimeError(reason)
+
+        auth = r["auth"]
+        if not auth["success"]:
+            logger.info("User not authenticated")
+            defer.returnValue(False)
+
+        localpart = user_id.split(":", 1)[0][1:]
+        logger.info("User %s authenticated", user_id)
+
+        registration = False
+        if not (yield self.account_handler.check_user_exists(user_id)):
+            logger.info("User %s does not exist yet, creating...", user_id)
+            
+            if localpart != localpart.lower() and self.regLower:
+                logger.info('User %s was cannot be created due to username lowercase policy', localpart)
+                defer.returnValue(False)
+            
+            user_id, access_token = (yield self.account_handler.register(localpart=localpart))
+            registration = True
+            logger.info("Registration based on REST data was successful for %s", user_id)
+        else:
+            logger.info("User %s already exists, registration skipped", user_id)
+
+        if auth["profile"]:
+            logger.info("Handling profile data")
+            profile = auth["profile"]
+
+            store = yield self.account_handler.hs.get_profile_handler().store
+            if "display_name" in profile and ((registration and self.config.setNameOnRegister) or (self.config.setNameOnLogin)):
+                display_name = profile["display_name"]
+                logger.info("Setting display name to '%s' based on profile data", display_name)
+                yield store.set_profile_displayname(localpart, display_name)
+            else:
+                logger.info("Display name was not set because it was not given or policy restricted it")
+            
+            if (self.config.updateThreepid):
+                if "three_pids" in profile:
+                    logger.info("Handling 3PIDs")
+                    for threepid in profile["three_pids"]:
+                        medium = threepid["medium"].lower()
+                        address = threepid["address"].lower()
+                        logger.info("Looking for 3PID %s:%s in user profile", medium, address)
+
+                        validated_at = self.account_handler.hs.get_clock().time_msec()
+                        if not (yield store.get_user_id_by_threepid(medium, address)):
+                            logger.info("3PID is not present, adding")
+                            yield store.user_add_threepid(
+                                user_id,
+                                medium,
+                                address,
+                                validated_at,
+                                validated_at
+                            )
+                        else:
+                            logger.info("3PID is present, skipping")
+            else:
+                logger.info("3PIDs were not updated due to policy")
+        else:
+            logger.info("No profile data")
+
+        defer.returnValue(True)
+
+    @staticmethod
+    def parse_config(config):
+        # verify config sanity
+        _require_keys(config, ["endpoint"])
+
+        class _RestConfig(object):
+            endpoint = ''
+            regLower = True
+            setNameOnRegister = True
+            setNameOnLogin = False
+            updateThreepid = True
+
+        rest_config = _RestConfig()
+        rest_config.endpoint = config["endpoint"]
+
+        try:
+            rest_config.regLower = config['policy']['registration']['username']['enforceLowercase']
+        except TypeError:
+            # we don't care
+            pass
+        except KeyError:
+            # we don't care
+            pass
+
+        try:
+            rest_config.setNameOnRegister = config['policy']['registration']['profile']['name']
+        except TypeError:
+            # we don't care
+            pass
+        except KeyError:
+            # we don't care
+            pass
+        
+        try:
+            rest_config.setNameOnLogin = config['policy']['login']['profile']['name']
+        except TypeError:
+            # we don't care
+            pass
+        except KeyError:
+            # we don't care
+            pass
+
+        try:
+            rest_config.updateThreepid = config['policy']['all']['threepid']['update']
+        except TypeError:
+            # we don't care
+            pass
+        except KeyError:
+            # we don't care
+            pass
+
+        return rest_config
+
+def _require_keys(config, required):
+    missing = [key for key in required if key not in config]
+    if missing:
+        raise Exception(
+            "REST Auth enabled but missing required config values: {}".format(
+                ", ".join(missing)
+            )
+        )
+

roles/matrix-synapse/tasks/main.yml

@@ -4,6 +4,9 @@
     name: matrix-synapse
     update_cache: true
     default_release: stretch-backports
+  register: apt_result
+  retries: 3
+  until: apt_result is succeeded
 
 - name: Configure matrix-synapse
   template:
@@ -22,8 +25,8 @@
   notify: Restart matrix-synapse service
 
 - name: Install rest auth provider
-  get_url:
+  copy:
-    url: "{{ synapse_rest_auth_url }}"
+    src: rest_auth_provider.py
     dest: /usr/local/lib/python3.5/dist-packages/rest_auth_provider.py
     mode: 0755
   notify: Restart matrix-synapse service

roles/nginx-reverse-proxy/tasks/main.yml

@@ -4,6 +4,9 @@
   apt:
     name: nginx
     update_cache: true
+  register: apt_result
+  retries: 3
+  until: apt_result is succeeded
 
 # Install sites
 - name: Configure NGINX sites

upgrade.yml

@@ -9,3 +9,6 @@
         upgrade: dist
         update_cache: true
         cache_valid_time: 86400  # one day
+      register: apt_result
+      retries: 3
+      until: apt_result is succeeded