ci: check for failed services after boot

This should, hopefully, catch issues like systemd/systemd#21671
automagically.
main
Frantisek Sumsal 2 years ago
parent 1f013e0c49
commit 24acd4064e

@ -17,6 +17,11 @@ on:
permissions:
contents: read
env:
# Enable debug logging in systemd, but keep udev's log level to info,
# since it's _very_ verbose in the QEMU task
KERNEL_CMDLINE: "systemd.unit=mkosi-check-and-shutdown.service !quiet systemd.log_level=debug systemd.log_target=console udev.log_level=info systemd.default_standard_output=journal+console"
jobs:
ci:
runs-on: ubuntu-20.04
@ -57,13 +62,20 @@ jobs:
systemd-nspawn --version
- name: Build ${{ matrix.distro }}
run: sudo python3 -m mkosi build
run: |
sudo python3 -m mkosi --build-environment=CI_BUILD=1 --kernel-command-line "${{ env.KERNEL_CMDLINE }}" build
- name: Show ${{ matrix.distro }} image summary
run: sudo python3 -m mkosi summary
- name: Boot ${{ matrix.distro }} systemd-nspawn
run: sudo ./.github/workflows/test_mkosi_boot.py python3 -m mkosi boot
run: sudo python3 -m mkosi boot ${{ env.KERNEL_CMDLINE }}
- name: Check ${{ matrix.distro }} systemd-nspawn
run: sudo python3 -m mkosi shell bash -c "[[ -e /testok ]] || { cat /failed-services; exit 1; }"
- name: Boot ${{ matrix.distro }} QEMU
run: sudo ./.github/workflows/test_mkosi_boot.py python3 -m mkosi qemu
run: sudo python3 -m mkosi qemu
- name: Check ${{ matrix.distro }} QEMU
run: sudo python3 -m mkosi shell bash -c "[[ -e /testok ]] || { cat /failed-services; exit 1; }"

@ -1,26 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: LGPL-2.1-or-later
import pexpect
import re
import sys
def run() -> None:
p = pexpect.spawnu(" ".join(sys.argv[1:]), logfile=sys.stdout, timeout=300)
# distro-independent root prompt
p.expect(re.compile("~[^#]{0,3}#"))
p.sendline("systemctl poweroff")
p.expect(pexpect.EOF)
try:
run()
except pexpect.EOF:
print("UNEXPECTED EOF")
sys.exit(1)
except pexpect.TIMEOUT:
print("TIMED OUT")
sys.exit(1)

@ -110,3 +110,12 @@ if [ -n "$IMAGE_VERSION" ] ; then
cat /tmp/os-release.tmp > "$DESTDIR"/usr/lib/os-release
rm /tmp/os-release.tmp
fi
# If $CI_BUILD is set, copy over the CI service which executes a service check
# after boot and then shuts down the machine
if [ -n "$CI_BUILD" ]; then
mkdir -p "$DESTDIR/usr/lib/systemd/system"
cp -v "$SRCDIR/test/mkosi-check-and-shutdown.service" "$DESTDIR/usr/lib/systemd/system/mkosi-check-and-shutdown.service"
cp -v "$SRCDIR/test/mkosi-check-and-shutdown.sh" "$DESTDIR/usr/lib/systemd/mkosi-check-and-shutdown.sh"
chmod +x "$DESTDIR/usr/lib/systemd/mkosi-check-and-shutdown.sh"
fi

@ -23,6 +23,7 @@ BuildPackages=
libcryptsetup-devel
libcurl-devel
libgcrypt-devel
libgnutls-devel
libkmod-devel
liblz4-devel
libmicrohttpd-devel
@ -35,8 +36,8 @@ BuildPackages=
pciutils-devel
pcre-devel
python3
python3-lxml
python3-Jinja2
python3-lxml
qrencode-devel
system-user-nobody
systemd-sysvinit
@ -61,6 +62,7 @@ Packages=
libcrypt1
libcryptsetup12
libgcrypt20
libgnutls30
libkmod2
liblz4-1
libmount1

@ -4,3 +4,13 @@
if [ "$1" = "final" ] && command -v bootctl > /dev/null; then
bootctl install
fi
# Temporary workaround until https://github.com/openSUSE/suse-module-tools/commit/158643414ddb8d8208016a5f03a4484d58944d7a
# gets into OpenSUSE repos
if [ "$1" = "final" ] && grep -q openSUSE /etc/os-release; then
if [ -e "/usr/lib/systemd/system/boot-sysctl.service" ] && \
! grep -F -q 'ConditionPathExists=/boot/sysctl.conf' "/usr/lib/systemd/system/boot-sysctl.service"; then
mkdir -p "/etc/systemd/system/boot-sysctl.service.d/"
printf '[Unit]\nConditionPathExists=/boot/sysctl.conf-%%v' >"/etc/systemd/system/boot-sysctl.service.d/99-temporary-workaround.conf"
fi
fi

@ -0,0 +1,14 @@
# SPDX-License-Identifier: LGPL-2.1-or-later
[Unit]
Description=Check if any service failed and then shutdown the machine
After=multi-user.target network-online.target
Requires=multi-user.target
Wants=systemd-resolved.service systemd-networkd.service network-online.target
OnFailure=poweroff.target
OnFailureJobMode=replace-irreversibly
[Service]
Type=oneshot
ExecStartPre=-rm -f /failed-services
ExecStart=/usr/lib/systemd/mkosi-check-and-shutdown.sh
ExecStartPost=systemctl poweroff --no-block

@ -0,0 +1,9 @@
#!/bin/bash -eux
# SPDX-License-Identifier: LGPL-2.1-or-later
systemctl --failed --no-legend | tee /failed-services
# Exit with non-zero EC if the /failed-services file is not empty (we have -e set)
[[ ! -s /failed-services ]]
: >/testok
Loading…
Cancel
Save