From 09e38c2fce261d512b81bbe941d7d804b54cd048 Mon Sep 17 00:00:00 2001
From: Jouni Malinen <j@w1.fi>
Date: Mon, 24 Mar 2014 01:35:58 +0200
Subject: [PATCH] tests: Add scripts to allow parallel execution of tests in
 VMs

"parallel-vm.sh <number of VMs> [arguments..]" can now be used to run
multiple VMs in parallel to speed up full test cycle significantly. In
addition, the "--split srv/total" argument used in this design would
also make it possible to split this to multiple servers to speed up
testing.

Signed-off-by: Jouni Malinen <j@w1.fi>
---
 tests/hwsim/run-tests.py      | 10 ++++++++++
 tests/hwsim/vm/README         |  9 +++++++++
 tests/hwsim/vm/parallel-vm.sh | 27 +++++++++++++++++++++++++++
 tests/hwsim/vm/vm-run.sh      |  8 +++++++-
 4 files changed, 53 insertions(+), 1 deletion(-)
 create mode 100755 tests/hwsim/vm/parallel-vm.sh

diff --git a/tests/hwsim/run-tests.py b/tests/hwsim/run-tests.py
index a5ddbc313..7afe108de 100755
--- a/tests/hwsim/run-tests.py
+++ b/tests/hwsim/run-tests.py
@@ -184,6 +184,7 @@ def main():
     parser.add_argument('--shuffle-tests', action='store_true',
                         dest='shuffle_tests',
                         help='Shuffle test cases to randomize order')
+    parser.add_argument('--split', help='split tests for parallel execution (<server number>/<total servers>)')
     parser.add_argument('--no-reset', action='store_true', dest='no_reset',
                         help='Do not reset devices at the end of the test')
     parser.add_argument('-f', dest='testmodules', metavar='<test module>',
@@ -301,6 +302,15 @@ def main():
             report(conn, False, args.build, args.commit, run, name, 'NOTRUN', 0,
                    args.logdir)
 
+    if args.split:
+        vals = args.split.split('/')
+        split_server = int(vals[0])
+        split_total = int(vals[1])
+        logger.info("Parallel execution - %d/%d" % (split_server, split_total))
+        split_server -= 1
+        tests_to_run.sort(key=lambda t: t.__name__)
+        tests_to_run = [x for i,x in enumerate(tests_to_run) if i % split_total == split_server]
+
     if args.shuffle_tests:
         from random import shuffle
         shuffle(tests_to_run)
diff --git a/tests/hwsim/vm/README b/tests/hwsim/vm/README
index 88cfc9f23..68d740d21 100644
--- a/tests/hwsim/vm/README
+++ b/tests/hwsim/vm/README
@@ -18,6 +18,15 @@ Now you can run the vm-run.sh script and it will execute the tests using
 your system's root filesystem (read-only) inside the VM. The options you
 give it are passed through to run-all.sh, see there.
 
+To speed up testing, it is possible to run multiple VMs concurrently and
+split the test cases between all the VMs. If the host system has enough
+memory and CPU resources, this can significantly speed up the full test
+cycle. For example, a 4 core system with 4 GB of RAM can easily run 8
+parallel VMs (assuming valgrind is not used with its higher memory
+requirements). This can be run with:
+
+./parallel-vm.sh <number of VMs> [arguments..]
+
 
 --------------------------------------------------------------------------------
 
diff --git a/tests/hwsim/vm/parallel-vm.sh b/tests/hwsim/vm/parallel-vm.sh
new file mode 100755
index 000000000..9496493b3
--- /dev/null
+++ b/tests/hwsim/vm/parallel-vm.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+cd "$(dirname $0)"
+
+NUM=$1
+if [ -z "$NUM" ]; then
+    echo "usage: $0 <num servers> [params..]"
+    exit 1
+fi
+shift
+
+LOGS=/tmp/hwsim-test-logs
+mkdir -p $LOGS
+DATE=$(date +%s)
+
+for i in `seq 1 $NUM`; do
+    echo "Starting virtual machine $i/$NUM"
+    ./vm-run.sh --ext srv.$i --split $i/$NUM $* >> $LOGS/parallel-$DATE.srv.$i 2>&1 &
+done
+
+echo "Waiting for virtual machines to complete testing"
+wait
+echo "Testing completed"
+
+echo -n "PASS count: "
+grep ^PASS $LOGS/parallel-$DATE.srv.* | wc -l
+cat $LOGS/parallel-$DATE.srv.* | grep FAIL | sort
diff --git a/tests/hwsim/vm/vm-run.sh b/tests/hwsim/vm/vm-run.sh
index 825d3b9ee..5e903d81b 100755
--- a/tests/hwsim/vm/vm-run.sh
+++ b/tests/hwsim/vm/vm-run.sh
@@ -41,7 +41,13 @@ fi
 
 
 CMD=$TESTDIR/vm/inside.sh
-DATE=$(date +%s)
+if [ "$1" = "--ext" ]; then
+    shift
+    DATE=$(date +%s).$1
+    shift
+else
+    DATE=$(date +%s)
+fi
 LOGDIR=$LOGS/$DATE
 mkdir -p $LOGDIR