From 3f33b3ad8c5cd25c9b1f09bdc06c14ce9332503d Mon Sep 17 00:00:00 2001
From: Jouni Malinen <j@w1.fi>
Date: Fri, 27 Dec 2013 10:16:50 +0200
Subject: [PATCH] tests: Collect code coverage separately from each component
 in vm

Use a more robust design for collecting the gcov logs from the case
where test cases are run within a virtual machine. This generates a
writable-from-vm build tree for each component separately so that the
lcov and gcov can easily find the matching source code and data files.
In addition, prepare the reports automatically at the end of the
vm-run.sh --codecov execution.

Signed-hostap: Jouni Malinen <j@w1.fi>
---
 tests/hwsim/example-wpa_supplicant.config |   4 -
 tests/hwsim/start.sh                      |  14 ++-
 tests/hwsim/vm/README                     |  15 +++-
 tests/hwsim/vm/vm-run.sh                  | 102 ++++++++++++++++++++--
 4 files changed, 120 insertions(+), 15 deletions(-)

diff --git a/tests/hwsim/example-wpa_supplicant.config b/tests/hwsim/example-wpa_supplicant.config
index 67c9768d4..2923e835d 100644
--- a/tests/hwsim/example-wpa_supplicant.config
+++ b/tests/hwsim/example-wpa_supplicant.config
@@ -111,7 +111,3 @@ CFLAGS += -DALL_DH_GROUPS
 CONFIG_WNM=y
 
 CONFIG_TESTING_OPTIONS=y
-
-# gcov code coverage from the virtual machine
-#CONFIG_CODE_COVERAGE=y
-#CFLAGS += -fprofile-dir=/tmp/logs/gcov
diff --git a/tests/hwsim/start.sh b/tests/hwsim/start.sh
index f2722ac87..346ea2c08 100755
--- a/tests/hwsim/start.sh
+++ b/tests/hwsim/start.sh
@@ -4,6 +4,7 @@ DIR="$( cd "$( dirname "$0" )" && pwd )"
 WPAS=$DIR/../../wpa_supplicant/wpa_supplicant
 WPACLI=$DIR/../../wpa_supplicant/wpa_cli
 HAPD=$DIR/../../hostapd/hostapd
+HAPD_AS=$DIR/../../hostapd/hostapd
 WLANTEST=$DIR/../../wlantest/wlantest
 HLR_AUC_GW=$DIR/../../hostapd/hlr_auc_gw
 
@@ -13,6 +14,17 @@ if [ -z "$LOGDIR" ] ; then
     mkdir -p $LOGDIR
     rm -rf $DIR/logs/current
     ln -sf $DATE $DIR/logs/current
+else
+    if [ -e $LOGDIR/alt-wpa_supplicant/wpa_supplicant/wpa_supplicant ]; then
+	WPAS=$LOGDIR/alt-wpa_supplicant/wpa_supplicant/wpa_supplicant
+    fi
+    if [ -e $LOGDIR/alt-hostapd/hostapd/hostapd ]; then
+	HAPD=$LOGDIR/alt-hostapd/hostapd/hostapd
+    fi
+    if [ -e $LOGDIR/alt-hostapd-as/hostapd/hostapd ]; then
+	HAPD_AS=$LOGDIR/alt-hostapd-as/hostapd/hostapd
+	HLR_AUC_GW=$LOGDIR/alt-hostapd-as/hostapd/hlr_auc_gw
+    fi
 fi
 
 if groups | tr ' ' "\n" | grep -q ^admin$; then
@@ -65,7 +77,7 @@ if [ -x $HLR_AUC_GW ]; then
     $HLR_AUC_GW -m $DIR/auth_serv/hlr_auc_gw.milenage_db > $LOGDIR/hlr_auc_gw &
 fi
 
-$HAPD -ddKt $DIR/auth_serv/as.conf > $LOGDIR/auth_serv &
+$HAPD_AS -ddKt $DIR/auth_serv/as.conf > $LOGDIR/auth_serv &
 
 # wait for programs to be fully initialized
 for i in 0 1 2; do
diff --git a/tests/hwsim/vm/README b/tests/hwsim/vm/README
index ba124ef58..88cfc9f23 100644
--- a/tests/hwsim/vm/README
+++ b/tests/hwsim/vm/README
@@ -21,7 +21,20 @@ give it are passed through to run-all.sh, see there.
 
 --------------------------------------------------------------------------------
 
-Code Coverage Analysis
+Code Coverage Analysis for user space code
+
+Code coverage for wpa_supplicant and hostapd can be generated from the
+test run with following command line:
+
+./vm-run.sh --codecov [other arguments..]
+
+This builds a separate copies of wpa_supplicant and hostapd into a
+directory that is writable from the virtual machine to collect the gcov
+data. lcov is then used to prepare the reports at the end of the test
+run.
+
+
+Code Coverage Analysis for kernel code
 
 In order to do code coverage analysis, reconfigure the kernel to include
 
diff --git a/tests/hwsim/vm/vm-run.sh b/tests/hwsim/vm/vm-run.sh
index bf6933b27..0f556decb 100755
--- a/tests/hwsim/vm/vm-run.sh
+++ b/tests/hwsim/vm/vm-run.sh
@@ -6,7 +6,7 @@ if [ -z "$TESTDIR" ] ; then
 	TESTDIR=$(pwd)/../
 fi
 
-LOGS=/tmp/hwsim-test-logs/
+LOGS=/tmp/hwsim-test-logs
 
 # increase the memory size if you want to run with valgrind, 512 MB works
 MEMORY=128
@@ -40,9 +40,63 @@ fi
 
 
 CMD=$TESTDIR/vm/inside.sh
-LOGDIR=$LOGS/$(date +%s)
+DATE=$(date +%s)
+LOGDIR=$LOGS/$DATE
 mkdir -p $LOGDIR
 
+if [ "$1" = "--codecov" ]; then
+    shift
+    CODECOV=yes
+    DIR=$PWD
+    if [ -e /tmp/logs ]; then
+	echo "/tmp/logs exists - cannot prepare build trees"
+	exit 1
+    fi
+    mkdir /tmp/logs
+    echo "Preparing separate build trees for hostapd/wpa_supplicant"
+    cd ../../..
+    git archive --format=tar --prefix=hostap/ HEAD > /tmp/logs/hostap.tar
+    cd $DIR
+    cat ../../../wpa_supplicant/.config > /tmp/logs/wpa_supplicant.config
+    echo "CONFIG_CODE_COVERAGE=y" >> /tmp/logs/wpa_supplicant.config
+    cat ../../../hostapd/.config > /tmp/logs/hostapd.config
+    echo "CONFIG_CODE_COVERAGE=y" >> /tmp/logs/hostapd.config
+
+    cd /tmp/logs
+    tar xf hostap.tar
+    mv hostap alt-wpa_supplicant
+    mv wpa_supplicant.config alt-wpa_supplicant/wpa_supplicant/.config
+    tar xf hostap.tar
+    mv hostap alt-hostapd
+    cp hostapd.config alt-hostapd/hostapd/.config
+    tar xf hostap.tar
+    mv hostap alt-hostapd-as
+    mv hostapd.config alt-hostapd-as/hostapd/.config
+    rm hostap.tar
+
+    cd /tmp/logs/alt-wpa_supplicant/wpa_supplicant
+    echo "Building wpa_supplicant"
+    make -j8 > /dev/null
+
+    cd /tmp/logs/alt-hostapd/hostapd
+    echo "Building hostapd"
+    make -j8 hostapd > /dev/null
+
+    cd /tmp/logs/alt-hostapd-as/hostapd
+    echo "Building hostapd/hlr_auc_gw (AS)"
+    make -j8 hostapd hlr_auc_gw > /dev/null
+
+    cd $DIR
+
+    mv /tmp/logs/alt-wpa_supplicant $LOGDIR
+    mv /tmp/logs/alt-hostapd $LOGDIR
+    mv /tmp/logs/alt-hostapd-as $LOGDIR
+else
+    CODECOV=no
+fi
+
+echo "Starting test run in a virtual machine"
+
 kvm \
 	-kernel $KERNEL -smp 4 \
 	$KVMARGS -m $MEMORY -nographic \
@@ -53,12 +107,42 @@ kvm \
 	-monitor null -serial stdio -serial file:$LOGDIR/console \
 	-append "mac80211_hwsim.channels=$CHANNELS mac80211_hwsim.radios=5 init=$CMD testdir=$TESTDIR console=$KVMOUT root=/dev/root rootflags=trans=virtio,version=9p2000.u ro rootfstype=9p EPATH=$EPATH ARGS=$*"
 
-echo LOGDIR=$LOGDIR
+if [ $CODECOV = "yes" ]; then
+    mv $LOGDIR/alt-wpa_supplicant /tmp/logs
+    mv $LOGDIR/alt-hostapd /tmp/logs
+    mv $LOGDIR/alt-hostapd-as /tmp/logs
 
-if [ -d $LOGDIR/gcov ]; then
-    echo "Move gcov data files from vm logdir to build directories"
-    for i in $LOGDIR/gcov/*.gcda; do
-	file=`basename $i | sed "s/.gcda$//"`
-	find ../../.. -name $file.gcno | sed s/.gcno/.gcda/ | xargs mv $i
-    done
+    echo "Generating code coverage report for wpa_supplicant"
+    cd /tmp/logs/alt-wpa_supplicant/wpa_supplicant
+    lcov -c -d .. > lcov.info 2> lcov.log
+    genhtml -t "wpa_supplicant hwsim test run $DATE" lcov.info --output-directory $LOGDIR/lcov-wpa_supplicant >> lcov.log 2>&1
+    mv lcov.info lcov.log $LOGDIR/lcov-wpa_supplicant
+
+    echo "Generating code coverage report for hostapd"
+    cd /tmp/logs/alt-hostapd/hostapd
+    lcov -c -d .. > lcov.info 2> lcov.log
+    genhtml -t "hostapd hwsim test run $DATE" lcov.info --output-directory $LOGDIR/lcov-hostapd >> lcov.log 2>&1
+    mv lcov.info lcov.log $LOGDIR/lcov-hostapd
+
+    echo "Generating code coverage report for hostapd/hlr_auc_gw (AS)"
+    cd /tmp/logs/alt-hostapd-as/hostapd
+    lcov -c -d .. > lcov.info 2> lcov.log
+    genhtml -t "hostapd/hlr_auc_gw (AS) hwsim test run $DATE" lcov.info --output-directory $LOGDIR/lcov-hostapd-as >> lcov.log 2>&1
+    mv lcov.info lcov.log $LOGDIR/lcov-hostapd-as
+
+    cd $DIR
+    rm -r /tmp/logs/alt-wpa_supplicant
+    rm -r /tmp/logs/alt-hostapd
+    rm -r /tmp/logs/alt-hostapd-as
+    rmdir /tmp/logs
+fi
+
+echo
+echo "Test run completed"
+echo "Logfiles are at $LOGDIR"
+if [ $CODECOV = "yes" ]; then
+    echo "Code coverage reports:"
+    echo "wpa_supplicant: file://$LOGDIR/lcov-wpa_supplicant/index.html"
+    echo "hostapd: file://$LOGDIR/lcov-hostapd/index.html"
+    echo "hostapd/hlr_auc_gw (AS): file://$LOGDIR/lcov-hostapd-as/index.html"
 fi