tests: Collect code coverage separately from each component in vm

Use a more robust design for collecting the gcov logs from the case
where test cases are run within a virtual machine. This generates a
writable-from-vm build tree for each component separately so that the
lcov and gcov can easily find the matching source code and data files.
In addition, prepare the reports automatically at the end of the
vm-run.sh --codecov execution.

Signed-hostap: Jouni Malinen <j@w1.fi>

tests/hwsim/example-wpa_supplicant.config

@@ -111,7 +111,3 @@ CFLAGS += -DALL_DH_GROUPS
 CONFIG_WNM=y
 
 CONFIG_TESTING_OPTIONS=y
-
-# gcov code coverage from the virtual machine
-#CONFIG_CODE_COVERAGE=y
-#CFLAGS += -fprofile-dir=/tmp/logs/gcov

tests/hwsim/start.sh

@@ -4,6 +4,7 @@ DIR="$( cd "$( dirname "$0" )" && pwd )"
 WPAS=$DIR/../../wpa_supplicant/wpa_supplicant
 WPACLI=$DIR/../../wpa_supplicant/wpa_cli
 HAPD=$DIR/../../hostapd/hostapd
+HAPD_AS=$DIR/../../hostapd/hostapd
 WLANTEST=$DIR/../../wlantest/wlantest
 HLR_AUC_GW=$DIR/../../hostapd/hlr_auc_gw
 
@@ -13,6 +14,17 @@ if [ -z "$LOGDIR" ] ; then
     mkdir -p $LOGDIR
     rm -rf $DIR/logs/current
     ln -sf $DATE $DIR/logs/current
+else
+    if [ -e $LOGDIR/alt-wpa_supplicant/wpa_supplicant/wpa_supplicant ]; then
+	WPAS=$LOGDIR/alt-wpa_supplicant/wpa_supplicant/wpa_supplicant
+    fi
+    if [ -e $LOGDIR/alt-hostapd/hostapd/hostapd ]; then
+	HAPD=$LOGDIR/alt-hostapd/hostapd/hostapd
+    fi
+    if [ -e $LOGDIR/alt-hostapd-as/hostapd/hostapd ]; then
+	HAPD_AS=$LOGDIR/alt-hostapd-as/hostapd/hostapd
+	HLR_AUC_GW=$LOGDIR/alt-hostapd-as/hostapd/hlr_auc_gw
+    fi
 fi
 
 if groups | tr ' ' "\n" | grep -q ^admin$; then
@@ -65,7 +77,7 @@ if [ -x $HLR_AUC_GW ]; then
     $HLR_AUC_GW -m $DIR/auth_serv/hlr_auc_gw.milenage_db > $LOGDIR/hlr_auc_gw &
 fi
 
-$HAPD -ddKt $DIR/auth_serv/as.conf > $LOGDIR/auth_serv &
+$HAPD_AS -ddKt $DIR/auth_serv/as.conf > $LOGDIR/auth_serv &
 
 # wait for programs to be fully initialized
 for i in 0 1 2; do

tests/hwsim/vm/README

@@ -21,7 +21,20 @@ give it are passed through to run-all.sh, see there.
 
 --------------------------------------------------------------------------------
 
-Code Coverage Analysis
+Code Coverage Analysis for user space code
+
+Code coverage for wpa_supplicant and hostapd can be generated from the
+test run with following command line:
+
+./vm-run.sh --codecov [other arguments..]
+
+This builds a separate copies of wpa_supplicant and hostapd into a
+directory that is writable from the virtual machine to collect the gcov
+data. lcov is then used to prepare the reports at the end of the test
+run.
+
+
+Code Coverage Analysis for kernel code
 
 In order to do code coverage analysis, reconfigure the kernel to include
 

tests/hwsim/vm/vm-run.sh

@@ -6,7 +6,7 @@ if [ -z "$TESTDIR" ] ; then
 	TESTDIR=$(pwd)/../
 fi
 
-LOGS=/tmp/hwsim-test-logs/
+LOGS=/tmp/hwsim-test-logs
 
 # increase the memory size if you want to run with valgrind, 512 MB works
 MEMORY=128
@@ -40,9 +40,63 @@ fi
 
 
 CMD=$TESTDIR/vm/inside.sh
-LOGDIR=$LOGS/$(date +%s)
+DATE=$(date +%s)
+LOGDIR=$LOGS/$DATE
 mkdir -p $LOGDIR
 
+if [ "$1" = "--codecov" ]; then
+    shift
+    CODECOV=yes
+    DIR=$PWD
+    if [ -e /tmp/logs ]; then
+	echo "/tmp/logs exists - cannot prepare build trees"
+	exit 1
+    fi
+    mkdir /tmp/logs
+    echo "Preparing separate build trees for hostapd/wpa_supplicant"
+    cd ../../..
+    git archive --format=tar --prefix=hostap/ HEAD > /tmp/logs/hostap.tar
+    cd $DIR
+    cat ../../../wpa_supplicant/.config > /tmp/logs/wpa_supplicant.config
+    echo "CONFIG_CODE_COVERAGE=y" >> /tmp/logs/wpa_supplicant.config
+    cat ../../../hostapd/.config > /tmp/logs/hostapd.config
+    echo "CONFIG_CODE_COVERAGE=y" >> /tmp/logs/hostapd.config
+
+    cd /tmp/logs
+    tar xf hostap.tar
+    mv hostap alt-wpa_supplicant
+    mv wpa_supplicant.config alt-wpa_supplicant/wpa_supplicant/.config
+    tar xf hostap.tar
+    mv hostap alt-hostapd
+    cp hostapd.config alt-hostapd/hostapd/.config
+    tar xf hostap.tar
+    mv hostap alt-hostapd-as
+    mv hostapd.config alt-hostapd-as/hostapd/.config
+    rm hostap.tar
+
+    cd /tmp/logs/alt-wpa_supplicant/wpa_supplicant
+    echo "Building wpa_supplicant"
+    make -j8 > /dev/null
+
+    cd /tmp/logs/alt-hostapd/hostapd
+    echo "Building hostapd"
+    make -j8 hostapd > /dev/null
+
+    cd /tmp/logs/alt-hostapd-as/hostapd
+    echo "Building hostapd/hlr_auc_gw (AS)"
+    make -j8 hostapd hlr_auc_gw > /dev/null
+
+    cd $DIR
+
+    mv /tmp/logs/alt-wpa_supplicant $LOGDIR
+    mv /tmp/logs/alt-hostapd $LOGDIR
+    mv /tmp/logs/alt-hostapd-as $LOGDIR
+else
+    CODECOV=no
+fi
+
+echo "Starting test run in a virtual machine"
+
 kvm \
 	-kernel $KERNEL -smp 4 \
 	$KVMARGS -m $MEMORY -nographic \
@@ -53,12 +107,42 @@ kvm \
 	-monitor null -serial stdio -serial file:$LOGDIR/console \
 	-append "mac80211_hwsim.channels=$CHANNELS mac80211_hwsim.radios=5 init=$CMD testdir=$TESTDIR console=$KVMOUT root=/dev/root rootflags=trans=virtio,version=9p2000.u ro rootfstype=9p EPATH=$EPATH ARGS=$*"
 
-echo LOGDIR=$LOGDIR
+if [ $CODECOV = "yes" ]; then
+    mv $LOGDIR/alt-wpa_supplicant /tmp/logs
+    mv $LOGDIR/alt-hostapd /tmp/logs
+    mv $LOGDIR/alt-hostapd-as /tmp/logs
+
+    echo "Generating code coverage report for wpa_supplicant"
+    cd /tmp/logs/alt-wpa_supplicant/wpa_supplicant
+    lcov -c -d .. > lcov.info 2> lcov.log
+    genhtml -t "wpa_supplicant hwsim test run $DATE" lcov.info --output-directory $LOGDIR/lcov-wpa_supplicant >> lcov.log 2>&1
+    mv lcov.info lcov.log $LOGDIR/lcov-wpa_supplicant
+
+    echo "Generating code coverage report for hostapd"
+    cd /tmp/logs/alt-hostapd/hostapd
+    lcov -c -d .. > lcov.info 2> lcov.log
+    genhtml -t "hostapd hwsim test run $DATE" lcov.info --output-directory $LOGDIR/lcov-hostapd >> lcov.log 2>&1
+    mv lcov.info lcov.log $LOGDIR/lcov-hostapd
+
+    echo "Generating code coverage report for hostapd/hlr_auc_gw (AS)"
+    cd /tmp/logs/alt-hostapd-as/hostapd
+    lcov -c -d .. > lcov.info 2> lcov.log
+    genhtml -t "hostapd/hlr_auc_gw (AS) hwsim test run $DATE" lcov.info --output-directory $LOGDIR/lcov-hostapd-as >> lcov.log 2>&1
+    mv lcov.info lcov.log $LOGDIR/lcov-hostapd-as
+
+    cd $DIR
+    rm -r /tmp/logs/alt-wpa_supplicant
+    rm -r /tmp/logs/alt-hostapd
+    rm -r /tmp/logs/alt-hostapd-as
+    rmdir /tmp/logs
+fi
 
-if [ -d $LOGDIR/gcov ]; then
-    echo "Move gcov data files from vm logdir to build directories"
-    for i in $LOGDIR/gcov/*.gcda; do
-	file=`basename $i | sed "s/.gcda$//"`
-	find ../../.. -name $file.gcno | sed s/.gcno/.gcda/ | xargs mv $i
-    done
+echo
+echo "Test run completed"
+echo "Logfiles are at $LOGDIR"
+if [ $CODECOV = "yes" ]; then
+    echo "Code coverage reports:"
+    echo "wpa_supplicant: file://$LOGDIR/lcov-wpa_supplicant/index.html"
+    echo "hostapd: file://$LOGDIR/lcov-hostapd/index.html"
+    echo "hostapd/hlr_auc_gw (AS): file://$LOGDIR/lcov-hostapd-as/index.html"
 fi