tests: Add --codecov support to parallel-vm.py

This allows code coverage report to be generated must faster with the
help of parallel VMs executing test cases.

Signed-off-by: Jouni Malinen <j@w1.fi>

tests/hwsim/vm/build-codecov.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+LOGDIR=$1
+DIR=$PWD
+TMPDIR=/tmp/logs
+
+if [ -e $TMPDIR ]; then
+	echo "$TMPDIR exists - cannot prepare build trees"
+	exit 1
+fi
+mkdir $TMPDIR
+echo "Preparing separate build trees for hostapd/wpa_supplicant"
+cd ../../..
+git archive --format=tar --prefix=hostap/ HEAD > $TMPDIR/hostap.tar
+cd $DIR
+cat ../../../wpa_supplicant/.config > $TMPDIR/wpa_supplicant.config
+echo "CONFIG_CODE_COVERAGE=y" >> $TMPDIR/wpa_supplicant.config
+cat ../../../hostapd/.config > $TMPDIR/hostapd.config
+echo "CONFIG_CODE_COVERAGE=y" >> $TMPDIR/hostapd.config
+
+cd $TMPDIR
+tar xf hostap.tar
+mv hostap alt-wpa_supplicant
+mv wpa_supplicant.config alt-wpa_supplicant/wpa_supplicant/.config
+tar xf hostap.tar
+mv hostap alt-hostapd
+cp hostapd.config alt-hostapd/hostapd/.config
+tar xf hostap.tar
+mv hostap alt-hostapd-as
+cp hostapd.config alt-hostapd-as/hostapd/.config
+tar xf hostap.tar
+mv hostap alt-hlr_auc_gw
+mv hostapd.config alt-hlr_auc_gw/hostapd/.config
+rm hostap.tar
+
+cd $TMPDIR/alt-wpa_supplicant/wpa_supplicant
+echo "Building wpa_supplicant"
+make -j8 > /dev/null
+
+cd $TMPDIR/alt-hostapd/hostapd
+echo "Building hostapd"
+make -j8 hostapd > /dev/null
+
+cd $TMPDIR/alt-hostapd-as/hostapd
+echo "Building hostapd (AS)"
+make -j8 hostapd > /dev/null
+
+cd $TMPDIR/alt-hlr_auc_gw/hostapd
+echo "Building hlr_auc_gw"
+make -j8 hlr_auc_gw > /dev/null
+
+cd $DIR
+
+mv $TMPDIR/alt-wpa_supplicant $LOGDIR
+mv $TMPDIR/alt-hostapd $LOGDIR
+mv $TMPDIR/alt-hostapd-as $LOGDIR
+mv $TMPDIR/alt-hlr_auc_gw $LOGDIR

tests/hwsim/vm/combine-codecov.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+LOGDIR=$1
+TMPDIR=/tmp/logs
+
+mv $LOGDIR/alt-* $TMPDIR
+
+cd $TMPDIR
+args=""
+for i in lcov-*.info-*; do
+    args="$args -a $i"
+done
+
+lcov $args -o $LOGDIR/combined.info > $LOGDIR/combined-lcov.log 2>&1
+cd $LOGDIR
+genhtml -t "wpa_supplicant/hostapd combined for hwsim test run $(date +%s)" combined.info --output-directory . > lcov.log 2>&1
+
+rm -r /tmp/logs/alt-wpa_supplicant
+rm -r /tmp/logs/alt-hostapd
+rm -r /tmp/logs/alt-hostapd-as
+rm -r /tmp/logs/alt-hlr_auc_gw
+rm /tmp/logs/lcov-*info-*
+rmdir /tmp/logs

tests/hwsim/vm/parallel-vm.py

@@ -126,20 +126,38 @@ def main():
     global tests
 
     if len(sys.argv) < 2:
-        sys.exit("Usage: %s <number of VMs> [params..]" % sys.argv[0])
+        sys.exit("Usage: %s <number of VMs> [--codecov] [params..]" % sys.argv[0])
     num_servers = int(sys.argv[1])
     if num_servers < 1:
         sys.exit("Too small number of VMs")
 
+    timestamp = int(time.time())
+
+    if len(sys.argv) > 2 and sys.argv[2] == "--codecov":
+        idx = 3
+        print "Code coverage - build separate binaries"
+        logdir = "/tmp/hwsim-test-logs/" + str(timestamp)
+        os.makedirs(logdir)
+        subprocess.check_call(['./build-codecov.sh', logdir])
+        codecov_args = ['--codecov_dir', logdir]
+        codecov = True
+    else:
+        idx = 2
+        codecov_args = []
+        codecov = False
+
     tests = []
-    cmd = [ '../run-tests.py', '-L' ] + sys.argv[2:]
+    cmd = [ '../run-tests.py', '-L' ] + sys.argv[idx:]
     lst = subprocess.Popen(cmd, stdout=subprocess.PIPE)
     for l in lst.stdout.readlines():
         name = l.split(' ')[0]
         tests.append(name)
     if len(tests) == 0:
         sys.exit("No test cases selected")
-    extra_args = [x for x in sys.argv[2:] if x not in tests]
+    if '-f' in sys.argv[idx:]:
+        extra_args = sys.argv[idx:]
+    else:
+        extra_args = [x for x in sys.argv[idx:] if x not in tests]
 
     dir = '/tmp/hwsim-test-logs'
     try:
@@ -147,13 +165,12 @@ def main():
     except:
         pass
 
-    timestamp = int(time.time())
     vm = {}
     for i in range(0, num_servers):
         print("\rStarting virtual machine {}/{}".format(i + 1, num_servers)),
         cmd = ['./vm-run.sh', '--timestamp', str(timestamp),
                '--ext', 'srv.%d' % (i + 1),
-               '-i'] + extra_args
+               '-i'] + codecov_args + extra_args
         vm[i] = {}
         vm[i]['proc'] = subprocess.Popen(cmd,
                                          stdin=subprocess.PIPE,
@@ -189,5 +206,14 @@ def main():
             if "Kernel panic" in f.read():
                 print "Kernel panic in " + log
 
+    if codecov:
+        print "Code coverage - preparing report"
+        for i in range(num_servers):
+            subprocess.check_call(['./process-codecov.sh',
+                                   logdir + ".srv.%d" % (i + 1),
+                                   str(i)])
+        subprocess.check_call(['./combine-codecov.sh', logdir])
+        print "file://%s/index.html" % logdir
+
 if __name__ == "__main__":
     main()

tests/hwsim/vm/process-codecov.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+LOGDIR=$1
+POSTFIX=$2
+
+DIR=$PWD
+TMPDIR=/tmp/logs
+
+mv $LOGDIR/alt-wpa_supplicant $TMPDIR
+mv $LOGDIR/alt-hostapd $TMPDIR
+mv $LOGDIR/alt-hostapd-as $TMPDIR
+mv $LOGDIR/alt-hlr_auc_gw $TMPDIR
+
+cd $TMPDIR/alt-wpa_supplicant/wpa_supplicant
+lcov -c -d .. 2> lcov.log | sed s%SF:/tmp/logs/alt-[^/]*/%SF:/tmp/logs/alt-wpa_supplicant/% > $TMPDIR/lcov-wpa_supplicant.info-$POSTFIX &
+
+cd $TMPDIR/alt-hostapd/hostapd
+lcov -c -d .. 2> lcov.log | sed s%SF:/tmp/logs/alt-[^/]*/%SF:/tmp/logs/alt-wpa_supplicant/% > $TMPDIR/lcov-hostapd.info-$POSTFIX &
+
+cd $TMPDIR/alt-hostapd-as/hostapd
+lcov -c -d .. 2> lcov.log | sed s%SF:/tmp/logs/alt-[^/]*/%SF:/tmp/logs/alt-wpa_supplicant/% > $TMPDIR/lcov-hostapd-as.info-$POSTFIX &
+
+cd $TMPDIR/alt-hlr_auc_gw/hostapd
+lcov -c -d .. 2> lcov.log | sed s%SF:/tmp/logs/alt-[^/]*/%SF:/tmp/logs/alt-wpa_supplicant/% > $TMPDIR/lcov-hlr_auc_gw.info-$POSTFIX &
+wait
+
+cd $DIR
+rm -r $TMPDIR/alt-wpa_supplicant
+rm -r $TMPDIR/alt-hostapd
+rm -r $TMPDIR/alt-hostapd-as
+rm -r $TMPDIR/alt-hlr_auc_gw

tests/hwsim/vm/vm-run.sh

@@ -47,6 +47,7 @@ TIMESTAMP=$(date +%s)
 DATE=$TIMESTAMP
 CODECOV=no
 TIMEWARP=0
+CODECOV_DIR=
 while [ "$1" != "" ]; do
 	case $1 in
 		--timestamp ) shift
@@ -60,6 +61,10 @@ while [ "$1" != "" ]; do
 		--codecov ) shift
 			CODECOV=yes
 			;;
+		--codecov_dir ) shift
+			CODECOV_DIR=$1
+			shift
+			;;
 		--timewrap ) shift
 			TIMEWARP=1
 			;;
@@ -73,59 +78,13 @@ done
 LOGDIR=$LOGS/$DATE
 mkdir -p $LOGDIR
 
-if [ $CODECOV = "yes" ]; then
-    DIR=$PWD
-    if [ -e /tmp/logs ]; then
-	echo "/tmp/logs exists - cannot prepare build trees"
-	exit 1
-    fi
-    mkdir /tmp/logs
-    echo "Preparing separate build trees for hostapd/wpa_supplicant"
-    cd ../../..
-    git archive --format=tar --prefix=hostap/ HEAD > /tmp/logs/hostap.tar
-    cd $DIR
-    cat ../../../wpa_supplicant/.config > /tmp/logs/wpa_supplicant.config
-    echo "CONFIG_CODE_COVERAGE=y" >> /tmp/logs/wpa_supplicant.config
-    cat ../../../hostapd/.config > /tmp/logs/hostapd.config
-    echo "CONFIG_CODE_COVERAGE=y" >> /tmp/logs/hostapd.config
-
-    cd /tmp/logs
-    tar xf hostap.tar
-    mv hostap alt-wpa_supplicant
-    mv wpa_supplicant.config alt-wpa_supplicant/wpa_supplicant/.config
-    tar xf hostap.tar
-    mv hostap alt-hostapd
-    cp hostapd.config alt-hostapd/hostapd/.config
-    tar xf hostap.tar
-    mv hostap alt-hostapd-as
-    cp hostapd.config alt-hostapd-as/hostapd/.config
-    tar xf hostap.tar
-    mv hostap alt-hlr_auc_gw
-    mv hostapd.config alt-hlr_auc_gw/hostapd/.config
-    rm hostap.tar
-
-    cd /tmp/logs/alt-wpa_supplicant/wpa_supplicant
-    echo "Building wpa_supplicant"
-    make -j8 > /dev/null
-
-    cd /tmp/logs/alt-hostapd/hostapd
-    echo "Building hostapd"
-    make -j8 hostapd > /dev/null
-
-    cd /tmp/logs/alt-hostapd-as/hostapd
-    echo "Building hostapd (AS)"
-    make -j8 hostapd > /dev/null
-
-    cd /tmp/logs/alt-hlr_auc_gw/hostapd
-    echo "Building hlr_auc_gw"
-    make -j8 hlr_auc_gw > /dev/null
-
-    cd $DIR
-
-    mv /tmp/logs/alt-wpa_supplicant $LOGDIR
-    mv /tmp/logs/alt-hostapd $LOGDIR
-    mv /tmp/logs/alt-hostapd-as $LOGDIR
-    mv /tmp/logs/alt-hlr_auc_gw $LOGDIR
+if [ -n "$CODECOV_DIR" ]; then
+    cp -a $CODECOV_DIR/alt-wpa_supplicant $LOGDIR
+    cp -a $CODECOV_DIR/alt-hostapd $LOGDIR
+    cp -a $CODECOV_DIR/alt-hostapd-as $LOGDIR
+    cp -a $CODECOV_DIR/alt-hlr_auc_gw $LOGDIR
+elif [ $CODECOV = "yes" ]; then
+    ./build-codecov.sh $LOGDIR || exit 1
 else
     CODECOV=no
 fi
@@ -143,6 +102,7 @@ kvm \
 	-append "mac80211_hwsim.support_p2p_device=0 mac80211_hwsim.channels=$CHANNELS mac80211_hwsim.radios=6 init=$CMD testdir=$TESTDIR timewarp=$TIMEWARP console=$KVMOUT root=/dev/root rootflags=trans=virtio,version=9p2000.u ro rootfstype=9p EPATH=$EPATH ARGS=$RUN_TEST_ARGS"
 
 if [ $CODECOV = "yes" ]; then
+    DIR=$PWD
     mv $LOGDIR/alt-wpa_supplicant /tmp/logs
     mv $LOGDIR/alt-hostapd /tmp/logs
     mv $LOGDIR/alt-hostapd-as /tmp/logs