tests: Add support for sqlite results database

This is more convenient to use directly than going through the
text-based results file.

Signed-hostap: Jouni Malinen <j@w1.fi>

tests/hwsim/README

@@ -175,3 +175,18 @@ line is a convenient way of verifying functionality.
 run-tests.py will automatically import all test cases from the test_*.py
 files in this directory. All functions starting with the "test_" prefix
 in these files are assumed to be test cases.
+
+
+Results database
+----------------
+
+run-tests.py can be requested to write results from the execution of
+each test case into an sqlite database. The "-S <path to database>" and
+"-b <build id>" command line arguments can be used to do that. The
+database must have been prepared before this, e.g., with following:
+
+cat | sqlite3 /tmp/example.db <<EOF
+CREATE TABLE results (test,result,run,time,duration,build,commitid);
+CREATE INDEX results_idx ON results (test);
+CREATE INDEX results_idx2 ON results (run);
+EOF

tests/hwsim/run-all.sh

@@ -3,6 +3,15 @@
 errors=0
 umask 0002
 
+if [ -z "$DBFILE" ]; then
+    DB=""
+else
+    DB="-S $DBFILE"
+    if [ -n "$BUILD" ]; then
+	DB="$DB -b $BUILD"
+    fi
+fi
+
 if [ "x$1" = "xconcurrent-valgrind" ]; then
     if ! ./start.sh concurrent valgrind; then
 	echo "Could not start test environment" > logs/last-debug
@@ -11,7 +20,7 @@ if [ "x$1" = "xconcurrent-valgrind" ]; then
     DATE=`ls -1tr logs | tail -1 | cut -f1 -d-`
     rm logs/last-debug
     for i in autogo discovery grpform; do
-	./run-tests.py -l logs/$DATE-run-$i -e logs/$DATE-failed-$i -r logs/results.txt -f test_p2p_$i.py || errors=1
+	./run-tests.py -l logs/$DATE-run-$i $DB -e logs/$DATE-failed-$i -r logs/results.txt -f test_p2p_$i.py || errors=1
 	cat logs/$DATE-run-$i >> logs/last-debug
     done
     ./stop-wifi.sh
@@ -32,7 +41,7 @@ elif [ "x$1" = "xconcurrent" ]; then
     DATE=`ls -1tr logs | tail -1 | cut -f1 -d-`
     rm logs/last-debug
     for i in autogo discovery grpform; do
-	./run-tests.py -l logs/$DATE-run-$i -e logs/$DATE-failed-$i -r logs/results.txt -f test_p2p_$i.py || errors=1
+	./run-tests.py -l logs/$DATE-run-$i $DB -e logs/$DATE-failed-$i -r logs/results.txt -f test_p2p_$i.py || errors=1
 	cat logs/$DATE-run-$i >> logs/last-debug
     done
     ./stop-wifi.sh
@@ -46,7 +55,7 @@ elif [ "x$1" = "xvalgrind" ]; then
 	exit 1
     fi
     DATE=`ls -1tr logs | tail -1 | cut -f1 -d-`
-    ./run-tests.py -l logs/$DATE-run -e logs/$DATE-failed -r logs/results.txt || errors=1
+    ./run-tests.py -l logs/$DATE-run $DB -e logs/$DATE-failed -r logs/results.txt || errors=1
     cat logs/$DATE-run > logs/last-debug
     ./stop-wifi.sh
     failures=`grep "ERROR SUMMARY" logs/$DATE-valgrind-* | grep -v " 0 errors" | wc -l`
@@ -64,7 +73,7 @@ elif [ "x$1" = "xtrace" ]; then
 	exit 1
     fi
     DATE=`ls -1tr logs | tail -1 | cut -f1 -d-`
-    sudo trace-cmd record -o logs/$DATE-trace.dat -e mac80211 -e cfg80211 su $USER -c "./run-tests.py -l logs/$DATE-run -e logs/$DATE-failed -r logs/results.txt" || errors=1
+    sudo trace-cmd record -o logs/$DATE-trace.dat -e mac80211 -e cfg80211 su $USER -c "./run-tests.py -l logs/$DATE-run $DB -e logs/$DATE-failed -r logs/results.txt" || errors=1
     if [ -e logs/$DATE-failed ]; then
 	error=1
     fi
@@ -81,7 +90,7 @@ else
 	exit 1
     fi
     DATE=`ls -1tr logs | tail -1 | cut -f1 -d-`
-    ./run-tests.py -l logs/$DATE-run -e logs/$DATE-failed -r logs/results.txt || errors=1
+    ./run-tests.py -l logs/$DATE-run $DB -e logs/$DATE-failed -r logs/results.txt || errors=1
     cat logs/$DATE-run > logs/last-debug
     ./stop-wifi.sh
     if [ $errors -gt 0 ]; then

tests/hwsim/run-tests.py

@@ -10,6 +10,7 @@ import os
 import re
 import sys
 import time
+import sqlite3
 from datetime import datetime
 
 import logging
@@ -29,11 +30,29 @@ def reset_devs(dev, apdev):
     for ap in apdev:
         hapd.remove(ap['ifname'])
 
+def report(conn, build, commit, run, test, result, diff):
+    if conn:
+        if not build:
+            build = ''
+        if not commit:
+            commit = ''
+        sql = "INSERT INTO results(test,result,run,time,duration,build,commitid) VALUES('" + test.replace('test_', '', 1) + "', '" + result + "', " + str(run) + ", " + str(time.time()) + ", " + str(diff.total_seconds()) + ", '" + build + "', '" + commit + "')"
+        try:
+            conn.execute(sql)
+            conn.commit()
+        except Exception, e:
+            print "sqlite: " + str(e)
+            print "sql: " + sql
+
 def main():
     test_file = None
     error_file = None
     log_file = None
     results_file = None
+    conn = None
+    run = None
+    build = None
+    commit = None
     idx = 1
     print_res = False
     if len(sys.argv) > 1 and sys.argv[1] == '-d':
@@ -60,9 +79,22 @@ def main():
         elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-f':
             test_file = sys.argv[idx + 1]
             idx = idx + 2
+        elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-S':
+            conn = sqlite3.connect(sys.argv[idx + 1])
+            idx = idx + 2
+        elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-b':
+            build = sys.argv[idx + 1]
+            idx = idx + 2
         else:
             break
 
+    if conn:
+        run = str(int(time.time()))
+        with open("commit") as f:
+            val = f.readlines()
+            if len(val) > 0:
+                commit = val[0].rstrip()
+
     tests = []
     for t in os.listdir("."):
         m = re.match(r'(test_.*)\.py$', t)
@@ -125,6 +157,9 @@ def main():
                 logger.info("Failed to issue TEST-START before " + t.__name__ + " for " + d.ifname)
                 logger.info(e)
                 print "FAIL " + t.__name__ + " - could not start test"
+                if conn:
+                    conn.close()
+                    conn = None
                 sys.exit(1)
         try:
             if t.func_code.co_argcount > 1:
@@ -135,11 +170,12 @@ def main():
             diff = end - start
             if res == "skip":
                 skipped.append(t.__name__)
-                result = "SKIP "
+                result = "SKIP"
             else:
                 passed.append(t.__name__)
-                result = "PASS "
+                result = "PASS"
-            result = result + t.__name__ + " "
+            report(conn, build, commit, run, t.__name__, result, diff)
+            result = result + " " + t.__name__ + " "
             result = result + str(diff.total_seconds()) + " " + str(end)
             logger.info(result)
             if log_file or print_res:
@@ -154,6 +190,7 @@ def main():
             diff = end - start
             logger.info(e)
             failed.append(t.__name__)
+            report(conn, build, commit, run, t.__name__, "FAIL", diff)
             result = "FAIL " + t.__name__ + " " + str(diff.total_seconds()) + " " + str(end)
             logger.info(result)
             if log_file:
@@ -173,6 +210,9 @@ def main():
     if not test_filter:
         reset_devs(dev, apdev)
 
+    if conn:
+        conn.close()
+
     if len(failed):
         logger.info("passed " + str(len(passed)) + " test case(s)")
         logger.info("skipped " + str(len(skipped)) + " test case(s)")

tests/hwsim/start.sh

@@ -35,6 +35,7 @@ else
 fi
 
 $DIR/stop-wifi.sh
+git show -s --format=%H > commit
 sudo modprobe mac80211_hwsim radios=5
 if [ "$CONCURRENT" = "y" ]; then
     sudo iw wlan0 interface add sta0 type station