tests: Add support for sqlite results database
This is more convenient to use directly than going through the text-based results file. Signed-hostap: Jouni Malinen <j@w1.fi>
This commit is contained in:
Jouni Malinen
parent
00606de898
commit
b74b7e87bb
|
|
@ -175,3 +175,18 @@ line is a convenient way of verifying functionality.
|
|||
run-tests.py will automatically import all test cases from the test_*.py
|
||||
files in this directory. All functions starting with the "test_" prefix
|
||||
in these files are assumed to be test cases.
|
||||
|
||||
|
||||
Results database
|
||||
----------------
|
||||
|
||||
run-tests.py can be requested to write results from the execution of
|
||||
each test case into an sqlite database. The "-S <path to database>" and
|
||||
"-b <build id>" command line arguments can be used to do that. The
|
||||
database must have been prepared before this, e.g., with following:
|
||||
|
||||
cat | sqlite3 /tmp/example.db <<EOF
|
||||
CREATE TABLE results (test,result,run,time,duration,build,commitid);
|
||||
CREATE INDEX results_idx ON results (test);
|
||||
CREATE INDEX results_idx2 ON results (run);
|
||||
EOF
|
||||
|
|
|
|||
|
|
@ -3,6 +3,15 @@
|
|||
errors=0
|
||||
umask 0002
|
||||
|
||||
if [ -z "$DBFILE" ]; then
|
||||
DB=""
|
||||
else
|
||||
DB="-S $DBFILE"
|
||||
if [ -n "$BUILD" ]; then
|
||||
DB="$DB -b $BUILD"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "x$1" = "xconcurrent-valgrind" ]; then
|
||||
if ! ./start.sh concurrent valgrind; then
|
||||
echo "Could not start test environment" > logs/last-debug
|
||||
|
|
@ -11,7 +20,7 @@ if [ "x$1" = "xconcurrent-valgrind" ]; then
|
|||
DATE=`ls -1tr logs | tail -1 | cut -f1 -d-`
|
||||
rm logs/last-debug
|
||||
for i in autogo discovery grpform; do
|
||||
./run-tests.py -l logs/$DATE-run-$i -e logs/$DATE-failed-$i -r logs/results.txt -f test_p2p_$i.py || errors=1
|
||||
./run-tests.py -l logs/$DATE-run-$i $DB -e logs/$DATE-failed-$i -r logs/results.txt -f test_p2p_$i.py || errors=1
|
||||
cat logs/$DATE-run-$i >> logs/last-debug
|
||||
done
|
||||
./stop-wifi.sh
|
||||
|
|
@ -32,7 +41,7 @@ elif [ "x$1" = "xconcurrent" ]; then
|
|||
DATE=`ls -1tr logs | tail -1 | cut -f1 -d-`
|
||||
rm logs/last-debug
|
||||
for i in autogo discovery grpform; do
|
||||
./run-tests.py -l logs/$DATE-run-$i -e logs/$DATE-failed-$i -r logs/results.txt -f test_p2p_$i.py || errors=1
|
||||
./run-tests.py -l logs/$DATE-run-$i $DB -e logs/$DATE-failed-$i -r logs/results.txt -f test_p2p_$i.py || errors=1
|
||||
cat logs/$DATE-run-$i >> logs/last-debug
|
||||
done
|
||||
./stop-wifi.sh
|
||||
|
|
@ -46,7 +55,7 @@ elif [ "x$1" = "xvalgrind" ]; then
|
|||
exit 1
|
||||
fi
|
||||
DATE=`ls -1tr logs | tail -1 | cut -f1 -d-`
|
||||
./run-tests.py -l logs/$DATE-run -e logs/$DATE-failed -r logs/results.txt || errors=1
|
||||
./run-tests.py -l logs/$DATE-run $DB -e logs/$DATE-failed -r logs/results.txt || errors=1
|
||||
cat logs/$DATE-run > logs/last-debug
|
||||
./stop-wifi.sh
|
||||
failures=`grep "ERROR SUMMARY" logs/$DATE-valgrind-* | grep -v " 0 errors" | wc -l`
|
||||
|
|
@ -64,7 +73,7 @@ elif [ "x$1" = "xtrace" ]; then
|
|||
exit 1
|
||||
fi
|
||||
DATE=`ls -1tr logs | tail -1 | cut -f1 -d-`
|
||||
sudo trace-cmd record -o logs/$DATE-trace.dat -e mac80211 -e cfg80211 su $USER -c "./run-tests.py -l logs/$DATE-run -e logs/$DATE-failed -r logs/results.txt" || errors=1
|
||||
sudo trace-cmd record -o logs/$DATE-trace.dat -e mac80211 -e cfg80211 su $USER -c "./run-tests.py -l logs/$DATE-run $DB -e logs/$DATE-failed -r logs/results.txt" || errors=1
|
||||
if [ -e logs/$DATE-failed ]; then
|
||||
error=1
|
||||
fi
|
||||
|
|
@ -81,7 +90,7 @@ else
|
|||
exit 1
|
||||
fi
|
||||
DATE=`ls -1tr logs | tail -1 | cut -f1 -d-`
|
||||
./run-tests.py -l logs/$DATE-run -e logs/$DATE-failed -r logs/results.txt || errors=1
|
||||
./run-tests.py -l logs/$DATE-run $DB -e logs/$DATE-failed -r logs/results.txt || errors=1
|
||||
cat logs/$DATE-run > logs/last-debug
|
||||
./stop-wifi.sh
|
||||
if [ $errors -gt 0 ]; then
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import os
|
|||
import re
|
||||
import sys
|
||||
import time
|
||||
import sqlite3
|
||||
from datetime import datetime
|
||||
|
||||
import logging
|
||||
|
|
@ -29,11 +30,29 @@ def reset_devs(dev, apdev):
|
|||
for ap in apdev:
|
||||
hapd.remove(ap['ifname'])
|
||||
|
||||
def report(conn, build, commit, run, test, result, diff):
|
||||
if conn:
|
||||
if not build:
|
||||
build = ''
|
||||
if not commit:
|
||||
commit = ''
|
||||
sql = "INSERT INTO results(test,result,run,time,duration,build,commitid) VALUES('" + test.replace('test_', '', 1) + "', '" + result + "', " + str(run) + ", " + str(time.time()) + ", " + str(diff.total_seconds()) + ", '" + build + "', '" + commit + "')"
|
||||
try:
|
||||
conn.execute(sql)
|
||||
conn.commit()
|
||||
except Exception, e:
|
||||
print "sqlite: " + str(e)
|
||||
print "sql: " + sql
|
||||
|
||||
def main():
|
||||
test_file = None
|
||||
error_file = None
|
||||
log_file = None
|
||||
results_file = None
|
||||
conn = None
|
||||
run = None
|
||||
build = None
|
||||
commit = None
|
||||
idx = 1
|
||||
print_res = False
|
||||
if len(sys.argv) > 1 and sys.argv[1] == '-d':
|
||||
|
|
@ -60,9 +79,22 @@ def main():
|
|||
elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-f':
|
||||
test_file = sys.argv[idx + 1]
|
||||
idx = idx + 2
|
||||
elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-S':
|
||||
conn = sqlite3.connect(sys.argv[idx + 1])
|
||||
idx = idx + 2
|
||||
elif len(sys.argv) > idx + 1 and sys.argv[idx] == '-b':
|
||||
build = sys.argv[idx + 1]
|
||||
idx = idx + 2
|
||||
else:
|
||||
break
|
||||
|
||||
if conn:
|
||||
run = str(int(time.time()))
|
||||
with open("commit") as f:
|
||||
val = f.readlines()
|
||||
if len(val) > 0:
|
||||
commit = val[0].rstrip()
|
||||
|
||||
tests = []
|
||||
for t in os.listdir("."):
|
||||
m = re.match(r'(test_.*)\.py$', t)
|
||||
|
|
@ -125,6 +157,9 @@ def main():
|
|||
logger.info("Failed to issue TEST-START before " + t.__name__ + " for " + d.ifname)
|
||||
logger.info(e)
|
||||
print "FAIL " + t.__name__ + " - could not start test"
|
||||
if conn:
|
||||
conn.close()
|
||||
conn = None
|
||||
sys.exit(1)
|
||||
try:
|
||||
if t.func_code.co_argcount > 1:
|
||||
|
|
@ -135,11 +170,12 @@ def main():
|
|||
diff = end - start
|
||||
if res == "skip":
|
||||
skipped.append(t.__name__)
|
||||
result = "SKIP "
|
||||
result = "SKIP"
|
||||
else:
|
||||
passed.append(t.__name__)
|
||||
result = "PASS "
|
||||
result = result + t.__name__ + " "
|
||||
result = "PASS"
|
||||
report(conn, build, commit, run, t.__name__, result, diff)
|
||||
result = result + " " + t.__name__ + " "
|
||||
result = result + str(diff.total_seconds()) + " " + str(end)
|
||||
logger.info(result)
|
||||
if log_file or print_res:
|
||||
|
|
@ -154,6 +190,7 @@ def main():
|
|||
diff = end - start
|
||||
logger.info(e)
|
||||
failed.append(t.__name__)
|
||||
report(conn, build, commit, run, t.__name__, "FAIL", diff)
|
||||
result = "FAIL " + t.__name__ + " " + str(diff.total_seconds()) + " " + str(end)
|
||||
logger.info(result)
|
||||
if log_file:
|
||||
|
|
@ -173,6 +210,9 @@ def main():
|
|||
if not test_filter:
|
||||
reset_devs(dev, apdev)
|
||||
|
||||
if conn:
|
||||
conn.close()
|
||||
|
||||
if len(failed):
|
||||
logger.info("passed " + str(len(passed)) + " test case(s)")
|
||||
logger.info("skipped " + str(len(skipped)) + " test case(s)")
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ else
|
|||
fi
|
||||
|
||||
$DIR/stop-wifi.sh
|
||||
git show -s --format=%H > commit
|
||||
sudo modprobe mac80211_hwsim radios=5
|
||||
if [ "$CONCURRENT" = "y" ]; then
|
||||
sudo iw wlan0 interface add sta0 type station
|
||||
|
|
|
|||
Loading…
Reference in a new issue