You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
302 lines
9.4 KiB
Python
302 lines
9.4 KiB
Python
#!/usr/bin/env python
|
|
|
|
|
|
from __future__ import print_function
|
|
import sys
|
|
import os
|
|
import re
|
|
import collections
|
|
|
|
debug = False
|
|
|
|
|
|
class Arch:
|
|
def __init__(self, arch):
|
|
arch = re.sub("^rv(32|64|128)g", 'rv\\1imafd', arch)
|
|
m = re.match('rv(32|64|128)', arch)
|
|
self.base_arch = m.group(0)
|
|
self.ext = []
|
|
i = len(self.base_arch)
|
|
|
|
while i < len(arch):
|
|
ext = arch[i]
|
|
if ext in ['x', 's', 'z']:
|
|
extlen = 1
|
|
while (i+extlen) < len(arch) and arch[i+extlen] != '_':
|
|
extlen += 1
|
|
self.ext.append(arch[i:i + extlen])
|
|
i += extlen
|
|
elif ext == '_':
|
|
i += 1
|
|
else:
|
|
self.ext.append(ext)
|
|
i += 1
|
|
|
|
|
|
def usage():
|
|
print("%s <toolname> <libc>" \
|
|
" <white-list-base-dir> <testsuite.sum>" % sys.argv[0])
|
|
|
|
|
|
def get_white_list_files(raw_arch, abi, libc, white_list_base_dir):
|
|
""" Return white file list according the arch, abi, libc name and component.
|
|
"""
|
|
white_list_files = []
|
|
arch = Arch(raw_arch)
|
|
|
|
def append_if_exist(filename):
|
|
if debug:
|
|
print ("Try append: %s" % filename)
|
|
filepath = os.path.join(white_list_base_dir, filename)
|
|
if os.path.exists(filepath):
|
|
if debug:
|
|
print ("Got: %s" % filename)
|
|
white_list_files.append(filepath)
|
|
|
|
libc_filename = "common.log"
|
|
append_if_exist(libc_filename)
|
|
|
|
libc_filename = "%s.log" % (libc)
|
|
append_if_exist(libc_filename)
|
|
|
|
filename = "%s.log" % (arch.base_arch)
|
|
append_if_exist(filename)
|
|
|
|
filename = "%s.log" % (abi)
|
|
append_if_exist(filename)
|
|
|
|
filename = "%s.%s.log" % (arch.base_arch, abi)
|
|
append_if_exist(filename)
|
|
|
|
filename = "%s.%s.log" % (libc, arch.base_arch)
|
|
append_if_exist(filename)
|
|
|
|
filename = "%s.%s.log" % (libc, abi)
|
|
append_if_exist(filename)
|
|
|
|
filename = "%s.%s.%s.log" % (libc, arch.base_arch, abi)
|
|
append_if_exist(filename)
|
|
|
|
for ext in arch.ext:
|
|
filename = "%s.log" % (ext)
|
|
append_if_exist(filename)
|
|
|
|
filename = "%s.%s.log" % (arch.base_arch, ext)
|
|
append_if_exist(filename)
|
|
|
|
filename = "%s.%s.log" % (ext, abi)
|
|
append_if_exist(filename)
|
|
|
|
filename = "%s.%s.%s.log" % (arch.base_arch, ext, abi)
|
|
append_if_exist(filename)
|
|
|
|
filename = "%s.%s.log" % (libc, ext)
|
|
append_if_exist(filename)
|
|
|
|
filename = "%s.%s.%s.log" % (libc, arch.base_arch, ext)
|
|
append_if_exist(filename)
|
|
|
|
filename = "%s.%s.%s.log" % (libc, ext, abi)
|
|
append_if_exist(filename)
|
|
|
|
filename = "%s.%s.%s.%s.log" % (libc, arch.base_arch, ext, abi)
|
|
append_if_exist(filename)
|
|
|
|
return white_list_files
|
|
|
|
|
|
def read_white_lists(white_list_files, is_gcc):
|
|
if is_gcc:
|
|
white_lists = dict()
|
|
else:
|
|
white_lists = set()
|
|
for fname in white_list_files:
|
|
with open(fname) as f:
|
|
content = f.readlines()
|
|
for l in content:
|
|
l = l.strip()
|
|
if len(l) == 0:
|
|
continue
|
|
if l[0] == '#':
|
|
continue
|
|
|
|
if is_gcc:
|
|
try:
|
|
key = l.split(' ')[1]
|
|
except:
|
|
print ("Corrupt allowlist file?")
|
|
print ("Each line must contail <STATUS>: .*")
|
|
print ("e.g. FAIL: g++.dg/pr83239.C")
|
|
print ("Or starts with # for comment")
|
|
if key not in white_lists:
|
|
white_lists[key] = []
|
|
white_lists[key].append(l)
|
|
else:
|
|
white_lists.add(l)
|
|
|
|
return white_lists
|
|
|
|
|
|
def read_sum(sum_files):
|
|
unexpected_results = dict()
|
|
for sum_file in sum_files:
|
|
with open(sum_file) as f:
|
|
content = f.readlines()
|
|
current_target = None
|
|
variations = []
|
|
scan_variations = False
|
|
unexpected_result = dict()
|
|
tool = os.path.basename(sum_file).split(".")[0]
|
|
for l in content:
|
|
if l.startswith("Schedule of variations"):
|
|
scan_variations = True
|
|
continue
|
|
if scan_variations and l.startswith(" "):
|
|
variations.append(l.strip())
|
|
continue
|
|
scan_variations = False
|
|
|
|
if l.startswith("Running target"):
|
|
# Parsing current running target.
|
|
current_target = l.split(" ")[-1].strip()
|
|
unexpected_result[current_target] = list()
|
|
elif l.startswith("FAIL") or l.startswith("XPASS") \
|
|
or l.startswith("ERROR"):
|
|
unexpected_result[current_target].append(l.strip())
|
|
unexpected_results[tool] = unexpected_result
|
|
# tool -> variation(target) -> list of unexpected result
|
|
return unexpected_results
|
|
|
|
|
|
def get_white_list(arch, abi, libc, white_list_base_dir, is_gcc):
|
|
white_list_files = \
|
|
get_white_list_files(arch, abi, libc, white_list_base_dir)
|
|
white_list = read_white_lists(white_list_files, is_gcc)
|
|
return white_list
|
|
|
|
|
|
|
|
def filter_result(tool, libc, white_list_base_dir, unexpected_results):
|
|
summary = dict()
|
|
any_fail = False
|
|
is_gcc = tool == 'gcc'
|
|
# Filter with white list.
|
|
for testtool, variation_unexpected_result in unexpected_results.items():
|
|
for variation, unexpected_result in variation_unexpected_result.items():
|
|
# Extract variation to arch/abi
|
|
arch = ""
|
|
abi = ""
|
|
cmodel = ""
|
|
for info in variation.split('/'):
|
|
if info.startswith('-march'):
|
|
arch = info[7:]
|
|
elif info.startswith('-mabi'):
|
|
abi = info[6:]
|
|
elif info.startswith('-mcmodel'):
|
|
cmodel = info[9:]
|
|
|
|
white_list = \
|
|
get_white_list(arch, abi, libc,
|
|
os.path.join(white_list_base_dir, tool),
|
|
is_gcc)
|
|
# filter!
|
|
config = (arch, abi, cmodel)
|
|
fail_count = 0
|
|
unexpected_result_list = []
|
|
if is_gcc:
|
|
case_count = set()
|
|
for ur in unexpected_result:
|
|
key = ur.split(' ')[1]
|
|
if key in white_list and \
|
|
any(map(lambda x:ur.startswith(x),
|
|
white_list[key])):
|
|
# This item can be ignored
|
|
continue
|
|
else:
|
|
unexpected_result_list.append(ur)
|
|
fail_count += 1
|
|
case_count.add(key)
|
|
any_fail = True
|
|
|
|
if config not in summary:
|
|
summary[config] = dict()
|
|
summary[config][testtool] = (fail_count, len(case_count))
|
|
else:
|
|
for ur in unexpected_result:
|
|
if ur not in white_list:
|
|
unexpected_result_list.append(ur)
|
|
fail_count += 1
|
|
any_fail = True
|
|
|
|
if config not in summary:
|
|
summary[config] = dict()
|
|
summary[config][testtool] = fail_count
|
|
|
|
|
|
if len(unexpected_result_list) != 0:
|
|
print ("\t\t=== %s: Unexpected fails for %s %s %s ===" \
|
|
% (testtool, arch, abi, cmodel))
|
|
for ur in unexpected_result_list:
|
|
print (ur)
|
|
|
|
|
|
# Generate summary report.
|
|
if tool == 'gcc':
|
|
toollist = ['gcc', 'g++', 'gfortran']
|
|
elif tool == 'binutils':
|
|
toollist = ['binutils', 'ld', 'gas']
|
|
else:
|
|
raise Exception("Unsupported tool `%s`" % tool)
|
|
|
|
bar_item = map(lambda x: "%13s" % x, toollist)
|
|
bar = " |".join(bar_item)
|
|
print ("\n ========= Summary of %s testsuite =========" % tool)
|
|
if is_gcc:
|
|
print (" | # of unexpected case / # of unique unexpected case")
|
|
else:
|
|
print (" | # of unexpected case")
|
|
print (" |%s |" % bar)
|
|
for config, result in summary.items():
|
|
arch, abi, cmodel = config
|
|
print (" %10s/ %6s/ %6s |" % (arch, abi, cmodel), end='')
|
|
for tool in toollist:
|
|
if tool not in summary[config]:
|
|
print ("%7s |" % '-', end='')
|
|
continue
|
|
|
|
if is_gcc:
|
|
fail_count, case_count = summary[config][tool]
|
|
print ("%5d / %5d |" % (fail_count, case_count), end='')
|
|
else:
|
|
fail_count = summary[config][tool]
|
|
print ("%13d |" % fail_count, end='')
|
|
print ("")
|
|
if any_fail:
|
|
return 1
|
|
else:
|
|
return 0
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) < 5:
|
|
usage()
|
|
sys.exit()
|
|
tool, libc, white_list_base_dir, sum_files = sys.argv[1:5]
|
|
|
|
rv = 0
|
|
|
|
sum_files = sum_files.split(',')
|
|
unexpected_results = read_sum(sum_files)
|
|
if tool in ['gcc', 'binutils']:
|
|
rv = filter_result(tool, libc, white_list_base_dir,
|
|
unexpected_results)
|
|
else:
|
|
print ("Unsupported tool: `%s`" % tool)
|
|
rv = 1
|
|
|
|
sys.exit(rv)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|