patrick/zephyr
1
0
Fork 0
Code Issues Pull requests Actions 8 Packages Projects Releases Wiki Activity

sanitycheck: add testcase.ini field for slow tests

Browse source 
Some tests take a very long time to complete, add a directive in
testcase.ini to mark them as such. Unless --enable-slow is passed
on the command line, these tests are not executed, only compiled.

app_kernel and test_sha256 marked as slow test cases. It appears
test_sha256 was being skipped earlier due to platform_whitelist line
which has been removed; the test should be able to run on all
microkernel-supporting boards. Same with test_aes, which completes
very quickly and is not marked as 'slow'.

Change-Id: I39ec8212f3fa10122ff786c10b6659d22bae64e3
Signed-off-by: Andrew Boie <andrew.p.boie@intel.com>
This commit is contained in:
Andrew Boie 2016-02-10 13:39:00 -08:00 committed by Gerrit Code Review
parent 0d8308ec3f
commit 6bb087c511
4 changed files with 26 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
samples/microkernel/benchmark/app_kernel/testcase.ini
View file

@ -3,4 +3,4 @@ tags = benchmark
arch_whitelist = x86
# On my machine, takes about 110 to run, 180 to be safe
timeout = 180
slow = True

1
samples/microkernel/test/test_aes/testcase.ini
View file

@ -1,4 +1,3 @@
[test]
tags = crypto aes
build_only = false
platform_whitelist = basic_minuteia basic_cortex_m3

2
samples/microkernel/test/test_sha256/testcase.ini
View file

@ -2,4 +2,4 @@
tags = crypto sha256
build_only = false
timeout = 10800
platform_whitelist = basic_minuteia basic_cortex_m3
slow = True

33
scripts/sanitycheck
View file

@ -21,14 +21,20 @@ Each testcase.ini block can define the following key/value pairs:
functional domains but can be anything. Command line invocations
of this script can filter the set of tests to run based on tag.
skip = <True|False>
skip = <True|False> (default False)
skip testcase unconditionally. This can be used for broken tests.
slow = <True|False> (default False)
Don't run this test case unless --enable-slow was passed in on the
command line. Intended for time-consuming test cases that are only
run under certain circumstances, like daily builds. These test cases
are still compiled.
extra_args = <list of extra arguments>
Extra arguments to pass to Make when building or running the
test case.
build_only = <True|False>
build_only = <True|False> (default False)
If true, don't try to run the test under QEMU even if the
selected platform supports it.
@ -614,7 +620,7 @@ class MakeGenerator:
run_logfile, qemu_logfile)
def add_test_instance(self, ti, build_only=False):
def add_test_instance(self, ti, build_only=False, enable_slow=False):
"""Add a goal to build/test a TestInstance object
@param ti TestInstance object to build. The status dictionary returned
@ -623,7 +629,8 @@ class MakeGenerator:
args = ti.test.extra_args[:]
args.extend(["ARCH=%s" % ti.platform.arch.name,
"BOARD=%s" % ti.platform.name])
if ti.platform.qemu_support and not ti.build_only and not build_only:
if (ti.platform.qemu_support and (not ti.build_only) and
(not build_only) and (enable_slow or not ti.test.slow)):
self.add_qemu_goal(ti.name, ti.test.code_location, ti.outdir,
args, ti.test.timeout)
else:
@ -716,6 +723,7 @@ testcase_valid_keys = {"tags" : {"type" : "set", "required" : True},
"extra_args" : {"type" : "list"},
"build_only" : {"type" : "bool", "default" : False},
"skip" : {"type" : "bool", "default" : False},
"slow" : {"type" : "bool", "default" : False},
"timeout" : {"type" : "int", "default" : 60},
"arch_whitelist" : {"type" : "set"},
"arch_exclude" : {"type" : "set"},
@ -965,6 +973,7 @@ class TestCase:
self.config_whitelist = tc_dict["config_whitelist"]
self.timeout = tc_dict["timeout"]
self.build_only = tc_dict["build_only"]
self.slow = tc_dict["slow"]
self.path = os.path.join(workdir, name)
self.name = self.path # for now
self.ktype = None
@ -993,7 +1002,8 @@ class TestInstance:
@param base_outdir Base directory for all test results. The actual
out directory used is <outdir>/<platform>/<test case name>
"""
def __init__(self, test, platform, base_outdir, build_only=False):
def __init__(self, test, platform, base_outdir, build_only=False,
slow=False):
self.test = test
self.platform = platform
self.name = os.path.join(platform.name, test.path)
@ -1293,10 +1303,10 @@ class TestSuite:
def add_instance(self, ti):
self.instances[ti.name] = ti
def execute(self, cb, cb_context, build_only):
def execute(self, cb, cb_context, build_only, enable_slow):
mg = MakeGenerator(self.outdir)
for i in self.instances.values():
mg.add_test_instance(i, build_only)
mg.add_test_instance(i, build_only, enable_slow)
self.goals = mg.execute(cb, cb_context)
for name, goal in self.goals.iteritems():
i = self.instances[name]
@ -1491,6 +1501,9 @@ def parse_arguments():
help="Don't run sanity checks. Instead, produce a report to "
"stdout detailing RAM/ROM sizes on the specified filenames. "
"All other command line arguments ignored.")
parser.add_argument("-S", "--enable-slow", action="store_true",
help="Execute time-consuming test cases that have been marked "
"as 'slow' in testcase.ini. Normally these are only built.")
return parser.parse_args()
@ -1605,9 +1618,11 @@ def main():
return
if VERBOSE or not TERMINAL:
goals = ts.execute(chatty_test_cb, ts.instances, args.build_only)
goals = ts.execute(chatty_test_cb, ts.instances, args.build_only,
args.enable_slow)
else:
goals = ts.execute(terse_test_cb, ts.instances, args.build_only)
goals = ts.execute(terse_test_cb, ts.instances, args.build_only,
args.enable_slow)
print
deltas = ts.compare_metrics(LAST_SANITY if args.last_metrics