patrick/zephyr
1
0
Fork 0
Code Issues Pull requests Actions 8 Packages Projects Releases Wiki Activity

twister: add options to shuffle tests across subsets

Browse source 
Add option to shuffle tests (randomly) before they are split into subsets.
This allow to fairly distribute tests across subsets,
so that long tests are not crowded in a single group.

This also allows to detect unwanted dependencies in test execution order.

Also, added option to provide custom seed to random generator
used to shuffle tests.
This will allow to reproduce certain test order if needed.
Used seed is printed at console.

Signed-off-by: Piotr Kosycarz <piotr.kosycarz@nordicsemi.no>
This commit is contained in:
Piotr Kosycarz 2023-04-21 14:09:10 +02:00 committed by Anas Nashif
parent 04d97569d1
commit 7892f40a52
2 changed files with 31 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
scripts/pylib/twister/twisterlib/environment.py
View file

@ -247,6 +247,17 @@ structure in the main Zephyr tree: boards/<arch>/<board_name>/""")
"This option is useful when running a large number of tests on "
"different hosts to speed up execution time.")
parser.add_argument(
"--shuffle-tests", action="store_true", default=None,
help="""Shuffle test execution order to get randomly distributed tests across subsets.
Used only when --subset is provided.""")
parser.add_argument(
"--shuffle-tests-seed", action="store", default=None,
help="""Seed value for random generator used to shuffle tests.
If not provided, seed in generated by system.
Used only when --shuffle-tests is provided.""")
parser.add_argument("-C", "--coverage", action="store_true",
help="Generate coverage reports. Implies "
"--enable-coverage.")
@ -698,6 +709,14 @@ def parse_arguments(parser, args, options = None):
logger.error("--device-flash-with-test requires --device-testing")
sys.exit(1)
if options.shuffle_tests and options.subset is None:
logger.error("--shuffle-tests requires --subset")
sys.exit(1)
if options.shuffle_tests_seed and options.shuffle_tests is None:
logger.error("--shuffle-tests-seed requires --shuffle-tests")
sys.exit(1)
if options.coverage_formats and (options.coverage_tool != "gcovr"):
logger.error("""--coverage-formats can only be used when coverage
tool is set to gcovr""")

12
scripts/pylib/twister/twisterlib/testplan.py
View file

@ -15,6 +15,7 @@ from itertools import islice
import logging
import copy
import shutil
import random
logger = logging.getLogger('twister')
logger.setLevel(logging.DEBUG)
@ -251,6 +252,17 @@ class TestPlan:
else:
self.instances = OrderedDict(sorted(self.instances.items()))
if self.options.shuffle_tests:
seed_value = int.from_bytes(os.urandom(8), byteorder="big")
if self.options.shuffle_tests_seed is not None:
seed_value = self.options.shuffle_tests_seed
logger.info(f"Shuffle tests with seed: {seed_value}")
random.seed(seed_value)
temp_list = list(self.instances.items())
random.shuffle(temp_list)
self.instances = OrderedDict(temp_list)
# Do calculation based on what is actually going to be run and evaluated
# at runtime, ignore the cases we already know going to be skipped.
# This fixes an issue where some sets would get majority of skips and