patrick/zephyr
1
0
Fork 0
Code Issues Pull requests Actions 7 Packages Projects Releases Wiki Activity

twister: introduce twister-level timeout multiplier

Browse source 
Twister allows us to control maximum execution time for each test
with timeout value in test's .yaml configuration and
platform level timeout multiplier which allows us to tweak
timeout value for specific platform.

However, sometimes we want to additionally adjust tests timeouts
when running twister. This is especially useful in case of
simulation platform as simulation time may depend on the host
speed & load, we may select different (i.e. cycle accurate but
slower one) simulation method, etc...

Let's introduce global (twister-level) timeout multiplier option.

Signed-off-by: Eugeniy Paltsev <Eugeniy.Paltsev@synopsys.com>
Signed-off-by: Evgeniy Paltsev <PaltsevEvgeniy@gmail.com>
This commit is contained in:
Evgeniy Paltsev 2023-09-21 18:06:17 +01:00 committed by Carles Cufí
parent c2de739f1f
commit 80d2872a41
3 changed files with 23 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
scripts/pylib/twister/twisterlib/environment.py
View file

@ -196,6 +196,11 @@ Artificially long but functional example:
help="""Only run device tests with current artifacts, do not build
the code""")
parser.add_argument("--timeout-multiplier", type=float, default=1,
help="""Globally adjust tests timeouts by specified multiplier. The resulting test
timeout would be multiplication of test timeout value, board-level timeout multiplier
and global timeout multiplier (this parameter)""")
test_xor_subtest.add_argument(
"-s", "--test", action="append",
help="Run only the specified testsuite scenario. These are named by "

19
scripts/pylib/twister/twisterlib/handlers.py
View file

@ -81,7 +81,6 @@ class Handler:
self.name = instance.name
self.instance = instance
self.timeout = math.ceil(instance.testsuite.timeout * instance.platform.timeout_multiplier)
self.sourcedir = instance.testsuite.source_dir
self.build_dir = instance.build_dir
self.log = os.path.join(self.build_dir, "handler.log")
@ -94,6 +93,11 @@ class Handler:
self.args = []
self.terminated = False
def get_test_timeout(self):
return math.ceil(self.instance.testsuite.timeout *
self.instance.platform.timeout_multiplier *
self.options.timeout_multiplier)
def record(self, harness):
if harness.recording:
filename = os.path.join(self.build_dir, "recording.csv")
@ -189,7 +193,7 @@ class BinaryHandler(Handler):
with open(self.log, "wt") as log_out_fp:
timeout_extended = False
timeout_time = time.time() + self.timeout
timeout_time = time.time() + self.get_test_timeout()
while True:
this_timeout = timeout_time - time.time()
if this_timeout < 0:
@ -537,7 +541,8 @@ class DeviceHandler(Handler):
parity=serial.PARITY_NONE,
stopbits=serial.STOPBITS_ONE,
bytesize=serial.EIGHTBITS,
timeout=max(flash_timeout, self.timeout) # the worst case of no serial input
# the worst case of no serial input
timeout=max(flash_timeout, self.get_test_timeout())
)
except serial.SerialException as e:
self.instance.status = "failed"
@ -623,7 +628,7 @@ class DeviceHandler(Handler):
flash_timeout = hardware.flash_timeout
if hardware.flash_with_test:
flash_timeout += self.timeout
flash_timeout += self.get_test_timeout()
try:
ser = self._create_serial_connection(
@ -683,7 +688,7 @@ class DeviceHandler(Handler):
if not flash_error:
# Always wait at most the test timeout here after flashing.
t.join(self.timeout)
t.join(self.get_test_timeout())
else:
# When the flash error is due exceptions,
# twister tell the monitor serial thread
@ -966,7 +971,7 @@ class QEMUHandler(Handler):
self._set_qemu_filenames(sysbuild_build_dir)
self.thread = threading.Thread(name=self.name, target=QEMUHandler._thread,
args=(self, self.timeout, self.build_dir,
args=(self, self.get_test_timeout(), self.build_dir,
self.log_fn, self.fifo_fn,
self.pid_fn, self.results, harness,
self.ignore_unexpected_eof))
@ -986,7 +991,7 @@ class QEMUHandler(Handler):
logger.debug("Spawning QEMUHandler Thread for %s" % self.name)
try:
proc.wait(self.timeout)
proc.wait(self.get_test_timeout())
except subprocess.TimeoutExpired:
# sometimes QEMU can't handle SIGTERM signal correctly
# in that case kill -9 QEMU process directly and leave

7
scripts/tests/twister/test_handlers.py
View file

@ -420,6 +420,7 @@ def test_binaryhandler_output_handler(
handler = BinaryHandler(mocked_instance, 'build')
handler.terminate = mock.Mock()
handler.options = mock.Mock(timeout_multiplier=1)
proc = MockProc(1, proc_stdout)
@ -1208,6 +1209,7 @@ def test_devicehandler_create_serial_connection(
handler.instance.add_missing_case_status = missing_mock
available_mock = mock.Mock()
handler.make_device_available = available_mock
handler.options = mock.Mock(timeout_multiplier=1)
hardware_baud = 14400
flash_timeout = 60
@ -1375,6 +1377,7 @@ def test_devicehandler_handle(
handler = DeviceHandler(mocked_instance, 'build')
handler.get_hardware = mock.Mock(return_value=hardware)
handler.options = mock.Mock(
timeout_multiplier=1,
west_flash=None,
west_runner=None
)
@ -1906,6 +1909,7 @@ def test_qemuhandler_thread(
handler.ignore_unexpected_eof = False
handler.pid_fn = 'pid_fn'
handler.fifo_fn = 'fifo_fn'
handler.options = mock.Mock(timeout_multiplier=1)
def mocked_open(filename, *args, **kwargs):
if filename == handler.pid_fn:
@ -1956,7 +1960,7 @@ def test_qemuhandler_thread(
mock_thread_update_instance_info):
QEMUHandler._thread(
handler,
handler.timeout,
handler.get_test_timeout(),
handler.build_dir,
handler.log,
handler.fifo_fn,
@ -2035,6 +2039,7 @@ def test_qemuhandler_handle(
command = ['generator_cmd', '-C', os.path.join('cmd', 'path'), 'run']
handler.options = mock.Mock(
timeout_multiplier=1,
west_flash=handler_options_west_flash,
west_runner=None
)