twister: footprint: Improve error handling at gather_metrics
Add error handling for 'gather_metrics' stage and separate build issues from 'scripts/footprint/size_report' issues. Signed-off-by: Dmitrii Golovanov <dmitrii.golovanov@intel.com>
This commit is contained in:
Dmitrii Golovanov
Carles Cufí
parent
02f4e71633
commit
50db868b58
|
|
@ -268,7 +268,7 @@ class CMake:
|
|||
p = self.jobserver.popen(cmd, **kwargs)
|
||||
else:
|
||||
p = subprocess.Popen(cmd, **kwargs)
|
||||
logger.debug(f'Running {"".join(cmd)}')
|
||||
logger.debug(f'Running {" ".join(cmd)}')
|
||||
|
||||
out, _ = p.communicate()
|
||||
|
||||
|
|
@ -662,8 +662,12 @@ class ProjectBuilder(FilterBuilder):
|
|||
pipeline.put({"op": "report", "test": self.instance})
|
||||
|
||||
elif op == "gather_metrics":
|
||||
self.gather_metrics(self.instance)
|
||||
if self.instance.run and self.instance.handler.ready:
|
||||
ret = self.gather_metrics(self.instance)
|
||||
if not ret or ret.get('returncode', 1) > 0:
|
||||
self.instance.status = "error"
|
||||
self.instance.reason = "Build Failure at gather_metrics."
|
||||
pipeline.put({"op": "report", "test": self.instance})
|
||||
elif self.instance.run and self.instance.handler.ready:
|
||||
pipeline.put({"op": "run", "test": self.instance})
|
||||
else:
|
||||
pipeline.put({"op": "report", "test": self.instance})
|
||||
|
|
@ -1133,8 +1137,9 @@ class ProjectBuilder(FilterBuilder):
|
|||
sys.stdout.flush()
|
||||
|
||||
def gather_metrics(self, instance: TestInstance):
|
||||
build_result = {"returncode": 0}
|
||||
if self.options.create_rom_ram_report:
|
||||
self.run_build(['--build', self.build_dir, "--target", "footprint"])
|
||||
build_result = self.run_build(['--build', self.build_dir, "--target", "footprint"])
|
||||
if self.options.enable_size_report and not self.options.cmake_only:
|
||||
self.calc_size(instance=instance, from_buildlog=self.options.footprint_from_buildlog)
|
||||
else:
|
||||
|
|
@ -1143,6 +1148,7 @@ class ProjectBuilder(FilterBuilder):
|
|||
instance.metrics["available_rom"] = 0
|
||||
instance.metrics["available_ram"] = 0
|
||||
instance.metrics["unrecognized"] = []
|
||||
return build_result
|
||||
|
||||
@staticmethod
|
||||
def calc_size(instance: TestInstance, from_buildlog: bool):
|
||||
|
|
|
|||
|
|
@ -884,6 +884,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
{'op': 'report', 'test': mock.ANY},
|
||||
'failed',
|
||||
|
|
@ -905,6 +906,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
['filtering dummy instance name'],
|
||||
{'op': 'report', 'test': mock.ANY},
|
||||
'filtered',
|
||||
|
|
@ -926,6 +928,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
{'op': 'cmake', 'test': mock.ANY},
|
||||
'passed',
|
||||
|
|
@ -947,6 +950,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
{'op': 'report', 'test': mock.ANY},
|
||||
'error',
|
||||
|
|
@ -968,6 +972,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
{'op': 'report', 'test': mock.ANY},
|
||||
'passed',
|
||||
|
|
@ -989,6 +994,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
{'op': 'report', 'test': mock.ANY},
|
||||
'success',
|
||||
|
|
@ -1010,6 +1016,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
['filtering dummy instance name'],
|
||||
{'op': 'report', 'test': mock.ANY},
|
||||
'filtered',
|
||||
|
|
@ -1031,6 +1038,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
{'op': 'build', 'test': mock.ANY},
|
||||
'success',
|
||||
|
|
@ -1052,6 +1060,7 @@ TESTDATA_6 = [
|
|||
None,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
['build test: dummy instance name'],
|
||||
{'op': 'report', 'test': mock.ANY},
|
||||
'error',
|
||||
|
|
@ -1073,6 +1082,7 @@ TESTDATA_6 = [
|
|||
{'returncode': 0},
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
['build test: dummy instance name',
|
||||
'Determine test cases for test instance: dummy instance name'],
|
||||
{'op': 'gather_metrics', 'test': mock.ANY},
|
||||
|
|
@ -1095,6 +1105,7 @@ TESTDATA_6 = [
|
|||
{'dummy': 'dummy'},
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
['build test: dummy instance name'],
|
||||
{'op': 'report', 'test': mock.ANY},
|
||||
'passed',
|
||||
|
|
@ -1116,6 +1127,7 @@ TESTDATA_6 = [
|
|||
{'returncode': 0},
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
['build test: dummy instance name',
|
||||
'Determine test cases for test instance: dummy instance name'],
|
||||
{'op': 'gather_metrics', 'test': mock.ANY},
|
||||
|
|
@ -1137,6 +1149,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
{'returncode': 0},
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
BuildError,
|
||||
['build test: dummy instance name',
|
||||
'Determine test cases for test instance: dummy instance name'],
|
||||
|
|
@ -1158,6 +1171,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
{'returncode': 0}, # metrics_res
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
|
|
@ -1166,7 +1180,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
0,
|
||||
None
|
||||
),
|
||||
), # 'gather metrics, run and ready handler'
|
||||
(
|
||||
{'op': 'gather_metrics'},
|
||||
mock.ANY,
|
||||
|
|
@ -1179,6 +1193,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
{'returncode': 0}, # metrics_res
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
|
|
@ -1187,7 +1202,29 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
0,
|
||||
None
|
||||
),
|
||||
), # 'gather metrics'
|
||||
(
|
||||
{'op': 'gather_metrics'},
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
False,
|
||||
True,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
{'returncode': 0}, # build_res
|
||||
{'returncode': 1}, # metrics_res
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
{'op': 'report', 'test': mock.ANY},
|
||||
'error',
|
||||
'Build Failure at gather_metrics.',
|
||||
0,
|
||||
None
|
||||
), # 'build ok, gather metrics fail',
|
||||
(
|
||||
{'op': 'run'},
|
||||
'success',
|
||||
|
|
@ -1200,6 +1237,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
None,
|
||||
mock.ANY,
|
||||
['run test: dummy instance name',
|
||||
|
|
@ -1222,6 +1260,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
RuntimeError,
|
||||
mock.ANY,
|
||||
['run test: dummy instance name',
|
||||
|
|
@ -1247,6 +1286,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
{'op': 'cleanup', 'mode': 'device', 'test': mock.ANY},
|
||||
mock.ANY,
|
||||
|
|
@ -1268,6 +1308,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
{'op': 'cleanup', 'mode': 'passed', 'test': mock.ANY},
|
||||
mock.ANY,
|
||||
|
|
@ -1289,6 +1330,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
{'op': 'cleanup', 'mode': 'all', 'test': mock.ANY},
|
||||
mock.ANY,
|
||||
|
|
@ -1310,6 +1352,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
None,
|
||||
mock.ANY,
|
||||
|
|
@ -1331,6 +1374,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
None,
|
||||
mock.ANY,
|
||||
|
|
@ -1352,6 +1396,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
None,
|
||||
mock.ANY,
|
||||
|
|
@ -1373,6 +1418,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
None,
|
||||
mock.ANY,
|
||||
|
|
@ -1394,6 +1440,7 @@ TESTDATA_6 = [
|
|||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
mock.ANY,
|
||||
[],
|
||||
None,
|
||||
mock.ANY,
|
||||
|
|
@ -1408,7 +1455,7 @@ TESTDATA_6 = [
|
|||
' instance_status, instance_reason, instance_run, instance_handler_ready,' \
|
||||
' options_cmake_only,' \
|
||||
' options_coverage, options_prep_artifacts, options_runtime_artifacts,' \
|
||||
' cmake_res, build_res,' \
|
||||
' cmake_res, build_res, metrics_res,' \
|
||||
' pipeline_runtime_error, determine_testcases_build_error,' \
|
||||
' expected_logs, resulting_message,' \
|
||||
' expected_status, expected_reason, expected_skipped, expected_missing',
|
||||
|
|
@ -1420,6 +1467,7 @@ TESTDATA_6 = [
|
|||
'build, no build res', 'build, skipped', 'build, blocked',
|
||||
'build, determine testcases', 'build, determine testcases Error',
|
||||
'gather metrics, run and ready handler', 'gather metrics',
|
||||
'build ok, gather metrics fail',
|
||||
'run', 'run, Pipeline Runtime Error',
|
||||
'report, prep artifacts for testing',
|
||||
'report, runtime artifact cleanup pass, status passed',
|
||||
|
|
@ -1442,6 +1490,7 @@ def test_projectbuilder_process(
|
|||
options_runtime_artifacts,
|
||||
cmake_res,
|
||||
build_res,
|
||||
metrics_res,
|
||||
pipeline_runtime_error,
|
||||
determine_testcases_build_error,
|
||||
expected_logs,
|
||||
|
|
@ -1485,7 +1534,7 @@ def test_projectbuilder_process(
|
|||
pb.cleanup_artifacts = mock.Mock()
|
||||
pb.cleanup_device_testing_artifacts = mock.Mock()
|
||||
pb.run = mock.Mock()
|
||||
pb.gather_metrics = mock.Mock()
|
||||
pb.gather_metrics = mock.Mock(return_value=metrics_res)
|
||||
|
||||
pipeline_mock = mock.Mock(put=mock.Mock(side_effect=mock_pipeline_put))
|
||||
done_mock = mock.Mock()
|
||||
|
|
|
|||
Loading…
Reference in a new issue