twister: rework testsuite parsing
Move testsuite setup and parsing to the testsuite class. Simplify reading data from yaml. Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
parent
4789f07a8f
commit
1463f4bdc5
|
@ -10,6 +10,35 @@ class TwisterConfigParser:
|
|||
"""Class to read testsuite yaml files with semantic checking
|
||||
"""
|
||||
|
||||
testsuite_valid_keys = {"tags": {"type": "set", "required": False},
|
||||
"type": {"type": "str", "default": "integration"},
|
||||
"extra_args": {"type": "list"},
|
||||
"extra_configs": {"type": "list"},
|
||||
"build_only": {"type": "bool", "default": False},
|
||||
"build_on_all": {"type": "bool", "default": False},
|
||||
"skip": {"type": "bool", "default": False},
|
||||
"slow": {"type": "bool", "default": False},
|
||||
"timeout": {"type": "int", "default": 60},
|
||||
"min_ram": {"type": "int", "default": 8},
|
||||
"modules": {"type": "list", "default": []},
|
||||
"depends_on": {"type": "set"},
|
||||
"min_flash": {"type": "int", "default": 32},
|
||||
"arch_allow": {"type": "set"},
|
||||
"arch_exclude": {"type": "set"},
|
||||
"extra_sections": {"type": "list", "default": []},
|
||||
"integration_platforms": {"type": "list", "default": []},
|
||||
"testcases": {"type": "list", "default": []},
|
||||
"platform_type": {"type": "list", "default": []},
|
||||
"platform_exclude": {"type": "set"},
|
||||
"platform_allow": {"type": "set"},
|
||||
"toolchain_exclude": {"type": "set"},
|
||||
"toolchain_allow": {"type": "set"},
|
||||
"filter": {"type": "str"},
|
||||
"harness": {"type": "str", "default": "test"},
|
||||
"harness_config": {"type": "map", "default": {}},
|
||||
"seed": {"type": "int", "default": 0}
|
||||
}
|
||||
|
||||
def __init__(self, filename, schema):
|
||||
"""Instantiate a new TwisterConfigParser object
|
||||
|
||||
|
@ -66,27 +95,10 @@ class TwisterConfigParser:
|
|||
raise ConfigurationError(
|
||||
self.filename, "unknown type '%s'" % value)
|
||||
|
||||
def get_scenario(self, name, valid_keys):
|
||||
def get_scenario(self, name):
|
||||
"""Get a dictionary representing the keys/values within a scenario
|
||||
|
||||
@param name The scenario in the .yaml file to retrieve data from
|
||||
@param valid_keys A dictionary representing the intended semantics
|
||||
for this scenario. Each key in this dictionary is a key that could
|
||||
be specified, if a key is given in the .yaml file which isn't in
|
||||
here, it will generate an error. Each value in this dictionary
|
||||
is another dictionary containing metadata:
|
||||
|
||||
"default" - Default value if not given
|
||||
"type" - Data type to convert the text value to. Simple types
|
||||
supported are "str", "float", "int", "bool" which will get
|
||||
converted to respective Python data types. "set" and "list"
|
||||
may also be specified which will split the value by
|
||||
whitespace (but keep the elements as strings). finally,
|
||||
"list:<type>" and "set:<type>" may be given which will
|
||||
perform a type conversion after splitting the value up.
|
||||
"required" - If true, raise an error if not defined. If false
|
||||
and "default" isn't specified, a type conversion will be
|
||||
done on an empty string
|
||||
@return A dictionary containing the scenario key-value pairs with
|
||||
type conversion and default values filled in per valid_keys
|
||||
"""
|
||||
|
@ -109,7 +121,7 @@ class TwisterConfigParser:
|
|||
else:
|
||||
d[k] = v
|
||||
|
||||
for k, kinfo in valid_keys.items():
|
||||
for k, kinfo in self.testsuite_valid_keys.items():
|
||||
if k not in d:
|
||||
if "required" in kinfo:
|
||||
required = kinfo["required"]
|
||||
|
|
|
@ -43,7 +43,7 @@ logger.setLevel(logging.DEBUG)
|
|||
class HarnessImporter:
|
||||
|
||||
def __init__(self, name):
|
||||
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
|
||||
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister/twisterlib"))
|
||||
module = __import__("harness")
|
||||
if name:
|
||||
my_class = getattr(module, name)
|
||||
|
|
|
@ -350,14 +350,14 @@ class FilterBuilder(CMake):
|
|||
filter_data.update(self.cmake_cache)
|
||||
|
||||
edt_pickle = os.path.join(self.build_dir, "zephyr", "edt.pickle")
|
||||
if self.testsuite and self.testsuite.ts_filter:
|
||||
if self.testsuite and self.testsuite.filter:
|
||||
try:
|
||||
if os.path.exists(edt_pickle):
|
||||
with open(edt_pickle, 'rb') as f:
|
||||
edt = pickle.load(f)
|
||||
else:
|
||||
edt = None
|
||||
res = expr_parser.parse(self.testsuite.ts_filter, filter_data, edt)
|
||||
res = expr_parser.parse(self.testsuite.filter, filter_data, edt)
|
||||
|
||||
except (ValueError, SyntaxError) as se:
|
||||
sys.stderr.write(
|
||||
|
|
|
@ -58,42 +58,13 @@ class TestPlan:
|
|||
config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
|
||||
dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
|
||||
|
||||
ts_schema = scl.yaml_load(
|
||||
suite_schema = scl.yaml_load(
|
||||
os.path.join(ZEPHYR_BASE,
|
||||
"scripts", "schemas", "twister", "testsuite-schema.yaml"))
|
||||
quarantine_schema = scl.yaml_load(
|
||||
os.path.join(ZEPHYR_BASE,
|
||||
"scripts", "schemas", "twister", "quarantine-schema.yaml"))
|
||||
|
||||
testsuite_valid_keys = {"tags": {"type": "set", "required": False},
|
||||
"type": {"type": "str", "default": "integration"},
|
||||
"extra_args": {"type": "list"},
|
||||
"extra_configs": {"type": "list"},
|
||||
"build_only": {"type": "bool", "default": False},
|
||||
"build_on_all": {"type": "bool", "default": False},
|
||||
"skip": {"type": "bool", "default": False},
|
||||
"slow": {"type": "bool", "default": False},
|
||||
"timeout": {"type": "int", "default": 60},
|
||||
"min_ram": {"type": "int", "default": 8},
|
||||
"modules": {"type": "list", "default": []},
|
||||
"depends_on": {"type": "set"},
|
||||
"min_flash": {"type": "int", "default": 32},
|
||||
"arch_allow": {"type": "set"},
|
||||
"arch_exclude": {"type": "set"},
|
||||
"extra_sections": {"type": "list", "default": []},
|
||||
"integration_platforms": {"type": "list", "default": []},
|
||||
"testcases": {"type": "list", "default": []},
|
||||
"platform_type": {"type": "list", "default": []},
|
||||
"platform_exclude": {"type": "set"},
|
||||
"platform_allow": {"type": "set"},
|
||||
"toolchain_exclude": {"type": "set"},
|
||||
"toolchain_allow": {"type": "set"},
|
||||
"filter": {"type": "str"},
|
||||
"harness": {"type": "str", "default": "test"},
|
||||
"harness_config": {"type": "map", "default": {}},
|
||||
"seed": {"type": "int", "default": 0}
|
||||
}
|
||||
|
||||
SAMPLE_FILENAME = 'sample.yaml'
|
||||
TESTSUITE_FILENAME = 'testcase.yaml'
|
||||
|
||||
|
@ -424,79 +395,30 @@ class TestPlan:
|
|||
else:
|
||||
continue
|
||||
|
||||
logger.debug("Found possible test case in " + dirpath)
|
||||
logger.debug("Found possible testsuite in " + dirpath)
|
||||
|
||||
ts_path = os.path.join(dirpath, filename)
|
||||
suite_yaml_path = os.path.join(dirpath, filename)
|
||||
|
||||
try:
|
||||
parsed_data = TwisterConfigParser(ts_path, self.ts_schema)
|
||||
parsed_data = TwisterConfigParser(suite_yaml_path, self.suite_schema)
|
||||
parsed_data.load()
|
||||
|
||||
ts_path = os.path.dirname(ts_path)
|
||||
workdir = os.path.relpath(ts_path, root)
|
||||
suite_path = os.path.dirname(suite_yaml_path)
|
||||
|
||||
subcases, ztest_suite_names = scan_testsuite_path(ts_path)
|
||||
subcases, ztest_suite_names = scan_testsuite_path(suite_path)
|
||||
|
||||
for name in parsed_data.scenarios.keys():
|
||||
ts = TestSuite(root, workdir, name)
|
||||
|
||||
ts_dict = parsed_data.get_scenario(name, self.testsuite_valid_keys)
|
||||
|
||||
ts.source_dir = ts_path
|
||||
ts.yamlfile = ts_path
|
||||
|
||||
ts.type = ts_dict["type"]
|
||||
ts.tags = ts_dict["tags"]
|
||||
ts.extra_args = ts_dict["extra_args"]
|
||||
ts.extra_configs = ts_dict["extra_configs"]
|
||||
ts.arch_allow = ts_dict["arch_allow"]
|
||||
ts.arch_exclude = ts_dict["arch_exclude"]
|
||||
ts.skip = ts_dict["skip"]
|
||||
ts.platform_exclude = ts_dict["platform_exclude"]
|
||||
ts.platform_allow = ts_dict["platform_allow"]
|
||||
ts.platform_type = ts_dict["platform_type"]
|
||||
ts.toolchain_exclude = ts_dict["toolchain_exclude"]
|
||||
ts.toolchain_allow = ts_dict["toolchain_allow"]
|
||||
ts.ts_filter = ts_dict["filter"]
|
||||
ts.timeout = ts_dict["timeout"]
|
||||
ts.harness = ts_dict["harness"]
|
||||
ts.harness_config = ts_dict["harness_config"]
|
||||
if ts.harness == 'console' and not ts.harness_config:
|
||||
raise Exception('Harness config error: console harness defined without a configuration.')
|
||||
ts.build_only = ts_dict["build_only"]
|
||||
ts.build_on_all = ts_dict["build_on_all"]
|
||||
ts.slow = ts_dict["slow"]
|
||||
ts.min_ram = ts_dict["min_ram"]
|
||||
ts.modules = ts_dict["modules"]
|
||||
ts.depends_on = ts_dict["depends_on"]
|
||||
ts.min_flash = ts_dict["min_flash"]
|
||||
ts.extra_sections = ts_dict["extra_sections"]
|
||||
ts.integration_platforms = ts_dict["integration_platforms"]
|
||||
ts.seed = ts_dict["seed"]
|
||||
|
||||
testcases = ts_dict.get("testcases", [])
|
||||
if testcases:
|
||||
for tc in testcases:
|
||||
ts.add_testcase(name=f"{name}.{tc}")
|
||||
else:
|
||||
# only add each testcase once
|
||||
for sub in set(subcases):
|
||||
name = "{}.{}".format(ts.id, sub)
|
||||
ts.add_testcase(name)
|
||||
|
||||
if not subcases:
|
||||
ts.add_testcase(ts.id, freeform=True)
|
||||
|
||||
ts.ztest_suite_names = ztest_suite_names
|
||||
|
||||
suite_dict = parsed_data.get_scenario(name)
|
||||
suite = TestSuite(root, suite_path, name, data=suite_dict)
|
||||
suite.add_subcases(suite_dict, subcases, ztest_suite_names)
|
||||
if testsuite_filter:
|
||||
if ts.name and ts.name in testsuite_filter:
|
||||
self.testsuites[ts.name] = ts
|
||||
if suite.name and suite.name in testsuite_filter:
|
||||
self.testsuites[suite.name] = suite
|
||||
else:
|
||||
self.testsuites[ts.name] = ts
|
||||
self.testsuites[suite.name] = suite
|
||||
|
||||
except Exception as e:
|
||||
logger.error("%s: can't load (skipping): %s" % (ts_path, e))
|
||||
logger.error("%s: can't load (skipping): %s" % (suite_path, e))
|
||||
self.load_errors += 1
|
||||
return len(self.testsuites)
|
||||
|
||||
|
|
|
@ -340,7 +340,7 @@ class TestSuite(DisablePyTestCollectionMixin):
|
|||
"""Class representing a test application
|
||||
"""
|
||||
|
||||
def __init__(self, testsuite_root, workdir, name):
|
||||
def __init__(self, suite_root, suite_path, name, data=None):
|
||||
"""TestSuite constructor.
|
||||
|
||||
This gets called by TestPlan as it finds and reads test yaml files.
|
||||
|
@ -352,8 +352,7 @@ class TestSuite(DisablePyTestCollectionMixin):
|
|||
the test case is <workdir>/<name>.
|
||||
|
||||
@param testsuite_root os.path.abspath() of one of the --testsuite-root
|
||||
@param workdir Sub-directory of testsuite_root where the
|
||||
.yaml test configuration file was found
|
||||
@param suite_path path to testsuite
|
||||
@param name Name of this test case, corresponding to the entry name
|
||||
in the test case configuration file. For many test cases that just
|
||||
define one test, can be anything and is usually "test". This is
|
||||
|
@ -361,39 +360,43 @@ class TestSuite(DisablePyTestCollectionMixin):
|
|||
the testcase.yaml defines multiple tests
|
||||
"""
|
||||
|
||||
|
||||
self.source_dir = ""
|
||||
self.yamlfile = ""
|
||||
self.testcases = []
|
||||
self.name = self.get_unique(testsuite_root, workdir, name)
|
||||
workdir = os.path.relpath(suite_path, suite_root)
|
||||
self.name = self.get_unique(os.path.dirname(suite_path), workdir, name)
|
||||
self.id = name
|
||||
|
||||
self.type = None
|
||||
self.tags = set()
|
||||
self.extra_args = None
|
||||
self.extra_configs = None
|
||||
self.arch_allow = None
|
||||
self.arch_exclude = None
|
||||
self.skip = False
|
||||
self.platform_exclude = None
|
||||
self.platform_allow = None
|
||||
self.platform_type = []
|
||||
self.toolchain_exclude = None
|
||||
self.toolchain_allow = None
|
||||
self.ts_filter = None
|
||||
self.timeout = 60
|
||||
self.harness = ""
|
||||
self.harness_config = {}
|
||||
self.build_only = True
|
||||
self.build_on_all = False
|
||||
self.slow = False
|
||||
self.min_ram = -1
|
||||
self.depends_on = None
|
||||
self.min_flash = -1
|
||||
self.extra_sections = None
|
||||
self.integration_platforms = []
|
||||
self.source_dir = suite_path
|
||||
self.yamlfile = suite_path
|
||||
self.testcases = []
|
||||
|
||||
self.ztest_suite_names = []
|
||||
|
||||
if data:
|
||||
self.load(data)
|
||||
|
||||
|
||||
def load(self, data):
|
||||
for k, v in data.items():
|
||||
if k != "testcases":
|
||||
setattr(self, k, v)
|
||||
|
||||
if self.harness == 'console' and not self.harness_config:
|
||||
raise Exception('Harness config error: console harness defined without a configuration.')
|
||||
|
||||
def add_subcases(self, data, parsed_subcases, suite_names):
|
||||
testcases = data.get("testcases", [])
|
||||
if testcases:
|
||||
for tc in testcases:
|
||||
self.add_testcase(name=f"{self.id}.{tc}")
|
||||
else:
|
||||
# only add each testcase once
|
||||
for sub in set(parsed_subcases):
|
||||
name = "{}.{}".format(self.id, sub)
|
||||
self.add_testcase(name)
|
||||
|
||||
if not parsed_subcases:
|
||||
self.add_testcase(self.id, freeform=True)
|
||||
|
||||
self.ztest_suite_names = suite_names
|
||||
|
||||
def add_testcase(self, name, freeform=False):
|
||||
tc = TestCase(name=name, testsuite=self)
|
||||
|
|
Loading…
Reference in a new issue