twister: rework testsuite parsing

Move testsuite setup and parsing to the testsuite class. Simplify
reading data from yaml.

Signed-off-by: Anas Nashif <anas.nashif@intel.com>
This commit is contained in:
Anas Nashif 2022-06-28 09:58:09 -04:00
parent 4789f07a8f
commit 1463f4bdc5
5 changed files with 82 additions and 145 deletions

View file

@ -10,6 +10,35 @@ class TwisterConfigParser:
"""Class to read testsuite yaml files with semantic checking
"""
testsuite_valid_keys = {"tags": {"type": "set", "required": False},
"type": {"type": "str", "default": "integration"},
"extra_args": {"type": "list"},
"extra_configs": {"type": "list"},
"build_only": {"type": "bool", "default": False},
"build_on_all": {"type": "bool", "default": False},
"skip": {"type": "bool", "default": False},
"slow": {"type": "bool", "default": False},
"timeout": {"type": "int", "default": 60},
"min_ram": {"type": "int", "default": 8},
"modules": {"type": "list", "default": []},
"depends_on": {"type": "set"},
"min_flash": {"type": "int", "default": 32},
"arch_allow": {"type": "set"},
"arch_exclude": {"type": "set"},
"extra_sections": {"type": "list", "default": []},
"integration_platforms": {"type": "list", "default": []},
"testcases": {"type": "list", "default": []},
"platform_type": {"type": "list", "default": []},
"platform_exclude": {"type": "set"},
"platform_allow": {"type": "set"},
"toolchain_exclude": {"type": "set"},
"toolchain_allow": {"type": "set"},
"filter": {"type": "str"},
"harness": {"type": "str", "default": "test"},
"harness_config": {"type": "map", "default": {}},
"seed": {"type": "int", "default": 0}
}
def __init__(self, filename, schema):
"""Instantiate a new TwisterConfigParser object
@ -66,27 +95,10 @@ class TwisterConfigParser:
raise ConfigurationError(
self.filename, "unknown type '%s'" % value)
def get_scenario(self, name, valid_keys):
def get_scenario(self, name):
"""Get a dictionary representing the keys/values within a scenario
@param name The scenario in the .yaml file to retrieve data from
@param valid_keys A dictionary representing the intended semantics
for this scenario. Each key in this dictionary is a key that could
be specified, if a key is given in the .yaml file which isn't in
here, it will generate an error. Each value in this dictionary
is another dictionary containing metadata:
"default" - Default value if not given
"type" - Data type to convert the text value to. Simple types
supported are "str", "float", "int", "bool" which will get
converted to respective Python data types. "set" and "list"
may also be specified which will split the value by
whitespace (but keep the elements as strings). finally,
"list:<type>" and "set:<type>" may be given which will
perform a type conversion after splitting the value up.
"required" - If true, raise an error if not defined. If false
and "default" isn't specified, a type conversion will be
done on an empty string
@return A dictionary containing the scenario key-value pairs with
type conversion and default values filled in per valid_keys
"""
@ -109,7 +121,7 @@ class TwisterConfigParser:
else:
d[k] = v
for k, kinfo in valid_keys.items():
for k, kinfo in self.testsuite_valid_keys.items():
if k not in d:
if "required" in kinfo:
required = kinfo["required"]

View file

@ -43,7 +43,7 @@ logger.setLevel(logging.DEBUG)
class HarnessImporter:
def __init__(self, name):
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister"))
sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts/pylib/twister/twisterlib"))
module = __import__("harness")
if name:
my_class = getattr(module, name)

View file

@ -350,14 +350,14 @@ class FilterBuilder(CMake):
filter_data.update(self.cmake_cache)
edt_pickle = os.path.join(self.build_dir, "zephyr", "edt.pickle")
if self.testsuite and self.testsuite.ts_filter:
if self.testsuite and self.testsuite.filter:
try:
if os.path.exists(edt_pickle):
with open(edt_pickle, 'rb') as f:
edt = pickle.load(f)
else:
edt = None
res = expr_parser.parse(self.testsuite.ts_filter, filter_data, edt)
res = expr_parser.parse(self.testsuite.filter, filter_data, edt)
except (ValueError, SyntaxError) as se:
sys.stderr.write(

View file

@ -58,42 +58,13 @@ class TestPlan:
config_re = re.compile('(CONFIG_[A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
dt_re = re.compile('([A-Za-z0-9_]+)[=]\"?([^\"]*)\"?$')
ts_schema = scl.yaml_load(
suite_schema = scl.yaml_load(
os.path.join(ZEPHYR_BASE,
"scripts", "schemas", "twister", "testsuite-schema.yaml"))
quarantine_schema = scl.yaml_load(
os.path.join(ZEPHYR_BASE,
"scripts", "schemas", "twister", "quarantine-schema.yaml"))
testsuite_valid_keys = {"tags": {"type": "set", "required": False},
"type": {"type": "str", "default": "integration"},
"extra_args": {"type": "list"},
"extra_configs": {"type": "list"},
"build_only": {"type": "bool", "default": False},
"build_on_all": {"type": "bool", "default": False},
"skip": {"type": "bool", "default": False},
"slow": {"type": "bool", "default": False},
"timeout": {"type": "int", "default": 60},
"min_ram": {"type": "int", "default": 8},
"modules": {"type": "list", "default": []},
"depends_on": {"type": "set"},
"min_flash": {"type": "int", "default": 32},
"arch_allow": {"type": "set"},
"arch_exclude": {"type": "set"},
"extra_sections": {"type": "list", "default": []},
"integration_platforms": {"type": "list", "default": []},
"testcases": {"type": "list", "default": []},
"platform_type": {"type": "list", "default": []},
"platform_exclude": {"type": "set"},
"platform_allow": {"type": "set"},
"toolchain_exclude": {"type": "set"},
"toolchain_allow": {"type": "set"},
"filter": {"type": "str"},
"harness": {"type": "str", "default": "test"},
"harness_config": {"type": "map", "default": {}},
"seed": {"type": "int", "default": 0}
}
SAMPLE_FILENAME = 'sample.yaml'
TESTSUITE_FILENAME = 'testcase.yaml'
@ -424,79 +395,30 @@ class TestPlan:
else:
continue
logger.debug("Found possible test case in " + dirpath)
logger.debug("Found possible testsuite in " + dirpath)
ts_path = os.path.join(dirpath, filename)
suite_yaml_path = os.path.join(dirpath, filename)
try:
parsed_data = TwisterConfigParser(ts_path, self.ts_schema)
parsed_data = TwisterConfigParser(suite_yaml_path, self.suite_schema)
parsed_data.load()
ts_path = os.path.dirname(ts_path)
workdir = os.path.relpath(ts_path, root)
suite_path = os.path.dirname(suite_yaml_path)
subcases, ztest_suite_names = scan_testsuite_path(ts_path)
subcases, ztest_suite_names = scan_testsuite_path(suite_path)
for name in parsed_data.scenarios.keys():
ts = TestSuite(root, workdir, name)
ts_dict = parsed_data.get_scenario(name, self.testsuite_valid_keys)
ts.source_dir = ts_path
ts.yamlfile = ts_path
ts.type = ts_dict["type"]
ts.tags = ts_dict["tags"]
ts.extra_args = ts_dict["extra_args"]
ts.extra_configs = ts_dict["extra_configs"]
ts.arch_allow = ts_dict["arch_allow"]
ts.arch_exclude = ts_dict["arch_exclude"]
ts.skip = ts_dict["skip"]
ts.platform_exclude = ts_dict["platform_exclude"]
ts.platform_allow = ts_dict["platform_allow"]
ts.platform_type = ts_dict["platform_type"]
ts.toolchain_exclude = ts_dict["toolchain_exclude"]
ts.toolchain_allow = ts_dict["toolchain_allow"]
ts.ts_filter = ts_dict["filter"]
ts.timeout = ts_dict["timeout"]
ts.harness = ts_dict["harness"]
ts.harness_config = ts_dict["harness_config"]
if ts.harness == 'console' and not ts.harness_config:
raise Exception('Harness config error: console harness defined without a configuration.')
ts.build_only = ts_dict["build_only"]
ts.build_on_all = ts_dict["build_on_all"]
ts.slow = ts_dict["slow"]
ts.min_ram = ts_dict["min_ram"]
ts.modules = ts_dict["modules"]
ts.depends_on = ts_dict["depends_on"]
ts.min_flash = ts_dict["min_flash"]
ts.extra_sections = ts_dict["extra_sections"]
ts.integration_platforms = ts_dict["integration_platforms"]
ts.seed = ts_dict["seed"]
testcases = ts_dict.get("testcases", [])
if testcases:
for tc in testcases:
ts.add_testcase(name=f"{name}.{tc}")
else:
# only add each testcase once
for sub in set(subcases):
name = "{}.{}".format(ts.id, sub)
ts.add_testcase(name)
if not subcases:
ts.add_testcase(ts.id, freeform=True)
ts.ztest_suite_names = ztest_suite_names
suite_dict = parsed_data.get_scenario(name)
suite = TestSuite(root, suite_path, name, data=suite_dict)
suite.add_subcases(suite_dict, subcases, ztest_suite_names)
if testsuite_filter:
if ts.name and ts.name in testsuite_filter:
self.testsuites[ts.name] = ts
if suite.name and suite.name in testsuite_filter:
self.testsuites[suite.name] = suite
else:
self.testsuites[ts.name] = ts
self.testsuites[suite.name] = suite
except Exception as e:
logger.error("%s: can't load (skipping): %s" % (ts_path, e))
logger.error("%s: can't load (skipping): %s" % (suite_path, e))
self.load_errors += 1
return len(self.testsuites)

View file

@ -340,7 +340,7 @@ class TestSuite(DisablePyTestCollectionMixin):
"""Class representing a test application
"""
def __init__(self, testsuite_root, workdir, name):
def __init__(self, suite_root, suite_path, name, data=None):
"""TestSuite constructor.
This gets called by TestPlan as it finds and reads test yaml files.
@ -352,8 +352,7 @@ class TestSuite(DisablePyTestCollectionMixin):
the test case is <workdir>/<name>.
@param testsuite_root os.path.abspath() of one of the --testsuite-root
@param workdir Sub-directory of testsuite_root where the
.yaml test configuration file was found
@param suite_path path to testsuite
@param name Name of this test case, corresponding to the entry name
in the test case configuration file. For many test cases that just
define one test, can be anything and is usually "test". This is
@ -361,39 +360,43 @@ class TestSuite(DisablePyTestCollectionMixin):
the testcase.yaml defines multiple tests
"""
self.source_dir = ""
self.yamlfile = ""
self.testcases = []
self.name = self.get_unique(testsuite_root, workdir, name)
workdir = os.path.relpath(suite_path, suite_root)
self.name = self.get_unique(os.path.dirname(suite_path), workdir, name)
self.id = name
self.type = None
self.tags = set()
self.extra_args = None
self.extra_configs = None
self.arch_allow = None
self.arch_exclude = None
self.skip = False
self.platform_exclude = None
self.platform_allow = None
self.platform_type = []
self.toolchain_exclude = None
self.toolchain_allow = None
self.ts_filter = None
self.timeout = 60
self.harness = ""
self.harness_config = {}
self.build_only = True
self.build_on_all = False
self.slow = False
self.min_ram = -1
self.depends_on = None
self.min_flash = -1
self.extra_sections = None
self.integration_platforms = []
self.source_dir = suite_path
self.yamlfile = suite_path
self.testcases = []
self.ztest_suite_names = []
if data:
self.load(data)
def load(self, data):
for k, v in data.items():
if k != "testcases":
setattr(self, k, v)
if self.harness == 'console' and not self.harness_config:
raise Exception('Harness config error: console harness defined without a configuration.')
def add_subcases(self, data, parsed_subcases, suite_names):
testcases = data.get("testcases", [])
if testcases:
for tc in testcases:
self.add_testcase(name=f"{self.id}.{tc}")
else:
# only add each testcase once
for sub in set(parsed_subcases):
name = "{}.{}".format(self.id, sub)
self.add_testcase(name)
if not parsed_subcases:
self.add_testcase(self.id, freeform=True)
self.ztest_suite_names = suite_names
def add_testcase(self, name, freeform=False):
tc = TestCase(name=name, testsuite=self)