patrick/zephyr
1
0
Fork 0
Code Issues Pull requests Actions 7 Packages Projects Releases Wiki Activity

scripts: parse_syscalls: generalize struct tags

Browse source 
Now we can build up lists of data structures matching a list
of particular tags, with __subsystem being just one case.

Relax searches to also look inside C files, since struct
prototypes may be declared there as well.

Signed-off-by: Andrew Boie <andrew.p.boie@intel.com>
This commit is contained in:
Andrew Boie 2020-05-29 13:24:51 -07:00 committed by Carles Cufí
parent 455e178b3b
commit 5960119f16
3 changed files with 47 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
CMakeLists.txt
View file

@ -497,7 +497,7 @@ endif()
set(syscall_list_h ${CMAKE_CURRENT_BINARY_DIR}/include/generated/syscall_list.h) set(syscall_list_h ${CMAKE_CURRENT_BINARY_DIR}/include/generated/syscall_list.h)
set(syscalls_json ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/syscalls.json) set(syscalls_json ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/syscalls.json)
set(subsys_json ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/subsystems.json) set(struct_tags_json ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/struct_tags.json)
# The syscalls subdirs txt file is constructed by python containing a list of folders to use for # The syscalls subdirs txt file is constructed by python containing a list of folders to use for
# dependency handling, including empty folders. # dependency handling, including empty folders.
@ -589,20 +589,20 @@ endforeach()
add_custom_command( add_custom_command(
OUTPUT OUTPUT
${syscalls_json} ${syscalls_json}
${subsys_json} ${struct_tags_json}
COMMAND COMMAND
${PYTHON_EXECUTABLE} ${PYTHON_EXECUTABLE}
${ZEPHYR_BASE}/scripts/parse_syscalls.py ${ZEPHYR_BASE}/scripts/parse_syscalls.py
--include ${ZEPHYR_BASE}/include # Read files from this dir --include ${ZEPHYR_BASE}/include # Read files from this dir
${parse_syscalls_include_args} # Read files from these dirs also ${parse_syscalls_include_args} # Read files from these dirs also
--json-file ${syscalls_json} # Write this file --json-file ${syscalls_json} # Write this file
--subsystem-file ${subsys_json} # Write subsystem list to this file --tag-struct-file ${struct_tags_json} # Write subsystem list to this file
DEPENDS ${syscalls_subdirs_trigger} ${PARSE_SYSCALLS_HEADER_DEPENDS} DEPENDS ${syscalls_subdirs_trigger} ${PARSE_SYSCALLS_HEADER_DEPENDS}
) )
add_custom_target(${SYSCALL_LIST_H_TARGET} DEPENDS ${syscall_list_h}) add_custom_target(${SYSCALL_LIST_H_TARGET} DEPENDS ${syscall_list_h})
add_custom_target(${PARSE_SYSCALLS_TARGET} add_custom_target(${PARSE_SYSCALLS_TARGET}
DEPENDS ${syscalls_json} ${subsys_json}) DEPENDS ${syscalls_json} ${struct_tags_json})
# 64-bit systems do not require special handling of 64-bit system call # 64-bit systems do not require special handling of 64-bit system call
# parameters or return values, indicate this to the system call boilerplate # parameters or return values, indicate this to the system call boilerplate
@ -632,7 +632,7 @@ add_custom_command(OUTPUT include/generated/syscall_dispatch.c ${syscall_list_h}
) )
# This is passed into all calls to the gen_kobject_list.py script. # This is passed into all calls to the gen_kobject_list.py script.
set(gen_kobject_list_include_args --include ${subsys_json}) set(gen_kobject_list_include_args --include ${struct_tags_json})
set(DRV_VALIDATION ${PROJECT_BINARY_DIR}/include/generated/driver-validation.h) set(DRV_VALIDATION ${PROJECT_BINARY_DIR}/include/generated/driver-validation.h)
add_custom_command( add_custom_command(
@ -646,7 +646,7 @@ add_custom_command(
DEPENDS DEPENDS
${ZEPHYR_BASE}/scripts/gen_kobject_list.py ${ZEPHYR_BASE}/scripts/gen_kobject_list.py
${PARSE_SYSCALLS_TARGET} ${PARSE_SYSCALLS_TARGET}
${subsys_json} ${struct_tags_json}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
) )
add_custom_target(${DRIVER_VALIDATION_H_TARGET} DEPENDS ${DRV_VALIDATION}) add_custom_target(${DRIVER_VALIDATION_H_TARGET} DEPENDS ${DRV_VALIDATION})

2
scripts/gen_kobject_list.py
View file

@ -918,7 +918,7 @@ def write_kobj_size_output(fp):
def parse_subsystems_list_file(path): def parse_subsystems_list_file(path):
with open(path, "r") as fp: with open(path, "r") as fp:
subsys_list = json.load(fp) subsys_list = json.load(fp)
subsystems.extend(subsys_list) subsystems.extend(subsys_list["__subsystem"])
def parse_args(): def parse_args():
global args global args

57
scripts/parse_syscalls.py
View file

@ -10,10 +10,14 @@ Script to scan Zephyr include directories and emit system call and subsystem met
System calls require a great deal of boilerplate code in order to implement System calls require a great deal of boilerplate code in order to implement
completely. This script is the first step in the build system's process of completely. This script is the first step in the build system's process of
auto-generating this code by doing a text scan of directories containing auto-generating this code by doing a text scan of directories containing
header files, and building up a database of system calls and their C or header files, and building up a database of system calls and their
function call prototypes. This information is emitted to a generated function call prototypes. This information is emitted to a generated
JSON file for further processing. JSON file for further processing.
This script also scans for struct definitions such as __subsystem and
__net_socket, emitting a JSON dictionary mapping tags to all the struct
declarations found that were tagged with them.
If the output JSON file already exists, its contents are checked against If the output JSON file already exists, its contents are checked against
what information this script would have outputted; if the result is that the what information this script would have outputted; if the result is that the
file would be unchanged, it is not modified to prevent unnecessary file would be unchanged, it is not modified to prevent unnecessary
@ -26,24 +30,37 @@ import argparse
import os import os
import json import json
regex_flags = re.MULTILINE | re.VERBOSE
syscall_regex = re.compile(r''' syscall_regex = re.compile(r'''
__syscall\s+ # __syscall attribute, must be first __syscall\s+ # __syscall attribute, must be first
([^(]+) # type and name of system call (split later) ([^(]+) # type and name of system call (split later)
[(] # Function opening parenthesis [(] # Function opening parenthesis
([^)]*) # Arg list (split later) ([^)]*) # Arg list (split later)
[)] # Closing parenthesis [)] # Closing parenthesis
''', re.MULTILINE | re.VERBOSE) ''', regex_flags)
subsys_regex = re.compile(r''' struct_tags = ["__subsystem"]
__subsystem\s+ # __subsystem attribute, must be first
tagged_struct_decl_template = r'''
%s\s+ # tag, must be first
struct\s+ # struct keyword is next struct\s+ # struct keyword is next
([^{]+) # name of subsystem ([^{]+) # name of subsystem
[{] # Open curly bracket [{] # Open curly bracket
''', re.MULTILINE | re.VERBOSE) '''
def tagged_struct_update(target_list, tag, contents):
regex = re.compile(tagged_struct_decl_template % tag, regex_flags)
items = [mo.groups()[0].strip() for mo in regex.finditer(contents)]
target_list.extend(items)
def analyze_headers(multiple_directories): def analyze_headers(multiple_directories):
syscall_ret = [] syscall_ret = []
subsys_ret = [] tagged_ret = {}
for tag in struct_tags:
tagged_ret[tag] = []
for base_path in multiple_directories: for base_path in multiple_directories:
for root, dirs, files in os.walk(base_path, topdown=True): for root, dirs, files in os.walk(base_path, topdown=True):
@ -51,10 +68,12 @@ def analyze_headers(multiple_directories):
files.sort() files.sort()
for fn in files: for fn in files:
# toolchain/common.h has the definitions of __syscall and __subsystem which we # toolchain/common.h has the definitions of these tagswhich we
# don't want to trip over # don't want to trip over
path = os.path.join(root, fn) path = os.path.join(root, fn)
if not fn.endswith(".h") or path.endswith(os.path.join(os.sep, 'toolchain', 'common.h')): if (not (path.endswith(".h") or path.endswith(".c")) or
path.endswith(os.path.join(os.sep, 'toolchain',
'common.h'))):
continue continue
with open(path, "r", encoding="utf-8") as fp: with open(path, "r", encoding="utf-8") as fp:
@ -63,16 +82,15 @@ def analyze_headers(multiple_directories):
try: try:
syscall_result = [(mo.groups(), fn) syscall_result = [(mo.groups(), fn)
for mo in syscall_regex.finditer(contents)] for mo in syscall_regex.finditer(contents)]
subsys_result = [mo.groups()[0].strip() for tag in struct_tags:
for mo in subsys_regex.finditer(contents)] tagged_struct_update(tagged_ret[tag], tag, contents)
except Exception: except Exception:
sys.stderr.write("While parsing %s\n" % fn) sys.stderr.write("While parsing %s\n" % fn)
raise raise
syscall_ret.extend(syscall_result) syscall_ret.extend(syscall_result)
subsys_ret.extend(subsys_result)
return syscall_ret, subsys_ret return syscall_ret, tagged_ret
def update_file_if_changed(path, new): def update_file_if_changed(path, new):
@ -102,18 +120,19 @@ def parse_args():
"-j", "--json-file", required=True, "-j", "--json-file", required=True,
help="Write system call prototype information as json to file") help="Write system call prototype information as json to file")
parser.add_argument( parser.add_argument(
"-s", "--subsystem-file", required=True, "-t", "--tag-struct-file", required=True,
help="Write subsystem name information as json to file") help="Write tagged struct name information as json to file")
args = parser.parse_args() args = parser.parse_args()
def main(): def main():
parse_args() parse_args()
syscalls, subsys = analyze_headers(args.include) syscalls, tagged = analyze_headers(args.include)
# Only write json files if they don't exist or have changes since # Only write json files if they don't exist or have changes since
# they will force and incremental rebuild. # they will force an incremental rebuild.
syscalls_in_json = json.dumps( syscalls_in_json = json.dumps(
syscalls, syscalls,
@ -122,12 +141,12 @@ def main():
) )
update_file_if_changed(args.json_file, syscalls_in_json) update_file_if_changed(args.json_file, syscalls_in_json)
subsys_in_json = json.dumps( tagged_struct_in_json = json.dumps(
subsys, tagged,
indent=4, indent=4,
sort_keys=True sort_keys=True
) )
update_file_if_changed(args.subsystem_file, subsys_in_json) update_file_if_changed(args.tag_struct_file, tagged_struct_in_json)
if __name__ == "__main__": if __name__ == "__main__":