patrick/zephyr
1
0
Fork 0
Code Issues Pull requests Actions 8 Packages Projects Releases Wiki Activity

linker: sort app shared mem partition by alignment

Browse source 
If CONFIG_MPU_REQUIRES_POWER_OF_TWO_ALIGNMENT is enabled,
the app shared memory partition may cause waste of memory
due to the need for padding.

For example, tests/subsys/jwt and board mps2_an385:

  z_test_mem_partition: addr 0x20000000, size 52
  z_libc_partition    : addr 0x20000040, size 4
  k_mbedtls_partition : addr 0x20008000, size 32736

    ending at 0x2000ffff, taking up 65536 bytes

With power-of-two size and alignment requirement,
k_mbedtls_partition takes up 32KB memory and needs to be
aligned on 32KB boundary. If the above partitions are
ordered as shown, there needs to be a lot of padding
after z_libc_partition before k_mbedtls_partition can
start. In order to minimize padding, these partitions
need to be sort by size in descending order.

After the changes here,	the partitions are:

  k_mbedtls_partition : addr 0x20000000, size 32736
  z_test_mem_partition: addr 0x20008000, size 52
  z_libc_partition    : addr 0x20008040, size 4

    ending at 0x2000805f, taking up 32864 bytes

With the above example, sorting results in a saving
of 32672 bytes of saving.

Fixes #14121

Signed-off-by: Daniel Leung <daniel.leung@intel.com>
This commit is contained in:
Daniel Leung 2019-03-10 14:20:21 -07:00 committed by Anas Nashif
parent e8a2348fac
commit 212ec9a29a
6 changed files with 203 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

85
CMakeLists.txt
View file

@ -363,7 +363,8 @@ zephyr_cc_option(-Wpointer-arith)
# Declare MPU userspace dependencies before the linker scripts to make
# sure the order of dependencies are met
if(CONFIG_USERSPACE)
set(APP_SMEM_DEP app_smem_linker)
set(APP_SMEM_ALIGNED_DEP app_smem_aligned_linker)
set(APP_SMEM_UNALIGNED_DEP app_smem_unaligned_linker)
if(CONFIG_ARM)
set(PRIV_STACK_DEP priv_stacks_prebuilt)
endif()
@ -446,6 +447,8 @@ function(construct_add_custom_command_for_linker_pass linker_output_name output_
if (${linker_output_name} MATCHES "^linker_pass_final$")
set(linker_pass_define -DLINKER_PASS2)
elseif (${linker_output_name} MATCHES "^linker_app_smem_unaligned$")
set(linker_pass_define -DLINKER_APP_SMEM_UNALIGNED)
else()
set(linker_pass_define "")
endif()
@ -761,7 +764,7 @@ construct_add_custom_command_for_linker_pass(
custom_command
${ALIGN_SIZING_DEP}
${PRIV_STACK_DEP}
${APP_SMEM_DEP}
${APP_SMEM_ALIGNED_DEP}
${CODE_RELOCATION_DEP}
${OFFSETS_H_TARGET}
)
@ -1134,14 +1137,29 @@ configure_file(
$ENV{ZEPHYR_BASE}/include/linker/app_smem.ld
${PROJECT_BINARY_DIR}/include/generated/app_smem.ld)
configure_file(
$ENV{ZEPHYR_BASE}/include/linker/app_smem_aligned.ld
${PROJECT_BINARY_DIR}/include/generated/app_smem_aligned.ld)
configure_file(
$ENV{ZEPHYR_BASE}/include/linker/app_smem_unaligned.ld
${PROJECT_BINARY_DIR}/include/generated/app_smem_unaligned.ld)
if(CONFIG_USERSPACE)
set(APP_SMEM_LD "${PROJECT_BINARY_DIR}/include/generated/app_smem.ld")
set(APP_SMEM_ALIGNED_LD "${PROJECT_BINARY_DIR}/include/generated/app_smem_aligned.ld")
set(APP_SMEM_UNALIGNED_LD "${PROJECT_BINARY_DIR}/include/generated/app_smem_unaligned.ld")
set(OBJ_FILE_DIR "${PROJECT_BINARY_DIR}/../")
add_custom_target(
${APP_SMEM_DEP}
${APP_SMEM_ALIGNED_DEP}
DEPENDS
${APP_SMEM_LD}
${APP_SMEM_ALIGNED_LD}
)
add_custom_target(
${APP_SMEM_UNALIGNED_DEP}
DEPENDS
${APP_SMEM_UNALIGNED_LD}
)
if(CONFIG_NEWLIB_LIBC)
@ -1152,18 +1170,65 @@ if(CONFIG_USERSPACE)
endif()
add_custom_command(
OUTPUT ${APP_SMEM_LD}
OUTPUT ${APP_SMEM_UNALIGNED_LD}
COMMAND ${PYTHON_EXECUTABLE}
${ZEPHYR_BASE}/scripts/gen_app_partitions.py
-d ${OBJ_FILE_DIR}
-o ${APP_SMEM_LD}
-o ${APP_SMEM_UNALIGNED_LD}
${NEWLIB_PART} ${MBEDTLS_PART}
$<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:--verbose>
DEPENDS
kernel
${ZEPHYR_LIBS_PROPERTY}
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/
COMMENT "Generating app_smem linker section"
COMMENT "Generating app_smem_unaligned linker section"
)
construct_add_custom_command_for_linker_pass(
linker_app_smem_unaligned
custom_command
${ALIGN_SIZING_DEP}
${CODE_RELOCATION_DEP}
${APP_SMEM_UNALIGNED_DEP}
${APP_SMEM_UNALIGNED_LD}
${OFFSETS_H_TARGET}
)
add_custom_command(
${custom_command}
)
add_custom_target(
linker_app_smem_unaligned_script
DEPENDS
linker_app_smem_unaligned.cmd
)
set_property(TARGET
linker_app_smem_unaligned_script
PROPERTY INCLUDE_DIRECTORIES
${ZEPHYR_INCLUDE_DIRS}
)
set(APP_SMEM_UNALIGNED_LIB app_smem_unaligned_output_obj_renamed_lib)
add_executable( app_smem_unaligned_prebuilt misc/empty_file.c)
target_link_libraries(app_smem_unaligned_prebuilt ${TOPT} ${PROJECT_BINARY_DIR}/linker_app_smem_unaligned.cmd ${zephyr_lnk} ${CODE_RELOCATION_DEP})
set_property(TARGET app_smem_unaligned_prebuilt PROPERTY LINK_DEPENDS ${PROJECT_BINARY_DIR}/linker_app_smem_unaligned.cmd)
add_dependencies( app_smem_unaligned_prebuilt ${ALIGN_SIZING_DEP} linker_app_smem_unaligned_script ${OFFSETS_LIB})
add_custom_command(
OUTPUT ${APP_SMEM_ALIGNED_LD}
COMMAND ${PYTHON_EXECUTABLE}
${ZEPHYR_BASE}/scripts/gen_app_partitions.py
-e $<TARGET_FILE:app_smem_unaligned_prebuilt>
-o ${APP_SMEM_ALIGNED_LD}
${NEWLIB_PART} ${MBEDTLS_PART}
$<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:--verbose>
DEPENDS
kernel
${ZEPHYR_LIBS_PROPERTY}
app_smem_unaligned_prebuilt
WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/
COMMENT "Generating app_smem_aligned linker section"
)
endif()
@ -1173,8 +1238,8 @@ if(CONFIG_USERSPACE AND CONFIG_ARM)
custom_command
${ALIGN_SIZING_DEP}
${CODE_RELOCATION_DEP}
${APP_SMEM_DEP}
${APP_SMEM_LD}
${APP_SMEM_ALIGNED_DEP}
${APP_SMEM_ALIGNED_LD}
${OFFSETS_H_TARGET}
)
add_custom_command(

1
CODEOWNERS
View file

@ -186,6 +186,7 @@
/include/kernel_version.h @andrewboie @andyross
/include/led.h @Mani-Sadhasivam
/include/led_strip.h @mbolivar
/include/linker/app_smem*.ld @andrewboie
/include/linker/linker-defs.h @andrewboie @andyross
/include/linker/linker-tool-gcc.h @andrewboie @andyross
/include/linker/linker-tool.h @andrewboie @andyross

38
include/linker/app_smem.ld
View file

@ -1,2 +1,36 @@
/* space holder */
APP_SMEM_SECTION()
/*
* This hackish way of including files is due to CMake issues:
* https://gitlab.kitware.com/cmake/cmake/issues/11985
* https://gitlab.kitware.com/cmake/cmake/issues/13718
*
* When using the "Unix Makefiles" generator, CMake simply
* greps for "#include" to generate dependency list.
* So if doing it normally, both files are being included
* in the dependency list. This creates weird dependency
* issue:
*
* 1. Using A.ld to create a linker script A.cmd.
* 2. Using A.cmd to generate A_prebuilt.elf.
* 3. Using A_prebuilt.elf to create B.ld.
* 4. Creating B.cmd with B.ld.
* 5. Creating B_prebuilt.elf using B.cmd.
*
* Since the dependency list of A.cmd contains both
* A.ld and B.ld, when make is invoked again, B.ld
* is newer than A.cmd so everything from this point on
* gets rebuilt. In order to break this cycle, this
* hackish needs to be used since CMake does not parse
* macros, and thus these will not appear in
* the dependency list. The dependencies should then be
* put in CMakeLists.txt instead.
*
* Note: Ninja generator does not suffer from this issue.
*/
#ifdef LINKER_APP_SMEM_UNALIGNED
#define APP_SMEM_LD <app_smem_unaligned.ld>
#else
#define APP_SMEM_LD <app_smem_aligned.ld>
#endif
#include APP_SMEM_LD
#undef APP_SMEM_LD

2
include/linker/app_smem_aligned.ld Normal file
View file

@ -0,0 +1,2 @@
/* space holder */
APP_SMEM_SECTION()

2
include/linker/app_smem_unaligned.ld Normal file
View file

@ -0,0 +1,2 @@
/* space holder */
APP_SMEM_SECTION()

111
scripts/gen_app_partitions.py
View file

@ -37,9 +37,16 @@ import argparse
import os
import re
import string
import subprocess
from collections import OrderedDict
from elf_helper import ElfHelper
from elftools.elf.elffile import ELFFile
from elftools.elf.sections import SymbolTableSection
from operator import itemgetter
SZ = 'size'
SRC = 'sources'
LIB = 'libraries'
# This script will create sections and linker variables to place the
# application shared memory partitions.
@ -91,7 +98,9 @@ size_cal_string = """
section_regex = re.compile(r'data_smem_([A-Za-z0-9_]*)_(data|bss)')
def find_partitions(filename, partitions, sources):
elf_part_size_regex = re.compile(r'z_data_smem_(.*)_part_size')
def find_obj_file_partitions(filename, partitions):
with open(filename, 'rb') as f:
full_lib = ELFFile( f)
if (not full_lib):
@ -106,23 +115,67 @@ def find_partitions(filename, partitions, sources):
partition_name = m.groups()[0]
if partition_name not in partitions:
partitions[partition_name] = []
if args.verbose:
sources.update({partition_name: filename})
partitions[partition_name] = {SZ: section.header.sh_size}
return (partitions, sources)
if args.verbose:
partitions[partition_name][SRC] = filename
else:
partitions[partition_name][SZ] += section.header.sh_size
return partitions
def parse_obj_files(partitions):
# Iterate over all object files to find partitions
for dirpath, dirs, files in os.walk(args.directory):
for filename in files:
if re.match(".*\.obj$",filename):
fullname = os.path.join(dirpath, filename)
find_obj_file_partitions(fullname, partitions)
def parse_elf_file(partitions):
with open(args.elf, 'rb') as f:
elffile = ELFFile(f)
symbol_tbls = [s for s in elffile.iter_sections()
if isinstance(s, SymbolTableSection)]
for section in symbol_tbls:
for nsym, symbol in enumerate(section.iter_symbols()):
if symbol['st_shndx'] != "SHN_ABS":
continue
x = elf_part_size_regex.match(symbol.name)
if not x:
continue
partition_name = x.groups()[0]
size = symbol['st_value']
if partition_name not in partitions:
partitions[partition_name] = {SZ: size}
if args.verbose:
partitions[partition_name][SRC] = args.elf
else:
partitions[partition_name][SZ] += size
def generate_final_linker(linker_file, partitions):
string = linker_start_seq
size_string = ''
for partition, libs in partitions.items():
for partition, item in partitions.items():
string += data_template.format(partition)
for lib in libs:
string += library_data_template.format(lib)
if LIB in item:
for lib in item[LIB]:
string += library_data_template.format(lib)
string += bss_template.format(partition)
for lib in libs:
string += library_bss_template.format(lib)
if LIB in item:
for lib in item[LIB]:
string += library_bss_template.format(lib)
string += footer_template.format(partition)
size_string += size_cal_string.format(partition)
@ -137,8 +190,10 @@ def parse_args():
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("-d", "--directory", required=True,
parser.add_argument("-d", "--directory", required=False, default=None,
help="Root build directory")
parser.add_argument("-e", "--elf", required=False, default=None,
help="ELF file")
parser.add_argument("-o", "--output", required=False,
help="Output ld file")
parser.add_argument("-v", "--verbose", action="count", default =0,
@ -154,26 +209,34 @@ def main():
parse_args()
linker_file = args.output
partitions = {}
sources = {}
for dirpath, dirs, files in os.walk(args.directory):
for filename in files:
if re.match(".*\.obj$",filename):
fullname = os.path.join(dirpath, filename)
find_partitions(fullname, partitions,
sources)
if args.directory is not None:
parse_obj_files(partitions)
elif args.elf is not None:
parse_elf_file(partitions)
else:
return
for lib, ptn in args.library:
if ptn not in partitions:
partitions[ptn] = [lib]
else:
partitions[ptn].append(lib)
partitions[ptn] = {}
generate_final_linker(args.output, partitions)
if LIB not in partitions[ptn]:
partitions[ptn][LIB] = [lib]
else:
partitions[ptn][LIB].append(lib)
partsorted = OrderedDict(sorted(partitions.items(),
key=lambda x: x[1][SZ], reverse=True))
generate_final_linker(args.output, partsorted)
if args.verbose:
print("Partitions retrieved:")
for key in partitions:
print(" %s: %s\n", key, sources[key])
for key in partsorted:
print(" {0}: size {1}: {2}".format(key,
partsorted[key][SZ],
partsorted[key][SRC]))
if __name__ == '__main__':
main()