From 1310ad6b0e1434ec6d347cef14203eaeac012456 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Tue, 23 Feb 2021 13:33:38 -0800 Subject: [PATCH] linker: add bits for pinned regions This adds the necessary bits for linker scripts and source code to specify which symbols need to be pinned in memory. This is needed for demand paging as some functions and data must reside in memory all the time and cannot be paged out (e.g. paging, scheduler, and interrupt routines for functionality). This is up to the arch/SoC/board to define the sections in their linker scripts as the pinned section may need special alignment which cannot be done in common script snippets. Signed-off-by: Daniel Leung --- Kconfig.zephyr | 11 ++++++++ include/linker/linker-defs.h | 26 +++++++++++++++++++ include/linker/section_tags.h | 14 +++++++++++ include/linker/sections.h | 22 ++++++++++++++++ include/sys/thread_stack.h | 43 ++++++++++++++++++++++++++++++++ kernel/include/kernel_internal.h | 9 +++++++ kernel/init.c | 22 ++++++++++++++++ kernel/mmu.c | 17 +++++++++++++ 8 files changed, 164 insertions(+) diff --git a/Kconfig.zephyr b/Kconfig.zephyr index 396aad9240..9d09ce3edf 100644 --- a/Kconfig.zephyr +++ b/Kconfig.zephyr @@ -192,6 +192,17 @@ config LINKER_USE_BOOT_SECTION Requires that boot sections exist in the architecture, SoC, board or custom linker script. +config LINKER_USE_PINNED_SECTION + bool "Enable Usage of Pinned Linker Section" + help + If enabled, the symbols which need to be pinned in memory + will be put into another linker section reserved for pinned + symbols. During boot, the corresponding memory will be marked + as pinned. + + Requires that pinned sections exist in the architecture, SoC, + board or custom linker script. + endmenu # "Linker Sections" endmenu diff --git a/include/linker/linker-defs.h b/include/linker/linker-defs.h index 7c032162ff..e0b23f62b3 100644 --- a/include/linker/linker-defs.h +++ b/include/linker/linker-defs.h @@ -341,6 +341,32 @@ extern char lnkr_boot_noinit_end[]; extern char lnkr_boot_noinit_size[]; #endif /* CONFIG_LINKER_USE_BOOT_SECTION */ +#ifdef CONFIG_LINKER_USE_PINNED_SECTION +/* lnkr_pinned_start[] and lnkr_pinned_end[] must encapsulate + * all the pinned sections as these are used by + * the MMU code to mark the physical page frames with + * Z_PAGE_FRAME_PINNED. + */ +extern char lnkr_pinned_start[]; +extern char lnkr_pinned_end[]; + +extern char lnkr_pinned_text_start[]; +extern char lnkr_pinned_text_end[]; +extern char lnkr_pinned_text_size[]; +extern char lnkr_pinned_data_start[]; +extern char lnkr_pinned_data_end[]; +extern char lnkr_pinned_data_size[]; +extern char lnkr_pinned_rodata_start[]; +extern char lnkr_pinned_rodata_end[]; +extern char lnkr_pinned_rodata_size[]; +extern char lnkr_pinned_bss_start[]; +extern char lnkr_pinned_bss_end[]; +extern char lnkr_pinned_bss_size[]; +extern char lnkr_pinned_noinit_start[]; +extern char lnkr_pinned_noinit_end[]; +extern char lnkr_pinned_noinit_size[]; +#endif /* CONFIG_LINKER_USE_PINNED_SECTION */ + #endif /* ! _ASMLANGUAGE */ #endif /* ZEPHYR_INCLUDE_LINKER_LINKER_DEFS_H_ */ diff --git a/include/linker/section_tags.h b/include/linker/section_tags.h index 5113411714..eaace84f51 100644 --- a/include/linker/section_tags.h +++ b/include/linker/section_tags.h @@ -66,6 +66,20 @@ #define __boot_noinit __noinit #endif /* CONFIG_LINKER_USE_BOOT_SECTION */ +#if defined(CONFIG_LINKER_USE_PINNED_SECTION) +#define __pinned_func Z_GENERIC_DOT_SECTION(PINNED_TEXT_SECTION_NAME) +#define __pinned_data Z_GENERIC_DOT_SECTION(PINNED_DATA_SECTION_NAME) +#define __pinned_rodata Z_GENERIC_DOT_SECTION(PINNED_RODATA_SECTION_NAME) +#define __pinned_bss Z_GENERIC_DOT_SECTION(PINNED_BSS_SECTION_NAME) +#define __pinned_noinit Z_GENERIC_DOT_SECTION(PINNED_NOINIT_SECTION_NAME) +#else +#define __pinned_func +#define __pinned_data +#define __pinned_rodata +#define __pinned_bss +#define __pinned_noinit __noinit +#endif /* CONFIG_LINKER_USE_PINNED_SECTION */ + #endif /* !_ASMLANGUAGE */ #endif /* ZEPHYR_INCLUDE_LINKER_SECTION_TAGS_H_ */ diff --git a/include/linker/sections.h b/include/linker/sections.h index 4ec09ce77e..f172cfb908 100644 --- a/include/linker/sections.h +++ b/include/linker/sections.h @@ -75,6 +75,14 @@ #define BOOT_NOINIT_SECTION_NAME boot_noinit #endif +#if defined(CONFIG_LINKER_USE_PINNED_SECTION) +#define PINNED_TEXT_SECTION_NAME pinned_text +#define PINNED_BSS_SECTION_NAME pinned_bss +#define PINNED_RODATA_SECTION_NAME pinned_rodata +#define PINNED_DATA_SECTION_NAME pinned_data +#define PINNED_NOINIT_SECTION_NAME pinned_noinit +#endif + /* Short section references for use in ASM files */ #if defined(_ASMLANGUAGE) /* Various text section names */ @@ -100,6 +108,20 @@ #define BOOT_NOINIT NOINIT #endif /* CONFIG_LINKER_USE_BOOT_SECTION */ +#if defined(CONFIG_LINKER_USE_PINNED_SECTION) +#define PINNED_TEXT PINNED_TEXT_SECTION_NAME +#define PINNED_BSS PINNED_BSS_SECTION_NAME +#define PINNED_RODATA PINNED_RODATA_SECTION_NAME +#define PINNED_DATA PINNED_DATA_SECTION_NAME +#define PINNED_NOINIT PINNED_NOINIT_SECTION_NAME +#else +#define PINNED_TEXT TEXT +#define PINNED_BSS BSS +#define PINNED_RODATA RODATA +#define PINNED_DATA DATA +#define PINNED_NOINIT NOINIT +#endif /* CONFIG_LINKER_USE_PINNED_SECTION */ + #endif /* _ASMLANGUAGE */ #include diff --git a/include/sys/thread_stack.h b/include/sys/thread_stack.h index 83cce2984a..99cef4b1da 100644 --- a/include/sys/thread_stack.h +++ b/include/sys/thread_stack.h @@ -175,6 +175,27 @@ static inline char *z_stack_ptr_align(char *ptr) #define K_KERNEL_STACK_DEFINE(sym, size) \ Z_KERNEL_STACK_DEFINE_IN(sym, size, __kstackmem) +/** + * @def K_KERNEL_PINNED_STACK_DEFINE + * @brief Define a toplevel kernel stack memory region in pinned section + * + * See K_KERNEL_STACK_DEFINE() for more information and constraints. + * + * This puts the stack into the pinned noinit linker section if + * CONFIG_LINKER_USE_PINNED_SECTION is enabled, or else it would + * put the stack into the same section as K_KERNEL_STACK_DEFINE(). + * + * @param sym Thread stack symbol name + * @param size Size of the stack memory region + */ +#if defined(CONFIG_LINKER_USE_PINNED_SECTION) +#define K_KERNEL_PINNED_STACK_DEFINE(sym, size) \ + Z_KERNEL_STACK_DEFINE_IN(sym, size, __pinned_noinit) +#else +#define K_KERNEL_PINNED_STACK_DEFINE(sym, size) \ + Z_KERNEL_STACK_DEFINE_IN(sym, size, __kstackmem) +#endif + #define Z_KERNEL_STACK_LEN(size) \ ROUND_UP(Z_KERNEL_STACK_SIZE_ADJUST(size), Z_KERNEL_STACK_OBJ_ALIGN) @@ -191,6 +212,28 @@ static inline char *z_stack_ptr_align(char *ptr) #define K_KERNEL_STACK_ARRAY_DEFINE(sym, nmemb, size) \ Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __kstackmem) +/** + * @def K_KERNEL_PINNED_STACK_ARRAY_DEFINE + * @brief Define a toplevel array of kernel stack memory regions in pinned section + * + * See K_KERNEL_STACK_ARRAY_DEFINE() for more information and constraints. + * + * This puts the stack into the pinned noinit linker section if + * CONFIG_LINKER_USE_PINNED_SECTION is enabled, or else it would + * put the stack into the same section as K_KERNEL_STACK_ARRAY_DEFINE(). + * + * @param sym Kernel stack array symbol name + * @param nmemb Number of stacks to declare + * @param size Size of the stack memory region + */ +#if defined(CONFIG_LINKER_USE_PINNED_SECTION) +#define K_KERNEL_PINNED_STACK_ARRAY_DEFINE(sym, nmemb, size) \ + Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __pinned_noinit) +#else +#define K_KERNEL_PINNED_STACK_ARRAY_DEFINE(sym, nmemb, size) \ + Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __kstackmem) +#endif + /** * @def K_KERNEL_STACK_MEMBER * @brief Declare an embedded stack memory region diff --git a/kernel/include/kernel_internal.h b/kernel/include/kernel_internal.h index 6e8bbeac17..4ad0d54117 100644 --- a/kernel/include/kernel_internal.h +++ b/kernel/include/kernel_internal.h @@ -45,6 +45,15 @@ static inline void z_bss_zero_boot(void) } #endif +#ifdef CONFIG_LINKER_USE_PINNED_SECTION +void z_bss_zero_pinned(void); +#else +static inline void z_bss_zero_pinned(void) +{ + /* Do nothing */ +} +#endif + FUNC_NORETURN void z_cstart(void); void z_device_state_init(void); diff --git a/kernel/init.c b/kernel/init.c index ce230a8ae9..396a77fd5e 100644 --- a/kernel/init.c +++ b/kernel/init.c @@ -127,6 +127,28 @@ void z_bss_zero_boot(void) } #endif /* CONFIG_LINKER_USE_BOOT_SECTION */ +#ifdef CONFIG_LINKER_USE_PINNED_SECTION +/** + * @brief Clear BSS within the pinned region + * + * This routine clears the BSS within the pinned region. + * This is separate from z_bss_zero() as pinned region may + * contain symbols required for the boot process before + * paging is initialized. + */ +#ifdef CONFIG_LINKER_USE_BOOT_SECTION +__boot_func +#else +__pinned_func +#endif +void z_bss_zero_pinned(void) +{ + (void)memset(&lnkr_pinned_bss_start, 0, + (uintptr_t)&lnkr_pinned_bss_end + - (uintptr_t)&lnkr_pinned_bss_start); +} +#endif /* CONFIG_LINKER_USE_PINNED_SECTION */ + #ifdef CONFIG_STACK_CANARIES extern volatile uintptr_t __stack_chk_guard; #endif /* CONFIG_STACK_CANARIES */ diff --git a/kernel/mmu.c b/kernel/mmu.c index 84cd98d6be..717f227fe8 100644 --- a/kernel/mmu.c +++ b/kernel/mmu.c @@ -761,6 +761,23 @@ void z_mem_manage_init(void) pf->flags |= Z_PAGE_FRAME_PINNED; } +#ifdef CONFIG_LINKER_USE_PINNED_SECTION + /* Pin the page frames correspondng to the pinned symbols */ + uintptr_t pinned_start = ROUND_DOWN(POINTER_TO_UINT(lnkr_pinned_start), + CONFIG_MMU_PAGE_SIZE); + uintptr_t pinned_end = ROUND_UP(POINTER_TO_UINT(lnkr_pinned_end), + CONFIG_MMU_PAGE_SIZE); + size_t pinned_size = pinned_end - pinned_start; + + VIRT_FOREACH(UINT_TO_POINTER(pinned_start), pinned_size, addr) + { + pf = z_phys_to_page_frame(Z_BOOT_VIRT_TO_PHYS(addr)); + frame_mapped_set(pf, addr); + + pf->flags |= Z_PAGE_FRAME_PINNED; + } +#endif + /* Any remaining pages that aren't mapped, reserved, or pinned get * added to the free pages list */