diff --git a/Kconfig.zephyr b/Kconfig.zephyr index 396aad9240..9d09ce3edf 100644 --- a/Kconfig.zephyr +++ b/Kconfig.zephyr @@ -192,6 +192,17 @@ config LINKER_USE_BOOT_SECTION Requires that boot sections exist in the architecture, SoC, board or custom linker script. +config LINKER_USE_PINNED_SECTION + bool "Enable Usage of Pinned Linker Section" + help + If enabled, the symbols which need to be pinned in memory + will be put into another linker section reserved for pinned + symbols. During boot, the corresponding memory will be marked + as pinned. + + Requires that pinned sections exist in the architecture, SoC, + board or custom linker script. + endmenu # "Linker Sections" endmenu diff --git a/include/linker/linker-defs.h b/include/linker/linker-defs.h index 7c032162ff..e0b23f62b3 100644 --- a/include/linker/linker-defs.h +++ b/include/linker/linker-defs.h @@ -341,6 +341,32 @@ extern char lnkr_boot_noinit_end[]; extern char lnkr_boot_noinit_size[]; #endif /* CONFIG_LINKER_USE_BOOT_SECTION */ +#ifdef CONFIG_LINKER_USE_PINNED_SECTION +/* lnkr_pinned_start[] and lnkr_pinned_end[] must encapsulate + * all the pinned sections as these are used by + * the MMU code to mark the physical page frames with + * Z_PAGE_FRAME_PINNED. + */ +extern char lnkr_pinned_start[]; +extern char lnkr_pinned_end[]; + +extern char lnkr_pinned_text_start[]; +extern char lnkr_pinned_text_end[]; +extern char lnkr_pinned_text_size[]; +extern char lnkr_pinned_data_start[]; +extern char lnkr_pinned_data_end[]; +extern char lnkr_pinned_data_size[]; +extern char lnkr_pinned_rodata_start[]; +extern char lnkr_pinned_rodata_end[]; +extern char lnkr_pinned_rodata_size[]; +extern char lnkr_pinned_bss_start[]; +extern char lnkr_pinned_bss_end[]; +extern char lnkr_pinned_bss_size[]; +extern char lnkr_pinned_noinit_start[]; +extern char lnkr_pinned_noinit_end[]; +extern char lnkr_pinned_noinit_size[]; +#endif /* CONFIG_LINKER_USE_PINNED_SECTION */ + #endif /* ! _ASMLANGUAGE */ #endif /* ZEPHYR_INCLUDE_LINKER_LINKER_DEFS_H_ */ diff --git a/include/linker/section_tags.h b/include/linker/section_tags.h index 5113411714..eaace84f51 100644 --- a/include/linker/section_tags.h +++ b/include/linker/section_tags.h @@ -66,6 +66,20 @@ #define __boot_noinit __noinit #endif /* CONFIG_LINKER_USE_BOOT_SECTION */ +#if defined(CONFIG_LINKER_USE_PINNED_SECTION) +#define __pinned_func Z_GENERIC_DOT_SECTION(PINNED_TEXT_SECTION_NAME) +#define __pinned_data Z_GENERIC_DOT_SECTION(PINNED_DATA_SECTION_NAME) +#define __pinned_rodata Z_GENERIC_DOT_SECTION(PINNED_RODATA_SECTION_NAME) +#define __pinned_bss Z_GENERIC_DOT_SECTION(PINNED_BSS_SECTION_NAME) +#define __pinned_noinit Z_GENERIC_DOT_SECTION(PINNED_NOINIT_SECTION_NAME) +#else +#define __pinned_func +#define __pinned_data +#define __pinned_rodata +#define __pinned_bss +#define __pinned_noinit __noinit +#endif /* CONFIG_LINKER_USE_PINNED_SECTION */ + #endif /* !_ASMLANGUAGE */ #endif /* ZEPHYR_INCLUDE_LINKER_SECTION_TAGS_H_ */ diff --git a/include/linker/sections.h b/include/linker/sections.h index 4ec09ce77e..f172cfb908 100644 --- a/include/linker/sections.h +++ b/include/linker/sections.h @@ -75,6 +75,14 @@ #define BOOT_NOINIT_SECTION_NAME boot_noinit #endif +#if defined(CONFIG_LINKER_USE_PINNED_SECTION) +#define PINNED_TEXT_SECTION_NAME pinned_text +#define PINNED_BSS_SECTION_NAME pinned_bss +#define PINNED_RODATA_SECTION_NAME pinned_rodata +#define PINNED_DATA_SECTION_NAME pinned_data +#define PINNED_NOINIT_SECTION_NAME pinned_noinit +#endif + /* Short section references for use in ASM files */ #if defined(_ASMLANGUAGE) /* Various text section names */ @@ -100,6 +108,20 @@ #define BOOT_NOINIT NOINIT #endif /* CONFIG_LINKER_USE_BOOT_SECTION */ +#if defined(CONFIG_LINKER_USE_PINNED_SECTION) +#define PINNED_TEXT PINNED_TEXT_SECTION_NAME +#define PINNED_BSS PINNED_BSS_SECTION_NAME +#define PINNED_RODATA PINNED_RODATA_SECTION_NAME +#define PINNED_DATA PINNED_DATA_SECTION_NAME +#define PINNED_NOINIT PINNED_NOINIT_SECTION_NAME +#else +#define PINNED_TEXT TEXT +#define PINNED_BSS BSS +#define PINNED_RODATA RODATA +#define PINNED_DATA DATA +#define PINNED_NOINIT NOINIT +#endif /* CONFIG_LINKER_USE_PINNED_SECTION */ + #endif /* _ASMLANGUAGE */ #include diff --git a/include/sys/thread_stack.h b/include/sys/thread_stack.h index 83cce2984a..99cef4b1da 100644 --- a/include/sys/thread_stack.h +++ b/include/sys/thread_stack.h @@ -175,6 +175,27 @@ static inline char *z_stack_ptr_align(char *ptr) #define K_KERNEL_STACK_DEFINE(sym, size) \ Z_KERNEL_STACK_DEFINE_IN(sym, size, __kstackmem) +/** + * @def K_KERNEL_PINNED_STACK_DEFINE + * @brief Define a toplevel kernel stack memory region in pinned section + * + * See K_KERNEL_STACK_DEFINE() for more information and constraints. + * + * This puts the stack into the pinned noinit linker section if + * CONFIG_LINKER_USE_PINNED_SECTION is enabled, or else it would + * put the stack into the same section as K_KERNEL_STACK_DEFINE(). + * + * @param sym Thread stack symbol name + * @param size Size of the stack memory region + */ +#if defined(CONFIG_LINKER_USE_PINNED_SECTION) +#define K_KERNEL_PINNED_STACK_DEFINE(sym, size) \ + Z_KERNEL_STACK_DEFINE_IN(sym, size, __pinned_noinit) +#else +#define K_KERNEL_PINNED_STACK_DEFINE(sym, size) \ + Z_KERNEL_STACK_DEFINE_IN(sym, size, __kstackmem) +#endif + #define Z_KERNEL_STACK_LEN(size) \ ROUND_UP(Z_KERNEL_STACK_SIZE_ADJUST(size), Z_KERNEL_STACK_OBJ_ALIGN) @@ -191,6 +212,28 @@ static inline char *z_stack_ptr_align(char *ptr) #define K_KERNEL_STACK_ARRAY_DEFINE(sym, nmemb, size) \ Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __kstackmem) +/** + * @def K_KERNEL_PINNED_STACK_ARRAY_DEFINE + * @brief Define a toplevel array of kernel stack memory regions in pinned section + * + * See K_KERNEL_STACK_ARRAY_DEFINE() for more information and constraints. + * + * This puts the stack into the pinned noinit linker section if + * CONFIG_LINKER_USE_PINNED_SECTION is enabled, or else it would + * put the stack into the same section as K_KERNEL_STACK_ARRAY_DEFINE(). + * + * @param sym Kernel stack array symbol name + * @param nmemb Number of stacks to declare + * @param size Size of the stack memory region + */ +#if defined(CONFIG_LINKER_USE_PINNED_SECTION) +#define K_KERNEL_PINNED_STACK_ARRAY_DEFINE(sym, nmemb, size) \ + Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __pinned_noinit) +#else +#define K_KERNEL_PINNED_STACK_ARRAY_DEFINE(sym, nmemb, size) \ + Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __kstackmem) +#endif + /** * @def K_KERNEL_STACK_MEMBER * @brief Declare an embedded stack memory region diff --git a/kernel/include/kernel_internal.h b/kernel/include/kernel_internal.h index 6e8bbeac17..4ad0d54117 100644 --- a/kernel/include/kernel_internal.h +++ b/kernel/include/kernel_internal.h @@ -45,6 +45,15 @@ static inline void z_bss_zero_boot(void) } #endif +#ifdef CONFIG_LINKER_USE_PINNED_SECTION +void z_bss_zero_pinned(void); +#else +static inline void z_bss_zero_pinned(void) +{ + /* Do nothing */ +} +#endif + FUNC_NORETURN void z_cstart(void); void z_device_state_init(void); diff --git a/kernel/init.c b/kernel/init.c index ce230a8ae9..396a77fd5e 100644 --- a/kernel/init.c +++ b/kernel/init.c @@ -127,6 +127,28 @@ void z_bss_zero_boot(void) } #endif /* CONFIG_LINKER_USE_BOOT_SECTION */ +#ifdef CONFIG_LINKER_USE_PINNED_SECTION +/** + * @brief Clear BSS within the pinned region + * + * This routine clears the BSS within the pinned region. + * This is separate from z_bss_zero() as pinned region may + * contain symbols required for the boot process before + * paging is initialized. + */ +#ifdef CONFIG_LINKER_USE_BOOT_SECTION +__boot_func +#else +__pinned_func +#endif +void z_bss_zero_pinned(void) +{ + (void)memset(&lnkr_pinned_bss_start, 0, + (uintptr_t)&lnkr_pinned_bss_end + - (uintptr_t)&lnkr_pinned_bss_start); +} +#endif /* CONFIG_LINKER_USE_PINNED_SECTION */ + #ifdef CONFIG_STACK_CANARIES extern volatile uintptr_t __stack_chk_guard; #endif /* CONFIG_STACK_CANARIES */ diff --git a/kernel/mmu.c b/kernel/mmu.c index 84cd98d6be..717f227fe8 100644 --- a/kernel/mmu.c +++ b/kernel/mmu.c @@ -761,6 +761,23 @@ void z_mem_manage_init(void) pf->flags |= Z_PAGE_FRAME_PINNED; } +#ifdef CONFIG_LINKER_USE_PINNED_SECTION + /* Pin the page frames correspondng to the pinned symbols */ + uintptr_t pinned_start = ROUND_DOWN(POINTER_TO_UINT(lnkr_pinned_start), + CONFIG_MMU_PAGE_SIZE); + uintptr_t pinned_end = ROUND_UP(POINTER_TO_UINT(lnkr_pinned_end), + CONFIG_MMU_PAGE_SIZE); + size_t pinned_size = pinned_end - pinned_start; + + VIRT_FOREACH(UINT_TO_POINTER(pinned_start), pinned_size, addr) + { + pf = z_phys_to_page_frame(Z_BOOT_VIRT_TO_PHYS(addr)); + frame_mapped_set(pf, addr); + + pf->flags |= Z_PAGE_FRAME_PINNED; + } +#endif + /* Any remaining pages that aren't mapped, reserved, or pinned get * added to the free pages list */