linker: add bits for pinned regions

This adds the necessary bits for linker scripts and source code
to specify which symbols need to be pinned in memory. This is
needed for demand paging as some functions and data must reside
in memory all the time and cannot be paged out (e.g. paging,
scheduler, and interrupt routines for functionality).

This is up to the arch/SoC/board to define the sections in
their linker scripts as the pinned section may need special
alignment which cannot be done in common script snippets.

Signed-off-by: Daniel Leung <daniel.leung@intel.com>

Kconfig.zephyr

@@ -192,6 +192,17 @@ config LINKER_USE_BOOT_SECTION
 	  Requires that boot sections exist in the architecture, SoC,
 	  board or custom linker script.
 
+config LINKER_USE_PINNED_SECTION
+	bool "Enable Usage of Pinned Linker Section"
+	help
+	  If enabled, the symbols which need to be pinned in memory
+	  will be put into another linker section reserved for pinned
+	  symbols. During boot, the corresponding memory will be marked
+	  as pinned.
+
+	  Requires that pinned sections exist in the architecture, SoC,
+	  board or custom linker script.
+
 endmenu # "Linker Sections"
 
 endmenu

include/linker/linker-defs.h

@@ -341,6 +341,32 @@ extern char lnkr_boot_noinit_end[];
 extern char lnkr_boot_noinit_size[];
 #endif /* CONFIG_LINKER_USE_BOOT_SECTION */
 
+#ifdef CONFIG_LINKER_USE_PINNED_SECTION
+/* lnkr_pinned_start[] and lnkr_pinned_end[] must encapsulate
+ * all the pinned sections as these are used by
+ * the MMU code to mark the physical page frames with
+ * Z_PAGE_FRAME_PINNED.
+ */
+extern char lnkr_pinned_start[];
+extern char lnkr_pinned_end[];
+
+extern char lnkr_pinned_text_start[];
+extern char lnkr_pinned_text_end[];
+extern char lnkr_pinned_text_size[];
+extern char lnkr_pinned_data_start[];
+extern char lnkr_pinned_data_end[];
+extern char lnkr_pinned_data_size[];
+extern char lnkr_pinned_rodata_start[];
+extern char lnkr_pinned_rodata_end[];
+extern char lnkr_pinned_rodata_size[];
+extern char lnkr_pinned_bss_start[];
+extern char lnkr_pinned_bss_end[];
+extern char lnkr_pinned_bss_size[];
+extern char lnkr_pinned_noinit_start[];
+extern char lnkr_pinned_noinit_end[];
+extern char lnkr_pinned_noinit_size[];
+#endif /* CONFIG_LINKER_USE_PINNED_SECTION */
+
 #endif /* ! _ASMLANGUAGE */
 
 #endif /* ZEPHYR_INCLUDE_LINKER_LINKER_DEFS_H_ */

include/linker/section_tags.h

@@ -66,6 +66,20 @@
 #define __boot_noinit	__noinit
 #endif /* CONFIG_LINKER_USE_BOOT_SECTION */
 
+#if defined(CONFIG_LINKER_USE_PINNED_SECTION)
+#define __pinned_func	Z_GENERIC_DOT_SECTION(PINNED_TEXT_SECTION_NAME)
+#define __pinned_data	Z_GENERIC_DOT_SECTION(PINNED_DATA_SECTION_NAME)
+#define __pinned_rodata	Z_GENERIC_DOT_SECTION(PINNED_RODATA_SECTION_NAME)
+#define __pinned_bss	Z_GENERIC_DOT_SECTION(PINNED_BSS_SECTION_NAME)
+#define __pinned_noinit	Z_GENERIC_DOT_SECTION(PINNED_NOINIT_SECTION_NAME)
+#else
+#define __pinned_func
+#define __pinned_data
+#define __pinned_rodata
+#define __pinned_bss
+#define __pinned_noinit	__noinit
+#endif /* CONFIG_LINKER_USE_PINNED_SECTION */
+
 #endif /* !_ASMLANGUAGE */
 
 #endif /* ZEPHYR_INCLUDE_LINKER_SECTION_TAGS_H_ */

include/linker/sections.h

@@ -75,6 +75,14 @@
 #define BOOT_NOINIT_SECTION_NAME	boot_noinit
 #endif
 
+#if defined(CONFIG_LINKER_USE_PINNED_SECTION)
+#define PINNED_TEXT_SECTION_NAME	pinned_text
+#define PINNED_BSS_SECTION_NAME		pinned_bss
+#define PINNED_RODATA_SECTION_NAME	pinned_rodata
+#define PINNED_DATA_SECTION_NAME	pinned_data
+#define PINNED_NOINIT_SECTION_NAME	pinned_noinit
+#endif
+
 /* Short section references for use in ASM files */
 #if defined(_ASMLANGUAGE)
 /* Various text section names */
@@ -100,6 +108,20 @@
 #define BOOT_NOINIT			NOINIT
 #endif /* CONFIG_LINKER_USE_BOOT_SECTION */
 
+#if defined(CONFIG_LINKER_USE_PINNED_SECTION)
+#define PINNED_TEXT			PINNED_TEXT_SECTION_NAME
+#define PINNED_BSS			PINNED_BSS_SECTION_NAME
+#define PINNED_RODATA			PINNED_RODATA_SECTION_NAME
+#define PINNED_DATA			PINNED_DATA_SECTION_NAME
+#define PINNED_NOINIT			PINNED_NOINIT_SECTION_NAME
+#else
+#define PINNED_TEXT			TEXT
+#define PINNED_BSS			BSS
+#define PINNED_RODATA			RODATA
+#define PINNED_DATA			DATA
+#define PINNED_NOINIT			NOINIT
+#endif /* CONFIG_LINKER_USE_PINNED_SECTION */
+
 #endif /* _ASMLANGUAGE */
 
 #include <linker/section_tags.h>

include/sys/thread_stack.h

@@ -175,6 +175,27 @@ static inline char *z_stack_ptr_align(char *ptr)
 #define K_KERNEL_STACK_DEFINE(sym, size) \
 	Z_KERNEL_STACK_DEFINE_IN(sym, size, __kstackmem)
 
+/**
+ * @def K_KERNEL_PINNED_STACK_DEFINE
+ * @brief Define a toplevel kernel stack memory region in pinned section
+ *
+ * See K_KERNEL_STACK_DEFINE() for more information and constraints.
+ *
+ * This puts the stack into the pinned noinit linker section if
+ * CONFIG_LINKER_USE_PINNED_SECTION is enabled, or else it would
+ * put the stack into the same section as K_KERNEL_STACK_DEFINE().
+ *
+ * @param sym Thread stack symbol name
+ * @param size Size of the stack memory region
+ */
+#if defined(CONFIG_LINKER_USE_PINNED_SECTION)
+#define K_KERNEL_PINNED_STACK_DEFINE(sym, size) \
+	Z_KERNEL_STACK_DEFINE_IN(sym, size, __pinned_noinit)
+#else
+#define K_KERNEL_PINNED_STACK_DEFINE(sym, size) \
+	Z_KERNEL_STACK_DEFINE_IN(sym, size, __kstackmem)
+#endif
+
 #define Z_KERNEL_STACK_LEN(size) \
 	ROUND_UP(Z_KERNEL_STACK_SIZE_ADJUST(size), Z_KERNEL_STACK_OBJ_ALIGN)
 
@@ -191,6 +212,28 @@ static inline char *z_stack_ptr_align(char *ptr)
 #define K_KERNEL_STACK_ARRAY_DEFINE(sym, nmemb, size) \
 	Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __kstackmem)
 
+/**
+ * @def K_KERNEL_PINNED_STACK_ARRAY_DEFINE
+ * @brief Define a toplevel array of kernel stack memory regions in pinned section
+ *
+ * See K_KERNEL_STACK_ARRAY_DEFINE() for more information and constraints.
+ *
+ * This puts the stack into the pinned noinit linker section if
+ * CONFIG_LINKER_USE_PINNED_SECTION is enabled, or else it would
+ * put the stack into the same section as K_KERNEL_STACK_ARRAY_DEFINE().
+ *
+ * @param sym Kernel stack array symbol name
+ * @param nmemb Number of stacks to declare
+ * @param size Size of the stack memory region
+ */
+#if defined(CONFIG_LINKER_USE_PINNED_SECTION)
+#define K_KERNEL_PINNED_STACK_ARRAY_DEFINE(sym, nmemb, size) \
+	Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __pinned_noinit)
+#else
+#define K_KERNEL_PINNED_STACK_ARRAY_DEFINE(sym, nmemb, size) \
+	Z_KERNEL_STACK_ARRAY_DEFINE_IN(sym, nmemb, size, __kstackmem)
+#endif
+
 /**
  * @def K_KERNEL_STACK_MEMBER
  * @brief Declare an embedded stack memory region

kernel/include/kernel_internal.h

@@ -45,6 +45,15 @@ static inline void z_bss_zero_boot(void)
 }
 #endif
 
+#ifdef CONFIG_LINKER_USE_PINNED_SECTION
+void z_bss_zero_pinned(void);
+#else
+static inline void z_bss_zero_pinned(void)
+{
+	/* Do nothing */
+}
+#endif
+
 FUNC_NORETURN void z_cstart(void);
 
 void z_device_state_init(void);

kernel/init.c

@@ -127,6 +127,28 @@ void z_bss_zero_boot(void)
 }
 #endif /* CONFIG_LINKER_USE_BOOT_SECTION */
 
+#ifdef CONFIG_LINKER_USE_PINNED_SECTION
+/**
+ * @brief Clear BSS within the pinned region
+ *
+ * This routine clears the BSS within the pinned region.
+ * This is separate from z_bss_zero() as pinned region may
+ * contain symbols required for the boot process before
+ * paging is initialized.
+ */
+#ifdef CONFIG_LINKER_USE_BOOT_SECTION
+__boot_func
+#else
+__pinned_func
+#endif
+void z_bss_zero_pinned(void)
+{
+	(void)memset(&lnkr_pinned_bss_start, 0,
+		(uintptr_t)&lnkr_pinned_bss_end
+		- (uintptr_t)&lnkr_pinned_bss_start);
+}
+#endif /* CONFIG_LINKER_USE_PINNED_SECTION */
+
 #ifdef CONFIG_STACK_CANARIES
 extern volatile uintptr_t __stack_chk_guard;
 #endif /* CONFIG_STACK_CANARIES */

kernel/mmu.c

@@ -761,6 +761,23 @@ void z_mem_manage_init(void)
 		pf->flags |= Z_PAGE_FRAME_PINNED;
 	}
 
+#ifdef CONFIG_LINKER_USE_PINNED_SECTION
+	/* Pin the page frames correspondng to the pinned symbols */
+	uintptr_t pinned_start = ROUND_DOWN(POINTER_TO_UINT(lnkr_pinned_start),
+					    CONFIG_MMU_PAGE_SIZE);
+	uintptr_t pinned_end = ROUND_UP(POINTER_TO_UINT(lnkr_pinned_end),
+					CONFIG_MMU_PAGE_SIZE);
+	size_t pinned_size = pinned_end - pinned_start;
+
+	VIRT_FOREACH(UINT_TO_POINTER(pinned_start), pinned_size, addr)
+	{
+		pf = z_phys_to_page_frame(Z_BOOT_VIRT_TO_PHYS(addr));
+		frame_mapped_set(pf, addr);
+
+		pf->flags |= Z_PAGE_FRAME_PINNED;
+	}
+#endif
+
 	/* Any remaining pages that aren't mapped, reserved, or pinned get
 	 * added to the free pages list
 	 */