a1b77fd589
git grep -l 'u\(8\|16\|32\|64\)_t' | \ xargs sed -i "s/u\(8\|16\|32\|64\)_t/uint\1_t/g" git grep -l 's\(8\|16\|32\|64\)_t' | \ xargs sed -i "s/s\(8\|16\|32\|64\)_t/int\1_t/g" Signed-off-by: Kumar Gala <kumar.gala@linaro.org>
139 lines
4.2 KiB
C
139 lines
4.2 KiB
C
/*
|
|
* Copyright (c) 2017 Intel Corporation
|
|
*
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
#include <kernel.h>
|
|
#include <sys/speculation.h>
|
|
#include <syscall_handler.h>
|
|
#include <kernel_arch_func.h>
|
|
#include <ksched.h>
|
|
|
|
#ifndef CONFIG_X86_KPTI
|
|
/* Change to new set of page tables. ONLY intended for use from
|
|
* z_x88_swap_update_page_tables(). This changes CR3, no memory access
|
|
* afterwards is legal unless it is known for sure that the relevant
|
|
* mappings are identical wrt supervisor mode until we iret out.
|
|
*/
|
|
static inline void page_tables_set(struct x86_page_tables *ptables)
|
|
{
|
|
#ifdef CONFIG_X86_64
|
|
__asm__ volatile("movq %0, %%cr3\n\t" : : "r" (ptables) : "memory");
|
|
#else
|
|
__asm__ volatile("movl %0, %%cr3\n\t" : : "r" (ptables) : "memory");
|
|
#endif
|
|
}
|
|
|
|
/* Update the to the incoming thread's page table, and update the location of
|
|
* the privilege elevation stack.
|
|
*
|
|
* May be called ONLY during context switch and when supervisor threads drop
|
|
* synchronously to user mode. Hot code path!
|
|
*
|
|
* Nothing to do here if KPTI is enabled. We are in supervisor mode, so the
|
|
* active page tables are the kernel's page tables. If the incoming thread is
|
|
* in user mode we are going to switch CR3 to the thread-specific tables when
|
|
* we go through z_x86_trampoline_to_user.
|
|
*
|
|
* We don't need to update the privilege mode initial stack pointer either,
|
|
* privilege elevation always lands on the trampoline stack and the irq/sycall
|
|
* code has to manually transition off of it to the appropriate stack after
|
|
* switching page tables.
|
|
*/
|
|
void z_x86_swap_update_page_tables(struct k_thread *incoming)
|
|
{
|
|
struct x86_page_tables *ptables;
|
|
|
|
#ifndef CONFIG_X86_64
|
|
/* 64-bit uses syscall/sysret which switches stacks manually,
|
|
* tss64.psp is updated unconditionally in __resume
|
|
*/
|
|
if ((incoming->base.user_options & K_USER) != 0) {
|
|
_main_tss.esp0 = (uintptr_t)incoming->arch.psp;
|
|
}
|
|
#endif
|
|
|
|
/* Check first that we actually need to do this, since setting
|
|
* CR3 involves an expensive full TLB flush.
|
|
*/
|
|
ptables = z_x86_thread_page_tables_get(incoming);
|
|
|
|
if (ptables != z_x86_page_tables_get()) {
|
|
page_tables_set(ptables);
|
|
}
|
|
}
|
|
#endif /* CONFIG_X86_KPTI */
|
|
|
|
FUNC_NORETURN static void drop_to_user(k_thread_entry_t user_entry,
|
|
void *p1, void *p2, void *p3)
|
|
{
|
|
uint32_t stack_end;
|
|
|
|
/* Transition will reset stack pointer to initial, discarding
|
|
* any old context since this is a one-way operation
|
|
*/
|
|
stack_end = Z_STACK_PTR_ALIGN(_current->stack_info.start +
|
|
_current->stack_info.size);
|
|
|
|
z_x86_userspace_enter(user_entry, p1, p2, p3, stack_end,
|
|
_current->stack_info.start);
|
|
CODE_UNREACHABLE;
|
|
}
|
|
|
|
/* Preparation steps needed for all threads if user mode is turned on.
|
|
*
|
|
* Returns the initial entry point to swap into.
|
|
*/
|
|
void *z_x86_userspace_prepare_thread(struct k_thread *thread)
|
|
{
|
|
void *initial_entry;
|
|
struct z_x86_thread_stack_header *header =
|
|
(struct z_x86_thread_stack_header *)thread->stack_obj;
|
|
|
|
thread->arch.psp =
|
|
header->privilege_stack + sizeof(header->privilege_stack);
|
|
|
|
if ((thread->base.user_options & K_USER) != 0U) {
|
|
z_x86_thread_pt_init(thread);
|
|
initial_entry = drop_to_user;
|
|
} else {
|
|
thread->arch.ptables = &z_x86_kernel_ptables;
|
|
initial_entry = z_thread_entry;
|
|
}
|
|
|
|
return initial_entry;
|
|
}
|
|
|
|
FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry,
|
|
void *p1, void *p2, void *p3)
|
|
{
|
|
z_x86_thread_pt_init(_current);
|
|
|
|
/* Apply memory domain configuration, if assigned. Threads that
|
|
* started in user mode already had this done via z_setup_new_thread()
|
|
*/
|
|
if (_current->mem_domain_info.mem_domain != NULL) {
|
|
z_x86_apply_mem_domain(_current->arch.ptables,
|
|
_current->mem_domain_info.mem_domain);
|
|
}
|
|
|
|
#ifndef CONFIG_X86_KPTI
|
|
/* We're synchronously dropping into user mode from a thread that
|
|
* used to be in supervisor mode. K_USER flag has now been set, but
|
|
* Need to swap from the kernel's page tables to the per-thread page
|
|
* tables.
|
|
*
|
|
* Safe to update page tables from here, all tables are identity-
|
|
* mapped and memory areas used before the ring 3 transition all
|
|
* have the same attributes wrt supervisor mode access.
|
|
*
|
|
* Threads that started in user mode already had this applied on
|
|
* initial context switch.
|
|
*/
|
|
z_x86_swap_update_page_tables(_current);
|
|
#endif
|
|
|
|
drop_to_user(user_entry, p1, p2, p3);
|
|
}
|