From dd04ee5ce4699ae0c41ea6fb50c6ed8141094b1b Mon Sep 17 00:00:00 2001 From: Youjie Zheng Date: Thu, 26 Sep 2024 14:39:19 +0845 Subject: [PATCH] [hal] Support user space for riscv64 and aarch64 --- modules/axhal/src/arch/aarch64/context.rs | 169 +++++++++++++++++++++- modules/axhal/src/arch/aarch64/mod.rs | 2 + modules/axhal/src/arch/aarch64/trap.S | 5 + modules/axhal/src/arch/aarch64/trap.rs | 5 +- modules/axhal/src/arch/riscv/context.rs | 166 ++++++++++++++++++++- modules/axhal/src/arch/riscv/mod.rs | 2 + modules/axhal/src/arch/riscv/trap.S | 12 +- modules/axhal/src/arch/riscv/trap.rs | 5 + modules/axhal/src/arch/x86_64/context.rs | 8 + modules/axhal/src/cpu.rs | 16 +- 10 files changed, 373 insertions(+), 17 deletions(-) diff --git a/modules/axhal/src/arch/aarch64/context.rs b/modules/axhal/src/arch/aarch64/context.rs index 498ff282be..29c2561cbb 100644 --- a/modules/axhal/src/arch/aarch64/context.rs +++ b/modules/axhal/src/arch/aarch64/context.rs @@ -1,5 +1,7 @@ +#![allow(unused_imports)] + use core::arch::asm; -use memory_addr::VirtAddr; +use memory_addr::{PhysAddr, VirtAddr}; /// Saved registers when a trap (exception) occurs. #[repr(C)] @@ -15,6 +17,147 @@ pub struct TrapFrame { pub spsr: u64, } +impl TrapFrame { + /// Gets the 0th syscall argument. + pub const fn arg0(&self) -> usize { + self.r[0] as _ + } + + /// Gets the 1st syscall argument. + pub const fn arg1(&self) -> usize { + self.r[1] as _ + } + + /// Gets the 2nd syscall argument. + pub const fn arg2(&self) -> usize { + self.r[2] as _ + } + + /// Gets the 3rd syscall argument. + pub const fn arg3(&self) -> usize { + self.r[3] as _ + } + + /// Gets the 4th syscall argument. + pub const fn arg4(&self) -> usize { + self.r[4] as _ + } + + /// Gets the 5th syscall argument. + pub const fn arg5(&self) -> usize { + self.r[5] as _ + } +} + +/// Context to enter user space. +#[cfg(feature = "uspace")] +pub struct UspaceContext(TrapFrame); + +#[cfg(feature = "uspace")] +impl UspaceContext { + /// Creates an empty context with all registers set to zero. + pub const fn empty() -> Self { + unsafe { core::mem::MaybeUninit::zeroed().assume_init() } + } + + /// Creates a new context with the given entry point, user stack pointer, + /// and the argument. + pub fn new(entry: usize, ustack_top: VirtAddr, arg0: usize) -> Self { + use aarch64_cpu::registers::SPSR_EL1; + let mut regs = [0; 31]; + regs[0] = arg0 as _; + Self(TrapFrame { + r: regs, + usp: ustack_top.as_usize() as _, + elr: entry as _, + spsr: (SPSR_EL1::M::EL0t + + SPSR_EL1::D::Masked + + SPSR_EL1::A::Masked + + SPSR_EL1::I::Unmasked + + SPSR_EL1::F::Masked) + .value, + }) + } + + /// Creates a new context from the given [`TrapFrame`]. + pub const fn from(trap_frame: &TrapFrame) -> Self { + Self(*trap_frame) + } + + /// Gets the instruction pointer. + pub const fn get_ip(&self) -> usize { + self.0.elr as _ + } + + /// Gets the stack pointer. + pub const fn get_sp(&self) -> usize { + self.0.usp as _ + } + + /// Sets the instruction pointer. + pub const fn set_ip(&mut self, pc: usize) { + self.0.elr = pc as _; + } + + /// Sets the stack pointer. + pub const fn set_sp(&mut self, sp: usize) { + self.0.usp = sp as _; + } + + /// Sets the return value register. + pub const fn set_retval(&mut self, r0: usize) { + self.0.r[0] = r0 as _; + } + + /// Enters user space. + /// + /// It restores the user registers and jumps to the user entry point + /// (saved in `elr`). + /// When an exception or syscall occurs, the kernel stack pointer is + /// switched to `kstack_top`. + /// + /// # Safety + /// + /// This function is unsafe because it changes processor mode and the stack. + #[inline(never)] + #[no_mangle] + pub unsafe fn enter_uspace(&self, kstack_top: VirtAddr) -> ! { + super::disable_irqs(); + // We do not handle traps that occur at the current exception level, + // so the kstack ptr(`sp_el1`) will not change during running in user space. + // Then we don't need to save the `sp_el1` to the taskctx. + asm!( + " + mov sp, x1 + ldp x30, x9, [x0, 30 * 8] + ldp x10, x11, [x0, 32 * 8] + msr sp_el0, x9 + msr elr_el1, x10 + msr spsr_el1, x11 + + ldp x28, x29, [x0, 28 * 8] + ldp x26, x27, [x0, 26 * 8] + ldp x24, x25, [x0, 24 * 8] + ldp x22, x23, [x0, 22 * 8] + ldp x20, x21, [x0, 20 * 8] + ldp x18, x19, [x0, 18 * 8] + ldp x16, x17, [x0, 16 * 8] + ldp x14, x15, [x0, 14 * 8] + ldp x12, x13, [x0, 12 * 8] + ldp x10, x11, [x0, 10 * 8] + ldp x8, x9, [x0, 8 * 8] + ldp x6, x7, [x0, 6 * 8] + ldp x4, x5, [x0, 4 * 8] + ldp x2, x3, [x0, 2 * 8] + ldp x0, x1, [x0] + eret", + in("x0") &self.0, + in("x1") kstack_top.as_usize() , + options(noreturn), + ) + } +} + /// FP & SIMD registers. #[repr(C, align(16))] #[derive(Debug, Default)] @@ -47,7 +190,7 @@ impl FpState { /// and the next task restores its context from memory to CPU. #[allow(missing_docs)] #[repr(C)] -#[derive(Debug)] +#[derive(Debug, Default)] pub struct TaskContext { pub sp: u64, pub tpidr_el0: u64, @@ -63,6 +206,9 @@ pub struct TaskContext { pub r28: u64, pub r29: u64, pub lr: u64, // r30 + /// The `ttbr0_el1` register value, i.e., the page table root. + #[cfg(feature = "uspace")] + pub ttbr0_el1: PhysAddr, #[cfg(feature = "fp_simd")] pub fp_state: FpState, } @@ -75,8 +221,8 @@ impl TaskContext { /// /// [`init`]: TaskContext::init /// [`switch_to`]: TaskContext::switch_to - pub const fn new() -> Self { - unsafe { core::mem::MaybeUninit::zeroed().assume_init() } + pub fn new() -> Self { + Self::default() } /// Initializes the context for a new task, with the given entry point and @@ -84,9 +230,18 @@ impl TaskContext { pub fn init(&mut self, entry: usize, kstack_top: VirtAddr, tls_area: VirtAddr) { self.sp = kstack_top.as_usize() as u64; self.lr = entry as u64; + // When under `uspace` feature, kernel will not use this register. self.tpidr_el0 = tls_area.as_usize() as u64; } + /// Changes the page table root for user space (`ttbr0_el1` register for aarch64 in el1 level). + /// + /// If not set, it means that this task is a kernel task and only `ttbr1_el1` register will be used. + #[cfg(feature = "uspace")] + pub fn set_page_table_root(&mut self, ttbr0_el1: PhysAddr) { + self.ttbr0_el1 = ttbr0_el1; + } + /// Switches to another task. /// /// It first saves the current task's context from CPU to this place, and then @@ -94,6 +249,12 @@ impl TaskContext { pub fn switch_to(&mut self, next_ctx: &Self) { #[cfg(feature = "fp_simd")] self.fp_state.switch_to(&next_ctx.fp_state); + #[cfg(feature = "uspace")] + { + if self.ttbr0_el1 != next_ctx.ttbr0_el1 { + unsafe { super::write_page_table_root0(next_ctx.ttbr0_el1) }; + } + } unsafe { context_switch(self, next_ctx) } } } diff --git a/modules/axhal/src/arch/aarch64/mod.rs b/modules/axhal/src/arch/aarch64/mod.rs index 1907528af1..d3be7e73f0 100644 --- a/modules/axhal/src/arch/aarch64/mod.rs +++ b/modules/axhal/src/arch/aarch64/mod.rs @@ -7,6 +7,8 @@ use aarch64_cpu::registers::{DAIF, TPIDR_EL0, TTBR0_EL1, TTBR1_EL1, VBAR_EL1}; use memory_addr::{PhysAddr, VirtAddr}; use tock_registers::interfaces::{Readable, Writeable}; +#[cfg(feature = "uspace")] +pub use self::context::UspaceContext; pub use self::context::{FpState, TaskContext, TrapFrame}; /// Allows the current CPU to respond to interrupts. diff --git a/modules/axhal/src/arch/aarch64/trap.S b/modules/axhal/src/arch/aarch64/trap.S index 7167610acb..662c0888d1 100644 --- a/modules/axhal/src/arch/aarch64/trap.S +++ b/modules/axhal/src/arch/aarch64/trap.S @@ -21,6 +21,11 @@ mrs x11, spsr_el1 stp x30, x9, [sp, 30 * 8] stp x10, x11, [sp, 32 * 8] + + # We may have interrupted userspace, or a guest, or exit-from or + # return-to either of those. So we can't trust sp_el0, and need to + # restore it. + bl {cache_current_task_ptr} .endm .macro RESTORE_REGS diff --git a/modules/axhal/src/arch/aarch64/trap.rs b/modules/axhal/src/arch/aarch64/trap.rs index 9bee97474e..0b1a67a7f8 100644 --- a/modules/axhal/src/arch/aarch64/trap.rs +++ b/modules/axhal/src/arch/aarch64/trap.rs @@ -6,7 +6,7 @@ use tock_registers::interfaces::Readable; use super::TrapFrame; -global_asm!(include_str!("trap.S")); +global_asm!(include_str!("trap.S"), cache_current_task_ptr = sym crate::cpu::cache_current_task_ptr); #[repr(u8)] #[derive(Debug)] @@ -98,8 +98,9 @@ fn handle_sync_exception(tf: &mut TrapFrame) { let esr = ESR_EL1.extract(); let iss = esr.read(ESR_EL1::ISS); match esr.read_as_enum(ESR_EL1::EC) { + #[cfg(feature = "uspace")] Some(ESR_EL1::EC::Value::SVC64) => { - warn!("No syscall is supported currently!"); + tf.r[0] = crate::trap::handle_syscall(tf, tf.r[8] as usize) as u64; } Some(ESR_EL1::EC::Value::InstrAbortLowerEL) => handle_instruction_abort(tf, iss, true), Some(ESR_EL1::EC::Value::InstrAbortCurrentEL) => handle_instruction_abort(tf, iss, false), diff --git a/modules/axhal/src/arch/riscv/context.rs b/modules/axhal/src/arch/riscv/context.rs index cb50a3415a..c004f685ec 100644 --- a/modules/axhal/src/arch/riscv/context.rs +++ b/modules/axhal/src/arch/riscv/context.rs @@ -1,12 +1,14 @@ +#![allow(unused_imports)] + use core::arch::asm; -use memory_addr::VirtAddr; +use memory_addr::{PhysAddr, VirtAddr}; include_asm_marcos!(); /// General registers of RISC-V. #[allow(missing_docs)] #[repr(C)] -#[derive(Debug, Default, Clone)] +#[derive(Debug, Default, Clone, Copy)] pub struct GeneralRegisters { pub ra: usize, pub sp: usize, @@ -43,7 +45,7 @@ pub struct GeneralRegisters { /// Saved registers when a trap (interrupt or exception) occurs. #[repr(C)] -#[derive(Debug, Default, Clone)] +#[derive(Debug, Default, Clone, Copy)] pub struct TrapFrame { /// All general registers. pub regs: GeneralRegisters, @@ -53,6 +55,136 @@ pub struct TrapFrame { pub sstatus: usize, } +impl TrapFrame { + /// Gets the 0th syscall argument. + pub const fn arg0(&self) -> usize { + self.regs.a0 + } + + /// Gets the 1st syscall argument. + pub const fn arg1(&self) -> usize { + self.regs.a1 + } + + /// Gets the 2nd syscall argument. + pub const fn arg2(&self) -> usize { + self.regs.a2 + } + + /// Gets the 3rd syscall argument. + pub const fn arg3(&self) -> usize { + self.regs.a3 + } + + /// Gets the 4th syscall argument. + pub const fn arg4(&self) -> usize { + self.regs.a4 + } + + /// Gets the 5th syscall argument. + pub const fn arg5(&self) -> usize { + self.regs.a5 + } +} + +/// Context to enter user space. +#[cfg(feature = "uspace")] +pub struct UspaceContext(TrapFrame); + +#[cfg(feature = "uspace")] +impl UspaceContext { + /// Creates an empty context with all registers set to zero. + pub const fn empty() -> Self { + unsafe { core::mem::MaybeUninit::zeroed().assume_init() } + } + + /// Creates a new context with the given entry point, user stack pointer, + /// and the argument. + pub fn new(entry: usize, ustack_top: VirtAddr, arg0: usize) -> Self { + const SPIE: usize = 1 << 5; + const SUM: usize = 1 << 18; + Self(TrapFrame { + regs: GeneralRegisters { + a0: arg0, + sp: ustack_top.as_usize(), + ..Default::default() + }, + sepc: entry, + sstatus: SPIE | SUM, + }) + } + + /// Creates a new context from the given [`TrapFrame`]. + pub const fn from(trap_frame: &TrapFrame) -> Self { + Self(*trap_frame) + } + + /// Gets the instruction pointer. + pub const fn get_ip(&self) -> usize { + self.0.sepc + } + + /// Gets the stack pointer. + pub const fn get_sp(&self) -> usize { + self.0.regs.sp + } + + /// Sets the instruction pointer. + pub const fn set_ip(&mut self, pc: usize) { + self.0.sepc = pc; + } + + /// Sets the stack pointer. + pub const fn set_sp(&mut self, sp: usize) { + self.0.regs.sp = sp; + } + + /// Sets the return value register. + pub const fn set_retval(&mut self, a0: usize) { + self.0.regs.a0 = a0; + } + + /// Enters user space. + /// + /// It restores the user registers and jumps to the user entry point + /// (saved in `sepc`). + /// When an exception or syscall occurs, the kernel stack pointer is + /// switched to `kstack_top`. + /// + /// # Safety + /// + /// This function is unsafe because it changes processor mode and the stack. + #[inline(never)] + #[no_mangle] + pub unsafe fn enter_uspace(&self, kstack_top: VirtAddr) -> ! { + use riscv::register::{sepc, sscratch}; + + super::disable_irqs(); + sscratch::write(kstack_top.as_usize()); + sepc::write(self.0.sepc); + // Address of the top of the kernel stack after saving the trap frame. + let kernel_trap_addr = kstack_top.as_usize() - core::mem::size_of::(); + asm!(" + mv sp, {tf} + + STR gp, {kernel_trap_addr}, 2 + LDR gp, sp, 2 + + STR tp, {kernel_trap_addr}, 3 + LDR tp, sp, 3 + + LDR t0, sp, 32 + csrw sstatus, t0 + POP_GENERAL_REGS + LDR sp, sp, 1 + sret", + tf = in(reg) &(self.0), + kernel_trap_addr = in(reg) kernel_trap_addr, + options(noreturn), + ) + } +} + /// Saved hardware states of a task. /// /// The context usually includes: @@ -86,6 +218,9 @@ pub struct TaskContext { pub s11: usize, pub tp: usize, + /// The `satp` register value, i.e., the page table root. + #[cfg(feature = "uspace")] + pub satp: PhysAddr, // TODO: FP states } @@ -97,8 +232,12 @@ impl TaskContext { /// /// [`init`]: TaskContext::init /// [`switch_to`]: TaskContext::switch_to - pub const fn new() -> Self { - unsafe { core::mem::MaybeUninit::zeroed().assume_init() } + pub fn new() -> Self { + Self { + #[cfg(feature = "uspace")] + satp: crate::paging::kernel_page_table_root(), + ..Default::default() + } } /// Initializes the context for a new task, with the given entry point and @@ -109,6 +248,17 @@ impl TaskContext { self.tp = tls_area.as_usize(); } + /// Changes the page table root (`satp` register for riscv64). + /// + /// If not set, the kernel page table root is used (obtained by + /// [`axhal::paging::kernel_page_table_root`][1]). + /// + /// [1]: crate::paging::kernel_page_table_root + #[cfg(feature = "uspace")] + pub fn set_page_table_root(&mut self, satp: PhysAddr) { + self.satp = satp; + } + /// Switches to another task. /// /// It first saves the current task's context from CPU to this place, and then @@ -119,6 +269,12 @@ impl TaskContext { self.tp = super::read_thread_pointer(); unsafe { super::write_thread_pointer(next_ctx.tp) }; } + #[cfg(feature = "uspace")] + unsafe { + if self.satp != next_ctx.satp { + super::write_page_table_root(next_ctx.satp); + } + } unsafe { // TODO: switch FP states context_switch(self, next_ctx) diff --git a/modules/axhal/src/arch/riscv/mod.rs b/modules/axhal/src/arch/riscv/mod.rs index eb3bc82278..36ce91da41 100644 --- a/modules/axhal/src/arch/riscv/mod.rs +++ b/modules/axhal/src/arch/riscv/mod.rs @@ -8,6 +8,8 @@ use memory_addr::{PhysAddr, VirtAddr}; use riscv::asm; use riscv::register::{satp, sstatus, stvec}; +#[cfg(feature = "uspace")] +pub use self::context::UspaceContext; pub use self::context::{GeneralRegisters, TaskContext, TrapFrame}; /// Allows the current CPU to respond to interrupts. diff --git a/modules/axhal/src/arch/riscv/trap.S b/modules/axhal/src/arch/riscv/trap.S index 5d19dfe76a..8a594b8ada 100644 --- a/modules/axhal/src/arch/riscv/trap.S +++ b/modules/axhal/src/arch/riscv/trap.S @@ -10,18 +10,22 @@ STR t2, sp, 1 // tf.regs.sp .if \from_user == 1 - LDR t0, sp, 3 // load supervisor tp + LDR t0, sp, 2 // load supervisor gp + LDR t1, sp, 3 // load supervisor tp STR gp, sp, 2 // save user gp and tp STR tp, sp, 3 - mv tp, t0 + mv gp, t0 + mv tp, t1 .endif .endm .macro RESTORE_REGS, from_user .if \from_user == 1 - LDR gp, sp, 2 // load user gp and tp + LDR t1, sp, 2 // load user gp and tp LDR t0, sp, 3 - STR tp, sp, 3 // save supervisor tp + STR gp, sp, 2 // save supervisor gp + STR tp, sp, 3 // save supervisor gp and tp + mv gp, t1 mv tp, t0 addi t0, sp, {trapframe_size} // put supervisor sp to scratch csrw sscratch, t0 diff --git a/modules/axhal/src/arch/riscv/trap.rs b/modules/axhal/src/arch/riscv/trap.rs index 5a7a623062..59a0f938b5 100644 --- a/modules/axhal/src/arch/riscv/trap.rs +++ b/modules/axhal/src/arch/riscv/trap.rs @@ -37,6 +37,11 @@ fn handle_page_fault(tf: &TrapFrame, mut access_flags: MappingFlags, is_user: bo fn riscv_trap_handler(tf: &mut TrapFrame, from_user: bool) { let scause = scause::read(); match scause.cause() { + #[cfg(feature = "uspace")] + Trap::Exception(E::UserEnvCall) => { + tf.regs.a0 = crate::trap::handle_syscall(tf, tf.regs.a7) as usize; + tf.sepc += 4; + } Trap::Exception(E::LoadPageFault) => handle_page_fault(tf, MappingFlags::READ, from_user), Trap::Exception(E::StorePageFault) => handle_page_fault(tf, MappingFlags::WRITE, from_user), Trap::Exception(E::InstructionPageFault) => { diff --git a/modules/axhal/src/arch/x86_64/context.rs b/modules/axhal/src/arch/x86_64/context.rs index ef9b29c050..dd49d8212e 100644 --- a/modules/axhal/src/arch/x86_64/context.rs +++ b/modules/axhal/src/arch/x86_64/context.rs @@ -274,6 +274,9 @@ pub struct TaskContext { pub rsp: u64, /// Thread Local Storage (TLS). pub fs_base: usize, + /// The `gs_base` register value. + #[cfg(feature = "uspace")] + pub gs_base: usize, /// Extended states, i.e., FP/SIMD states. #[cfg(feature = "fp_simd")] pub ext_state: ExtendedState, @@ -299,6 +302,8 @@ impl TaskContext { cr3: crate::paging::kernel_page_table_root(), #[cfg(feature = "fp_simd")] ext_state: ExtendedState::default(), + #[cfg(feature = "uspace")] + gs_base: 0, } } @@ -352,6 +357,9 @@ impl TaskContext { } #[cfg(feature = "uspace")] unsafe { + // Switch gs base for user space. + self.gs_base = x86::msr::rdmsr(x86::msr::IA32_KERNEL_GSBASE) as usize; + x86::msr::wrmsr(x86::msr::IA32_KERNEL_GSBASE, next_ctx.gs_base as u64); super::tss_set_rsp0(next_ctx.kstack_top); if next_ctx.cr3 != self.cr3 { super::write_page_table_root(next_ctx.cr3); diff --git a/modules/axhal/src/cpu.rs b/modules/axhal/src/cpu.rs index 7a874d4801..5f970182da 100644 --- a/modules/axhal/src/cpu.rs +++ b/modules/axhal/src/cpu.rs @@ -22,6 +22,16 @@ pub fn this_cpu_is_bsp() -> bool { IS_BSP.read_current() } +/// Stores the pointer to the current task in the SP_EL0 register. +/// +/// In aarch64 architecture, we use `SP_EL0` as the read cache for +/// the current task pointer. And this function will update this cache. +#[cfg(target_arch = "aarch64")] +pub(crate) unsafe fn cache_current_task_ptr() { + use tock_registers::interfaces::Writeable; + aarch64_cpu::registers::SP_EL0.set(CURRENT_TASK_PTR.read_current_raw() as u64); +} + /// Gets the pointer to the current task with preemption-safety. /// /// Preemption may be enabled when calling this function. This function will @@ -42,6 +52,7 @@ pub fn current_task_ptr() -> *const T { #[cfg(target_arch = "aarch64")] { // on ARM64, we use `SP_EL0` to store the task pointer. + // `SP_EL0` is equivalent to the cache of CURRENT_TASK_PTR here. use tock_registers::interfaces::Readable; aarch64_cpu::registers::SP_EL0.get() as _ } @@ -68,8 +79,9 @@ pub unsafe fn set_current_task_ptr(ptr: *const T) { } #[cfg(target_arch = "aarch64")] { - use tock_registers::interfaces::Writeable; - aarch64_cpu::registers::SP_EL0.set(ptr as u64) + let _guard = kernel_guard::IrqSave::new(); + CURRENT_TASK_PTR.write_current_raw(ptr as usize); + cache_current_task_ptr(); } }