diff --git a/example/linker/linker-aarch64.ld b/example/linker/linker-aarch64.ld index fac9f92..0e8fa89 100644 --- a/example/linker/linker-aarch64.ld +++ b/example/linker/linker-aarch64.ld @@ -48,20 +48,6 @@ SECTIONS _ebss = .; } - . = ALIGN(4K); - _percpu_start = .; - .percpu 0x0 : AT(_percpu_start) { - _percpu_load_start = .; - *(.percpu .percpu.*) - _percpu_load_end = .; - . = ALIGN(64); - _percpu_size_aligned = .; - - . = _percpu_load_start + _percpu_size_aligned * 1; - } - . = _percpu_start + SIZEOF(.percpu); - _percpu_end = .; - PROVIDE(end = .); /DISCARD/ : { *(.comment) *(.gnu*) *(.note*) *(.eh_frame*) diff --git a/example/linker/linker-loongarch64.ld b/example/linker/linker-loongarch64.ld index 7f7f2c1..b60f284 100644 --- a/example/linker/linker-loongarch64.ld +++ b/example/linker/linker-loongarch64.ld @@ -48,20 +48,6 @@ SECTIONS _ebss = .; } - . = ALIGN(4K); - _percpu_start = .; - .percpu 0x0 : AT(_percpu_start) { - _percpu_load_start = .; - *(.percpu .percpu.*) - _percpu_load_end = .; - . = ALIGN(64); - _percpu_size_aligned = .; - - . = _percpu_load_start + _percpu_size_aligned * 1; - } - . = _percpu_start + SIZEOF(.percpu); - _percpu_end = .; - PROVIDE(end = .); /DISCARD/ : { *(.comment) *(.gnu*) *(.note*) *(.eh_frame*) diff --git a/example/linker/linker-x86_64.ld b/example/linker/linker-x86_64.ld index da3bc80..785efd1 100644 --- a/example/linker/linker-x86_64.ld +++ b/example/linker/linker-x86_64.ld @@ -48,20 +48,6 @@ SECTIONS _ebss = .; } - . = ALIGN(4K); - _percpu_start = .; - .percpu 0x0 : AT(_percpu_start) { - _percpu_load_start = .; - *(.percpu .percpu.*) - _percpu_load_end = .; - . = ALIGN(64); - _percpu_size_aligned = .; - - . = _percpu_load_start + _percpu_size_aligned * 1; - } - . = _percpu_start + SIZEOF(.percpu); - _percpu_end = .; - PROVIDE(end = .); /DISCARD/ : { *(.comment) *(.gnu*) *(.note*) *(.eh_frame*) diff --git a/polyhal-macro/src/lib.rs b/polyhal-macro/src/lib.rs index 87fda76..9e8ca9f 100644 --- a/polyhal-macro/src/lib.rs +++ b/polyhal-macro/src/lib.rs @@ -4,6 +4,7 @@ use proc_macro::TokenStream; use proc_macro2::Span; use quote::{format_ident, quote}; use syn::{parse_macro_input, Error, ItemFn, ItemStatic}; +use percpu::PERCPU_RESERVED; #[proc_macro_attribute] pub fn arch_entry(_input: TokenStream, annotated_item: TokenStream) -> TokenStream { @@ -111,6 +112,11 @@ pub fn def_percpu(attr: TokenStream, item: TokenStream) -> TokenStream { extern "Rust" { fn __start_percpu(); } + #[cfg(target_arch = "x86_64")] + unsafe { + &#inner_symbol_name as *const _ as usize - __start_percpu as usize + #PERCPU_RESERVED + } + #[cfg(not(target_arch = "x86_64"))] unsafe { &#inner_symbol_name as *const _ as usize - __start_percpu as usize } @@ -162,15 +168,6 @@ pub fn def_percpu(attr: TokenStream, item: TokenStream) -> TokenStream { .into() } -#[doc(hidden)] -#[proc_macro] -pub fn percpu_symbol_offset(item: TokenStream) -> TokenStream { - let symbol = &format_ident!("{}", item.to_string()); - let offset = percpu::gen_offset(symbol); - quote!({ #offset }).into() -} - - fn compiler_error(err: Error) -> TokenStream { err.to_compile_error().into() } diff --git a/polyhal-macro/src/percpu.rs b/polyhal-macro/src/percpu.rs index 44f218c..8ceceeb 100644 --- a/polyhal-macro/src/percpu.rs +++ b/polyhal-macro/src/percpu.rs @@ -1,69 +1,27 @@ +use std::mem::size_of; + use quote::quote; use syn::{Ident, Type}; -pub fn gen_offset(symbol: &Ident) -> proc_macro2::TokenStream { - quote! { - let value: usize; - unsafe { - cfg_match! { - cfg(target_arch = "x86_64") => { - ::core::arch::asm!( - "movabs {0}, offset {VAR}", - out(reg) value, - VAR = sym #symbol, - ); - } - cfg(target_arch = "aarch64") => { - ::core::arch::asm!( - "movz {0}, #:abs_g0_nc:{VAR}", - out(reg) value, - VAR = sym #symbol, - ); - } - cfg(any(target_arch = "riscv32", target_arch = "riscv64")) => { - ::core::arch::asm!( - "lui {0}, %hi({VAR})", - "addi {0}, {0}, %lo({VAR})", - out(reg) value, - VAR = sym #symbol, - ); - } - cfg(target_arch = "loongarch64") => { - ::core::arch::asm!( - "la.abs {0}, {VAR}", - out(reg) value, - VAR = sym #symbol, - ); - } - } - } - value - } -} +/// Reserved for default usage, Just for x86_64 now. +pub const PERCPU_RESERVED: usize = 4 * size_of::(); -pub fn gen_current_ptr(symbol: &Ident, ty: &Type) -> proc_macro2::TokenStream { +pub fn gen_current_ptr(_symbol: &Ident, ty: &Type) -> proc_macro2::TokenStream { + // TODO: Move this consts into polyhal crate. quote! { let base: usize; + // The first index is SELF_PTR in the x86_64 format #[cfg(target_arch = "x86_64")] - { - // `__PERCPU_SELF_PTR` stores GS_BASE, which is defined in crate `percpu`. - ::core::arch::asm!( - "mov {0}, gs:[offset __PERCPU_SELF_PTR]", - "add {0}, offset {VAR}", - out(reg) base, - VAR = sym #symbol, - ); - base as *const #ty - } - #[cfg(not(target_arch = "x86_64"))] - { - #[cfg(target_arch = "aarch64")] - ::core::arch::asm!("mrs {}, TPIDR_EL1", out(reg) base); - #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] - ::core::arch::asm!("mv {}, gp", out(reg) base); - #[cfg(target_arch = "loongarch64")] - ::core::arch::asm!("move {}, $r21", out(reg) base); - (base + self.offset()) as *const #ty - } + ::core::arch::asm!( + "mov {0}, gs:0", + out(reg) base, + ); + #[cfg(target_arch = "aarch64")] + ::core::arch::asm!("mrs {}, TPIDR_EL1", out(reg) base); + #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] + ::core::arch::asm!("mv {}, gp", out(reg) base); + #[cfg(target_arch = "loongarch64")] + ::core::arch::asm!("move {}, $r21", out(reg) base); + (base + self.offset()) as *const #ty } } diff --git a/src/aarch64/mod.rs b/src/aarch64/mod.rs index b0abd17..92e84d4 100644 --- a/src/aarch64/mod.rs +++ b/src/aarch64/mod.rs @@ -25,6 +25,7 @@ use fdt::Fdt; pub use kcontext::{context_switch, context_switch_pt, read_current_tp, KContext}; pub use page_table::*; +use polyhal_macro::def_percpu; pub use psci::system_off as shutdown; pub use trap::run_user_task; @@ -32,12 +33,17 @@ use crate::debug::{display_info, println}; use crate::multicore::MultiCore; use crate::once::LazyInit; use crate::pagetable::PageTable; +use crate::percpu::percpu_area_init; use crate::{clear_bss, CPU_NUM, DTB_BIN, MEM_AREA}; static DTB_PTR: LazyInit = LazyInit::new(); +#[def_percpu] +static CPU_ID: usize = 0; + pub fn rust_tmp_main(hart_id: usize, device_tree: usize) { clear_bss(); + percpu_area_init(hart_id); pl011::init_early(); trap::init(); gic::init(); diff --git a/src/debug.rs b/src/debug.rs index b2e3ee1..9028648 100644 --- a/src/debug.rs +++ b/src/debug.rs @@ -20,7 +20,7 @@ pub(crate) macro println { $crate::debug::print(format_args!("\n")) }, ($fmt: expr $(, $($arg: tt)+)?) => { - $crate::debug::print(format_args!(concat!($fmt, "\n") $(, $($arg)+)?)) + $crate::debug::print(format_args!("{}\n", format_args!($fmt $(, $($arg)+)?))) }, } @@ -33,7 +33,7 @@ pub(crate) macro display_info{ $crate::debug::print(format_args!("\n")) }, ($item:literal,$fmt: expr $(, $($arg: tt)+)?) => { - $crate::debug::print(format_args!(concat!("{:<26}: ", $fmt, "\n"),$item $(, $($arg)+)?)) + $crate::debug::print(format_args!("{:<26}: {}\n", $item, format_args!($fmt $(, $($arg)+)?))) } } diff --git a/src/loongarch64/mod.rs b/src/loongarch64/mod.rs index d79cb2d..6f882a2 100644 --- a/src/loongarch64/mod.rs +++ b/src/loongarch64/mod.rs @@ -11,7 +11,7 @@ mod timer; mod trap; mod unaligned; -use crate::{clear_bss, multicore::MultiCore, CPU_NUM, DTB_BIN, MEM_AREA}; +use crate::{clear_bss, multicore::MultiCore, percpu::percpu_area_init, CPU_NUM, DTB_BIN, MEM_AREA}; use alloc::vec::Vec; pub use consts::*; pub use context::TrapFrame; @@ -23,6 +23,7 @@ pub use trap::{disable_irq, enable_external_irq, enable_irq, run_user_task}; pub fn rust_tmp_main(hart_id: usize) { clear_bss(); + percpu_area_init(hart_id); console::init(); trap::set_trap_vector_base(); sigtrx::init(); diff --git a/src/percpu.rs b/src/percpu.rs index e8806a3..abcddcf 100644 --- a/src/percpu.rs +++ b/src/percpu.rs @@ -1,11 +1,31 @@ -use core::{alloc::Layout, cell::UnsafeCell, mem::size_of, ptr::copy_nonoverlapping}; +/// !TIPS: x86_64 will + +use core::{alloc::Layout, mem::size_of, ptr::copy_nonoverlapping}; use alloc::alloc::alloc; -use crate::{debug::println, PAGE_SIZE}; +use crate::PAGE_SIZE; static BOOT_PERCPU_DATA_AREA: [u8; PAGE_SIZE] = [0; PAGE_SIZE]; +/// This is a empty seat for percpu section. +/// Force the linker to create the percpu section. +#[link_section = "percpu"] +#[used(linker)] +static _PERCPU_SEAT: [usize; 0] = [0; 0]; + +/// Reserved for default usage. +/// This is related to the [polyhal_macro::percpu::PERCPU_RESERVED] +/// Just for x86_64 now. +/// 0: SELF_PTR +/// 1: USER_RSP +/// 2: KERNEL_RSP +/// 3: USER_CONTEXT +#[cfg(target_arch = "x86_64")] +const PERCPU_RESERVED: usize = 4 * size_of::(); +#[cfg(not(target_arch = "x86_64"))] +const PERCPU_RESERVED: usize = 0; + /// Returns the base address of the per-CPU data area on the given CPU. /// /// if `cpu_id` is 0, it returns the base address of all per-CPU data areas. @@ -16,12 +36,7 @@ pub fn percpu_area_init(cpu_id: usize) -> usize { fn __stop_percpu(); } let start = __start_percpu as usize; - let size = __stop_percpu as usize - start; - // use polyhal_macro::percpu_symbol_offset; - // let start = percpu_symbol_offset!(__start_percpu); - // let size = percpu_symbol_offset!(__stop_percpu) - percpu_symbol_offset!(__start_percpu); - - println!("start: {:#x} size: {:#x}", start, size); + let size = __stop_percpu as usize - start + PERCPU_RESERVED; // Get the base address of the per-CPU data area // If cpu_id is boot,core then use BOOT_PERCPU_DATA_AREA. @@ -36,7 +51,7 @@ pub fn percpu_area_init(cpu_id: usize) -> usize { // Init the area with original data. unsafe { - copy_nonoverlapping(start as *const u8, dst, size); + copy_nonoverlapping(start as *const u8, dst.add(PERCPU_RESERVED), size); } dst as usize @@ -71,7 +86,8 @@ pub fn set_local_thread_pointer(cpu_id: usize) { cfg_if::cfg_if! { if #[cfg(target_arch = "x86_64")] { x86::msr::wrmsr(x86::msr::IA32_GS_BASE, tp as u64); - SELF_PTR.write_current_raw(tp); + // Write cpu_local pointer to the per-CPU data area + core::arch::asm!("mov gs:0, {0}", in(reg) tp); } else if #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] { core::arch::asm!("mv gp, {}", in(reg) tp) } else if #[cfg(target_arch = "aarch64")] { @@ -82,45 +98,3 @@ pub fn set_local_thread_pointer(cpu_id: usize) { } } } - -/// On x86, we use `gs:SELF_PTR` to store the address of the per-CPU data area base. -#[cfg(target_arch = "x86_64")] -#[no_mangle] -#[polyhal_macro::def_percpu] -static SELF_PTR: usize = 0; - -// 思路: 这里存放的 usize 数据，如果是 u8, u16, u32, u64, 或者 size 小于等于 u64 的直接存取 -// 否则这里仅表示指针。 -pub struct PerCpu { - value: UnsafeCell, -} - -unsafe impl Sync for PerCpu {} - -impl PerCpu { - pub const fn new(value: T) -> Self { - PerCpu { - value: UnsafeCell::new(value), - } - } - - pub fn get(&self) -> &T { - unsafe { &*self.value.get() } - } - - pub fn write(&self, value: T) { - unsafe { - core::ptr::write_volatile(self.value.get(), value); - } - } - - // pub fn offset(&self) -> usize { - // extern "Rust" { - // #[link_name = "__start_per_cpu"] - // fn SECION_START(); - // } - // self.value.get() as usize - SECION_START as usize - // } -} - -pub fn init_per_cpu() {} diff --git a/src/x86_64/consts.rs b/src/x86_64/consts.rs index e6586f8..10ff5f7 100644 --- a/src/x86_64/consts.rs +++ b/src/x86_64/consts.rs @@ -1,5 +1,12 @@ +use core::mem::size_of; + pub const VIRT_ADDR_START: usize = 0xffff_ff80_0000_0000; pub const SYSCALL_VECTOR: usize = 0x33445566; /// The offset of the pic irq. pub(super) const PIC_VECTOR_OFFSET: u8 = 0x20; + +/// Reserved percpu index +pub(super) const PERCPU_USER_RSP_OFFSET: usize = 1 * size_of::(); +pub(super) const PERCPU_KERNEL_RSP_OFFSET: usize = 2 * size_of::(); +pub(super) const PERCPU_USER_CONTEXT_OFFSET: usize = 3 * size_of::(); diff --git a/src/x86_64/interrupt.rs b/src/x86_64/interrupt.rs index bec0602..78a0829 100644 --- a/src/x86_64/interrupt.rs +++ b/src/x86_64/interrupt.rs @@ -14,7 +14,7 @@ use crate::currrent_arch::gdt::set_tss_kernel_sp; use crate::{currrent_arch::gdt::GdtStruct, TrapFrame, TrapType}; use super::apic::vectors::APIC_TIMER_VECTOR; -use super::consts::PIC_VECTOR_OFFSET; +use super::consts::{PERCPU_USER_CONTEXT_OFFSET, PERCPU_USER_RSP_OFFSET, PERCPU_KERNEL_RSP_OFFSET, PIC_VECTOR_OFFSET}; use super::context::FxsaveArea; global_asm!( @@ -44,18 +44,6 @@ global_asm!( global_asm!(include_str!("trap.S")); -#[no_mangle] -#[polyhal_macro::def_percpu] -static USER_RSP: usize = 0; - -#[no_mangle] -#[polyhal_macro::def_percpu] -static KERNEL_RSP: usize = 0; - -#[no_mangle] -#[polyhal_macro::def_percpu] -static USER_CONTEXT: usize = 0; - bitflags! { // https://wiki.osdev.org/Exceptions#Page_Fault #[derive(Debug)] @@ -193,7 +181,7 @@ pub unsafe extern "C" fn uservec() { swapgs mov rdi, rsp - mov rsp, gs:[offset __PERCPU_KERNEL_RSP] // kernel rsp + mov rsp, gs:{PERCPU_KERNEL_RSP_OFFSET} // kernel rsp pop r15 pop r14 @@ -210,6 +198,7 @@ pub unsafe extern "C" fn uservec() { ret ", + PERCPU_KERNEL_RSP_OFFSET = const PERCPU_KERNEL_RSP_OFFSET, options(noreturn) ); } @@ -234,7 +223,7 @@ pub extern "C" fn user_restore(context: *mut TrapFrame) { push r14 push r15 - mov gs:[offset __PERCPU_KERNEL_RSP], rsp + mov gs:{PERCPU_KERNEL_RSP_OFFSET}, rsp ", // Write fs_base and gs_base " @@ -275,6 +264,7 @@ pub extern "C" fn user_restore(context: *mut TrapFrame) { ", syscall_vector = const SYSCALL_VECTOR, sysretq = sym sysretq, + PERCPU_KERNEL_RSP_OFFSET = const PERCPU_KERNEL_RSP_OFFSET, options(noreturn) ) } @@ -325,11 +315,11 @@ unsafe extern "C" fn syscall_entry() { asm!( r" swapgs - mov gs:[offset __PERCPU_USER_RSP], rsp - mov rsp, gs:[offset __PERCPU_USER_CONTEXT] + mov gs:{PERCPU_USER_RSP_OFFSET}, rsp + mov rsp, gs:{PERCPU_USER_CONTEXT_OFFSET} sub rsp, 8 // skip user_ss - push gs:[offset __PERCPU_USER_RSP] // user_rsp + push gs:{PERCPU_USER_RSP_OFFSET} // user_rsp push r11 // rflags mov [rsp - 2 * 8], rcx // rip mov r11, {syscall_vector} @@ -362,7 +352,7 @@ unsafe extern "C" fn syscall_entry() { mov [rsp + 16*8+4], edx mov [rsp + 16*8], eax # push gs_base - mov rsp, gs:[offset __PERCPU_KERNEL_RSP] // kernel rsp + mov rsp, gs:{PERCPU_KERNEL_RSP_OFFSET} // kernel rsp pop r15 pop r14 pop r13 @@ -378,6 +368,9 @@ unsafe extern "C" fn syscall_entry() { ret ", syscall_vector = const SYSCALL_VECTOR, + PERCPU_USER_CONTEXT_OFFSET = const PERCPU_USER_CONTEXT_OFFSET, + PERCPU_USER_RSP_OFFSET = const PERCPU_USER_RSP_OFFSET, + PERCPU_KERNEL_RSP_OFFSET = const PERCPU_KERNEL_RSP_OFFSET, options(noreturn) ) } @@ -388,7 +381,14 @@ pub fn run_user_task(context: &mut TrapFrame) -> Option<()> { let cx_general_top = context as *mut TrapFrame as usize + TRAPFRAME_SIZE - size_of::(); set_tss_kernel_sp(cx_general_top); - USER_CONTEXT.write_current(cx_general_top); + // USER_CONTEXT.write_current(cx_general_top); + unsafe { + core::arch::asm!( + "mov gs:{USER_CONTEXT}, {0}", + in(reg) cx_general_top, + USER_CONTEXT = const PERCPU_USER_CONTEXT_OFFSET + ); + } context.fx_area.restore(); user_restore(context); context.fx_area.save(); diff --git a/src/x86_64/mod.rs b/src/x86_64/mod.rs index 25a5501..771ae28 100644 --- a/src/x86_64/mod.rs +++ b/src/x86_64/mod.rs @@ -40,7 +40,7 @@ use crate::{ debug::{display_info, println}, multicore::MultiCore, once::LazyInit, - percpu::PerCpu, + percpu::set_local_thread_pointer, CPU_NUM, DTB_BIN, MEM_AREA, }; @@ -61,8 +61,7 @@ fn rust_tmp_main(magic: usize, mboot_ptr: usize) { apic::init(); sigtrx::init(); // Init allocator - crate::percpu::init(1); - crate::percpu::set_local_thread_pointer(0); + set_local_thread_pointer(hart_id()); gdt::init(); interrupt::init_syscall(); time::init_early(); @@ -91,7 +90,7 @@ fn rust_tmp_main(magic: usize, mboot_ptr: usize) { // TODO: This is will be fixed with ACPI support CPU_NUM.init_by(1); - info!("magic: {:#x}, mboot_ptr: {:#x}", magic, mboot_ptr); + info!("magic: {magic:#x}, mboot_ptr: {mboot_ptr:#x}"); MBOOT_PTR.init_by(mboot_ptr); @@ -107,7 +106,7 @@ fn rust_tmp_main(magic: usize, mboot_ptr: usize) { ); display_info!("Platform FPU Support", "{}", features.has_fpu()); } - display_info!("Platform Virt Mem Offset", "{:#x}", VIRT_ADDR_START); + display_info!("Platform Virt Mem Offset", "{VIRT_ADDR_START:#x}"); // TODO: Use the dynamic uart information. display_info!("Platform UART Name", "Uart16550"); display_info!("Platform UART Port", "0x3f8"); @@ -134,7 +133,7 @@ fn rust_tmp_main(magic: usize, mboot_ptr: usize) { mboot.find_highest_address() ); } - display_info!("Boot HART ID", "{}", CPU_ID.read_current()); + display_info!("Boot HART ID", "{:#x}", CPU_ID.read_current()); display_info!(); unsafe { crate::api::_main_for_arch(0) }; diff --git a/src/x86_64/uart.rs b/src/x86_64/uart.rs index 0ab3f84..cece2c2 100644 --- a/src/x86_64/uart.rs +++ b/src/x86_64/uart.rs @@ -3,7 +3,7 @@ use irq_safety::MutexIrqSafe; use x86_64::instructions::port::{Port, PortReadOnly, PortWriteOnly}; -use crate::debug::{println, DebugConsole}; +use crate::debug::DebugConsole; const UART_CLOCK_FACTOR: usize = 16; const OSC_FREQ: usize = 1_843_200;