[hal] Refactor trap handler with linkme::distributed_slice

It allows trap handlers to be defined in any crate

modules/axhal/Cargo.toml

@@ -22,6 +22,7 @@ default = []
 [dependencies]
 log = "0.4"
 cfg-if = "1.0"
+linkme = "0.3"
 bitflags = "2.6"
 static_assertions = "1.1.0"
 kernel_guard = "0.1"

modules/axhal/linker.lds.S

@@ -49,17 +49,14 @@ SECTIONS
 
     . = ALIGN(4K);
     _percpu_start = .;
+    _percpu_end = _percpu_start + SIZEOF(.percpu);
     .percpu 0x0 : AT(_percpu_start) {
         _percpu_load_start = .;
         *(.percpu .percpu.*)
         _percpu_load_end = .;
-        . = ALIGN(64);
-        _percpu_size_aligned = .;
-
-        . = _percpu_load_start + _percpu_size_aligned * %SMP%;
+        . = _percpu_load_start + ALIGN(64) * %SMP%;
     }
-    . = _percpu_start + SIZEOF(.percpu);
-    _percpu_end = .;
+    . = _percpu_end;
 
     . = ALIGN(4K);
     _edata = .;
@@ -84,3 +81,11 @@ SECTIONS
         *(.comment) *(.gnu*) *(.note*) *(.eh_frame*)
     }
 }
+
+SECTIONS {
+    linkme_IRQ : { *(linkme_IRQ) }
+    linkm2_IRQ : { *(linkm2_IRQ) }
+    linkme_PAGE_FAULT : { *(linkme_PAGE_FAULT) }
+    linkm2_PAGE_FAULT : { *(linkm2_PAGE_FAULT) }
+}
+INSERT AFTER .tbss;

modules/axhal/src/arch/aarch64/trap.rs

@@ -1,6 +1,8 @@
 use core::arch::global_asm;
 
 use aarch64_cpu::registers::{ESR_EL1, FAR_EL1};
+use memory_addr::VirtAddr;
+use page_table_entry::MappingFlags;
 use tock_registers::interfaces::Readable;
 
 use super::TrapFrame;
@@ -35,39 +37,79 @@ fn invalid_exception(tf: &TrapFrame, kind: TrapKind, source: TrapSource) {
     );
 }
 
+#[no_mangle]
+fn handle_irq_exception(_tf: &TrapFrame) {
+    handle_trap!(IRQ, 0);
+}
+
+fn handle_instruction_abort(tf: &TrapFrame, iss: u64, is_user: bool) {
+    let mut access_flags = MappingFlags::EXECUTE;
+    if is_user {
+        access_flags |= MappingFlags::USER;
+    }
+    let vaddr = VirtAddr::from(FAR_EL1.get() as usize);
+
+    // Only handle Translation fault and Permission fault
+    if !matches!(iss & 0b111100, 0b0100 | 0b1100) // IFSC or DFSC bits
+        || !handle_trap!(PAGE_FAULT, vaddr, access_flags, is_user)
+    {
+        panic!(
+            "Unhandled {} Instruction Abort @ {:#x}, fault_vaddr={:#x}, ISS={:#x} ({:?}):\n{:#x?}",
+            if is_user { "EL0" } else { "EL1" },
+            tf.elr,
+            vaddr,
+            iss,
+            access_flags,
+            tf,
+        );
+    }
+}
+
+fn handle_data_abort(tf: &TrapFrame, iss: u64, is_user: bool) {
+    let wnr = (iss & (1 << 6)) != 0; // WnR: Write not Read
+    let cm = (iss & (1 << 8)) != 0; // CM: Cache maintenance
+    let mut access_flags = if wnr & !cm {
+        MappingFlags::WRITE
+    } else {
+        MappingFlags::READ
+    };
+    if is_user {
+        access_flags |= MappingFlags::USER;
+    }
+    let vaddr = VirtAddr::from(FAR_EL1.get() as usize);
+
+    // Only handle Translation fault and Permission fault
+    if !matches!(iss & 0b111100, 0b0100 | 0b1100) // IFSC or DFSC bits
+        || !handle_trap!(PAGE_FAULT, vaddr, access_flags, is_user)
+    {
+        panic!(
+            "Unhandled {} Data Abort @ {:#x}, fault_vaddr={:#x}, ISS=0b{:08b} ({:?}):\n{:#x?}",
+            if is_user { "EL0" } else { "EL1" },
+            tf.elr,
+            vaddr,
+            iss,
+            access_flags,
+            tf,
+        );
+    }
+}
+
 #[no_mangle]
 fn handle_sync_exception(tf: &mut TrapFrame) {
     let esr = ESR_EL1.extract();
+    let iss = esr.read(ESR_EL1::ISS);
     match esr.read_as_enum(ESR_EL1::EC) {
+        Some(ESR_EL1::EC::Value::SVC64) => {
+            warn!("No syscall is supported currently!");
+        }
+        Some(ESR_EL1::EC::Value::InstrAbortLowerEL) => handle_instruction_abort(tf, iss, true),
+        Some(ESR_EL1::EC::Value::InstrAbortCurrentEL) => handle_instruction_abort(tf, iss, false),
+        Some(ESR_EL1::EC::Value::DataAbortLowerEL) => handle_data_abort(tf, iss, true),
+        Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => handle_data_abort(tf, iss, false),
         Some(ESR_EL1::EC::Value::Brk64) => {
-            let iss = esr.read(ESR_EL1::ISS);
             debug!("BRK #{:#x} @ {:#x} ", iss, tf.elr);
             tf.elr += 4;
         }
-        Some(ESR_EL1::EC::Value::SVC64) => {
-            warn!("No supervisor call is supported currently!");
-        }
-        Some(ESR_EL1::EC::Value::DataAbortLowerEL)
-        | Some(ESR_EL1::EC::Value::InstrAbortLowerEL) => {
-            let iss = esr.read(ESR_EL1::ISS);
-            warn!(
-                "EL0 Page Fault @ {:#x}, FAR={:#x}, ISS={:#x}",
-                tf.elr,
-                FAR_EL1.get(),
-                iss
-            );
-        }
-        Some(ESR_EL1::EC::Value::DataAbortCurrentEL)
-        | Some(ESR_EL1::EC::Value::InstrAbortCurrentEL) => {
-            let iss = esr.read(ESR_EL1::ISS);
-            panic!(
-                "EL1 Page Fault @ {:#x}, FAR={:#x}, ISS={:#x}:\n{:#x?}",
-                tf.elr,
-                FAR_EL1.get(),
-                iss,
-                tf,
-            );
-        }
         _ => {
             panic!(
                 "Unhandled synchronous exception @ {:#x}: ESR={:#x} (EC {:#08b}, ISS {:#x})",
@@ -79,8 +121,3 @@ fn handle_sync_exception(tf: &mut TrapFrame) {
         }
     }
 }
-
-#[no_mangle]
-fn handle_irq_exception(_tf: &TrapFrame) {
-    crate::trap::handle_irq_extern(0)
-}

modules/axhal/src/arch/riscv/trap.rs

@@ -1,4 +1,7 @@
+use memory_addr::VirtAddr;
+use page_table_entry::MappingFlags;
 use riscv::register::scause::{self, Exception as E, Trap};
+use riscv::register::stval;
 
 use super::TrapFrame;
 
@@ -14,12 +17,36 @@ fn handle_breakpoint(sepc: &mut usize) {
     *sepc += 2
 }
 
+fn handle_page_fault(tf: &TrapFrame, mut access_flags: MappingFlags, is_user: bool) {
+    if is_user {
+        access_flags |= MappingFlags::USER;
+    }
+    let vaddr = VirtAddr::from(stval::read());
+    if !handle_trap!(PAGE_FAULT, vaddr, access_flags, is_user) {
+        panic!(
+            "Unhandled {} Page Fault @ {:#x}, fault_vaddr={:#x} ({:?}):\n{:#x?}",
+            if is_user { "User" } else { "Supervisor" },
+            tf.sepc,
+            vaddr,
+            access_flags,
+            tf,
+        );
+    }
+}
+
 #[no_mangle]
-fn riscv_trap_handler(tf: &mut TrapFrame, _from_user: bool) {
+fn riscv_trap_handler(tf: &mut TrapFrame, from_user: bool) {
     let scause = scause::read();
     match scause.cause() {
+        Trap::Exception(E::LoadPageFault) => handle_page_fault(tf, MappingFlags::READ, from_user),
+        Trap::Exception(E::StorePageFault) => handle_page_fault(tf, MappingFlags::WRITE, from_user),
+        Trap::Exception(E::InstructionPageFault) => {
+            handle_page_fault(tf, MappingFlags::EXECUTE, from_user)
+        }
         Trap::Exception(E::Breakpoint) => handle_breakpoint(&mut tf.sepc),
-        Trap::Interrupt(_) => crate::trap::handle_irq_extern(scause.bits()),
+        Trap::Interrupt(_) => {
+            handle_trap!(IRQ, scause.bits());
+        }
         _ => {
             panic!(
                 "Unhandled trap {:?} @ {:#x}:\n{:#x?}",

modules/axhal/src/arch/x86_64/trap.rs

@@ -1,4 +1,7 @@
+use memory_addr::VirtAddr;
+use page_table_entry::MappingFlags;
 use x86::{controlregs::cr2, irq::*};
+use x86_64::structures::idt::PageFaultErrorCode;
 
 use super::context::TrapFrame;
 
@@ -7,40 +10,79 @@ core::arch::global_asm!(include_str!("trap.S"));
 const IRQ_VECTOR_START: u8 = 0x20;
 const IRQ_VECTOR_END: u8 = 0xff;
 
+fn handle_page_fault(tf: &TrapFrame) {
+    let access_flags = err_code_to_flags(tf.error_code)
+        .unwrap_or_else(|e| panic!("Invalid #PF error code: {:#x}", e));
+    let vaddr = VirtAddr::from(unsafe { cr2() });
+    if !handle_trap!(PAGE_FAULT, vaddr, access_flags, tf.is_user()) {
+        panic!(
+            "Unhandled {} #PF @ {:#x}, fault_vaddr={:#x}, error_code={:#x} ({:?}):\n{:#x?}",
+            if tf.is_user() { "user" } else { "kernel" },
+            tf.rip,
+            vaddr,
+            tf.error_code,
+            access_flags,
+            tf,
+        );
+    }
+}
+
 #[no_mangle]
 fn x86_trap_handler(tf: &TrapFrame) {
     match tf.vector as u8 {
-        PAGE_FAULT_VECTOR => {
-            if tf.is_user() {
-                warn!(
-                    "User #PF @ {:#x}, fault_vaddr={:#x}, error_code={:#x}",
-                    tf.rip,
-                    unsafe { cr2() },
-                    tf.error_code,
-                );
-            } else {
-                panic!(
-                    "Kernel #PF @ {:#x}, fault_vaddr={:#x}, error_code={:#x}:\n{:#x?}",
-                    tf.rip,
-                    unsafe { cr2() },
-                    tf.error_code,
-                    tf,
-                );
-            }
-        }
+        PAGE_FAULT_VECTOR => handle_page_fault(tf),
         BREAKPOINT_VECTOR => debug!("#BP @ {:#x} ", tf.rip),
         GENERAL_PROTECTION_FAULT_VECTOR => {
             panic!(
                 "#GP @ {:#x}, error_code={:#x}:\n{:#x?}",
                 tf.rip, tf.error_code, tf
             );
         }
-        IRQ_VECTOR_START..=IRQ_VECTOR_END => crate::trap::handle_irq_extern(tf.vector as _),
+        IRQ_VECTOR_START..=IRQ_VECTOR_END => {
+            handle_trap!(IRQ, tf.vector as _);
+        }
         _ => {
             panic!(
-                "Unhandled exception {} (error_code = {:#x}) @ {:#x}:\n{:#x?}",
-                tf.vector, tf.error_code, tf.rip, tf
+                "Unhandled exception {} ({}, error_code={:#x}) @ {:#x}:\n{:#x?}",
+                tf.vector,
+                vec_to_str(tf.vector),
+                tf.error_code,
+                tf.rip,
+                tf
             );
         }
     }
 }
+
+fn vec_to_str(vec: u64) -> &'static str {
+    if vec < 32 {
+        EXCEPTIONS[vec as usize].mnemonic
+    } else {
+        "Unknown"
+    }
+}
+
+fn err_code_to_flags(err_code: u64) -> Result<MappingFlags, u64> {
+    let code = PageFaultErrorCode::from_bits_truncate(err_code);
+    let reserved_bits = (PageFaultErrorCode::CAUSED_BY_WRITE
+        | PageFaultErrorCode::USER_MODE
+        | PageFaultErrorCode::INSTRUCTION_FETCH)
+        .complement();
+    if code.intersects(reserved_bits) {
+        Err(err_code)
+    } else {
+        let mut flags = MappingFlags::empty();
+        if code.contains(PageFaultErrorCode::CAUSED_BY_WRITE) {
+            flags |= MappingFlags::WRITE;
+        } else {
+            flags |= MappingFlags::READ;
+        }
+        if code.contains(PageFaultErrorCode::USER_MODE) {
+            flags |= MappingFlags::USER;
+        }
+        if code.contains(PageFaultErrorCode::INSTRUCTION_FETCH) {
+            flags |= MappingFlags::EXECUTE;
+        }
+        Ok(flags)
+    }
+}

modules/axhal/src/irq.rs

@@ -2,9 +2,10 @@
 
 use handler_table::HandlerTable;
 
-use crate::platform::irq::MAX_IRQ_COUNT;
+use crate::platform::irq::{dispatch_irq, MAX_IRQ_COUNT};
+use crate::trap::{reg_trap_handler, IRQ};
 
-pub use crate::platform::irq::{dispatch_irq, register_handler, set_enable};
+pub use crate::platform::irq::{register_handler, set_enable};
 
 /// The type if an IRQ handler.
 pub type IrqHandler = handler_table::Handler;
@@ -33,3 +34,11 @@ pub(crate) fn register_handler_common(irq_num: usize, handler: IrqHandler) -> bo
     warn!("register handler for IRQ {} failed", irq_num);
     false
 }
+
+#[reg_trap_handler(IRQ)]
+fn handler_irq(irq_num: usize) -> bool {
+    let guard = kernel_guard::NoPreempt::new();
+    dispatch_irq(irq_num);
+    drop(guard); // rescheduling may occur when preemption is re-enabled.
+    true
+}

modules/axhal/src/lib.rs

@@ -36,11 +36,13 @@ extern crate log;
 
 mod platform;
 
+#[macro_use]
+pub mod trap;
+
 pub mod arch;
 pub mod cpu;
 pub mod mem;
 pub mod time;
-pub mod trap;
 
 #[cfg(feature = "tls")]
 pub mod tls;