From 4deb0b7122568856191f5307864b61670ce6098b Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Wed, 11 Sep 2024 15:29:38 +0000 Subject: [PATCH 01/10] caprevoke: Make assertions in pmap_assert_consistent_clg() more useful Print the PTE as well, since the debugger isn't able to retrieve it. While here, remove an XXX comment in the arm64 implementation: pmap_pte() returns NULL if the address is not mapped, which is a perfectly ordinary state of affairs. --- sys/arm64/arm64/pmap.c | 8 +++++--- sys/riscv/riscv/pmap.c | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c index 1f3df5ade6b0..0da8fc767d14 100644 --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -6719,7 +6719,7 @@ pmap_assert_consistent_clg(pmap_t pmap, vm_offset_t va) pte = pmap_pte(pmap, va, &level); if (pte == NULL) - return; /* XXX: why does this happen? */ + return; tpte = pmap_load(pte); if ((tpte & ATTR_SW_MANAGED) == 0 || !pmap_pte_capdirty(pmap, tpte)) return; @@ -6729,10 +6729,12 @@ pmap_assert_consistent_clg(pmap_t pmap, vm_offset_t va) case ATTR_LC_ENABLED: panic("no clg"); case ATTR_LC_GEN0: - KASSERT(pmap->flags.uclg == 0, ("GEN0 LCLG with GEN1 GCLG")); + KASSERT(pmap->flags.uclg == 0, + ("GEN0 LCLG with GEN1 GCLG (%#lx)", tpte)); break; case ATTR_LC_GEN1: - KASSERT(pmap->flags.uclg == 1, ("GEN1 LCLG with GEN0 GCLG")); + KASSERT(pmap->flags.uclg == 1, + ("GEN1 LCLG with GEN0 GCLG (%#lx)", tpte)); break; default: panic("impossible?"); diff --git a/sys/riscv/riscv/pmap.c b/sys/riscv/riscv/pmap.c index ac0e06c3f0ab..59d2c8efdef3 100644 --- a/sys/riscv/riscv/pmap.c +++ b/sys/riscv/riscv/pmap.c @@ -4156,9 +4156,11 @@ pmap_assert_consistent_clg(pmap_t pmap, vm_offset_t va) if ((tpte & PTE_CW) == 0) return; if ((tpte & PTE_CRG) == 0) { - KASSERT(pmap->flags.uclg == 0, ("PTR_CRG unset, but GCLG set")); + KASSERT(pmap->flags.uclg == 0, + ("PTR_CRG unset, but GCLG set (%#lx)", tpte)); } else { - KASSERT(pmap->flags.uclg == 1, ("PTR_CRG set, but GCLG unset")); + KASSERT(pmap->flags.uclg == 1, + ("PTR_CRG set, but GCLG unset (%#lx)", tpte)); } } #endif /* CHERI_CAPREVOKE */ From c784b9d988125e51a2ceee146686d879ef9dfcb9 Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Wed, 11 Sep 2024 23:01:57 +0000 Subject: [PATCH 02/10] arm64: Preserve ATTR_SC when demoting l3c mappings If any entry in a L3C mapping has ATTR_SC set (i.e., it is cap-dirty), the flag must be propagated to all other entries when demoting. --- sys/arm64/arm64/pmap.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c index 0da8fc767d14..279095d31f44 100644 --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -9196,6 +9196,9 @@ pmap_demote_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va) (ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_SW_DBM)) mask = ATTR_S1_AP_RW_BIT; nbits |= l3e & ATTR_AF; +#if __has_feature(capabilities) + nbits |= l3e & ATTR_SC; +#endif } if ((nbits & ATTR_AF) != 0) { pmap_invalidate_range(pmap, va & ~L3C_OFFSET, (va + L3C_SIZE) & From 508fd99bd133680919f8a110967e28222f001bf6 Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Fri, 13 Sep 2024 15:27:39 +0000 Subject: [PATCH 03/10] arm64: Remove an unneeded check in pmap_promote_l2() We don't need to check each PTE: we already verify that the first PTE in the run does not have CDBM set and SC clear, and in the loop we check that the PTEs are identical in virtually all attributes. No functional change intended. --- sys/arm64/arm64/pmap.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c index 279095d31f44..e33da9387980 100644 --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -4973,21 +4973,6 @@ pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, vm_page_t mpte, return (false); } all_l3e_AF &= oldl3; -#if __has_feature(capabilities) - /* - * Prohibit superpages involving CDBM-set SC-clear PTEs. The - * revoker creates these without TLB shootdown, and so there - * may be a SC-set TLBE still in the system. Thankfully, - * these are ephemera: either they'll transition to CD-set - * or CW-clear in the next revocation epoch. - */ - if ((oldl3 & (ATTR_CDBM | ATTR_SC)) == ATTR_CDBM) { - atomic_add_long(&pmap_l2_p_failures, 1); - CTR2(KTR_PMAP, "pmap_promote_l2: CDBM failure for va " - "%#lx in pmap %p", va, pmap); - return (false); - } -#endif pa -= PAGE_SIZE; } From d5d2f479e642748f93596650a1c7e768f3ed78cc Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Wed, 11 Sep 2024 23:26:46 +0000 Subject: [PATCH 04/10] arm64: Avoid promoting L3C runs CDBM-set SC-clear PTEs As in pmap_promote_l2(), we should avoid promotion if the mapping allows capability stores but is not yet capability-dirty. --- sys/arm64/arm64/pmap.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c index e33da9387980..5e12e0cea549 100644 --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -5066,6 +5066,18 @@ pmap_promote_l3c(pmap_t pmap, pd_entry_t *l3p, vm_offset_t va) * to a read-only mapping. See pmap_promote_l2() for the rationale. */ set_first: +#if __has_feature(capabilities) + /* + * Prohibit superpages involving CDBM-set SC-clear PTEs. See + * pmap_promote_l2() for the rationale. + */ + if ((firstl3c & (ATTR_CDBM | ATTR_SC)) == ATTR_CDBM) { + CTR2(KTR_PMAP, "pmap_promote_l3c: CDBM failure for va " + "%#lx in pmap %p", va, pmap); + return (false); + } +#endif + if ((firstl3c & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) == (ATTR_S1_AP(ATTR_S1_AP_RO) | ATTR_SW_DBM)) { /* From a1dc932cd79c07775fa203bf41d640dedc71e168 Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Wed, 11 Sep 2024 23:59:51 +0000 Subject: [PATCH 05/10] caprevoke: Demote L2 and L3C mappings in pmap_caploadgen_update() Previously, pmap_caploadgen_update() would return PMAP_CAPLOADGEN_UNABLE, which is overloaded to cover the case where no mapping exists at all. In this case, the MI machinery will look up the page via the VM object and scan it. If the 4KB page contains capabilities, vm_fault() is called upon to scan the page and update the pmap, which breaks the large mapping as a side effect. However, if the page is capability-clean, the large mapping will not be demoted, and during a background scan the subsequent pmap_caploadgen_update() will fail to update the CLG in the L2 or L3C PTE. Thus, the scan will end with some PTEs carrying the wrong CLG. When the GCLG is incremented during a subsequent scan, the kernel may fail to interpose load-cap operations from that mapping, which could potentially allow revoked capabilities to be leaked. Fix the problem by demoting the mapping as soon as it is encountered. This is not optimal (we end up demoting clean superpage mappings), but it fixes the problem of overloading PMAP_CAPLOADGEN_UNABLE and removes an assumption about pmap_enter()'s behaviour (i.e., that it'll demote L2 and L3C mappings), and pmap_caploadgen_update() already reconstructs superpage mappings when possible. L1 largepage mappings are left unhandled for now. --- sys/arm64/arm64/pmap.c | 38 +++++++++++++++++++++++++++----------- sys/riscv/riscv/pmap.c | 26 +++++++++++++++++++------- 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c index 5e12e0cea549..199d09709109 100644 --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -6437,6 +6437,7 @@ enum pmap_caploadgen_res pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags) { enum pmap_caploadgen_res res; + struct rwlock *lock; #if VM_NRESERVLEVEL > 0 pd_entry_t *l2, l2e; #endif @@ -6452,6 +6453,7 @@ pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags) ("pmap_caploadgen_update: pmap clg %d but CPU mismatch", (int)pmap->flags.uclg)); +retry: pte = pmap_pte(pmap, va, &lvl); if (pte == NULL) { m = NULL; @@ -6471,24 +6473,38 @@ pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags) break; } - switch(lvl) { - /* - * Large page: bouncing out here means we'll take a VM fault to - * find the page in question. - * - * XXX We'd rather just demote the pages right now, surely? - */ + switch (lvl) { + default: + __assert_unreachable(); case 1: - case 2: + /* XXX-MJ we can't really demote shm_largepage mappings */ m = NULL; res = PMAP_CAPLOADGEN_UNABLE; - goto out; - case 3: - if ((tpte & ATTR_CONTIGUOUS) != 0) { + break; + case 2: + /* + * Demote superpage and contiguous mappings. When performing + * load-side revocation, we don't want to pay the latency + * penalty of scanning a large mapping. Ideally, however, the + * background scan could avoid breaking these mappings. For + * now, we attempt to reconstruct them below. + */ + lock = NULL; + pte = pmap_demote_l2_locked(pmap, pte, va, &lock); + if (lock != NULL) + rw_wunlock(lock); + if (pte == NULL) { + /* Demotion failed. */ m = NULL; res = PMAP_CAPLOADGEN_UNABLE; goto out; } + goto retry; + case 3: + if ((tpte & ATTR_CONTIGUOUS) != 0) { + pmap_demote_l3c(pmap, pte, va); + goto retry; + } break; } diff --git a/sys/riscv/riscv/pmap.c b/sys/riscv/riscv/pmap.c index 59d2c8efdef3..d136bee27996 100644 --- a/sys/riscv/riscv/pmap.c +++ b/sys/riscv/riscv/pmap.c @@ -3875,6 +3875,7 @@ enum pmap_caploadgen_res pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags) { enum pmap_caploadgen_res res; + struct rwlock *lock; pd_entry_t *l2, l2e; pt_entry_t *pte, oldpte = 0; vm_page_t m; @@ -3885,6 +3886,7 @@ pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags) ("pmap_caploadgen_update: pmap crg %d but CPU mismatch", (int)pmap->flags.uclg)); +retry: l2 = pmap_l2(pmap, va); if (l2 == NULL || ((l2e = pmap_load(l2)) & PTE_V) == 0) { m = NULL; @@ -3892,15 +3894,25 @@ pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags) goto out; } if ((l2e & PTE_RWX) != 0) { + bool demoted; + /* - * Large page: bouncing out here means we'll take a VM fault - * to find the page in question. - * - * XXX We'd rather just demote the L2 page right now, surely? + * Demote superpage and contiguous mappings. When performing + * load-side revocation, we don't want to pay the latency + * penalty of scanning a large mapping. Ideally, however, the + * background scan could avoid breaking these mappings. For + * now, we attempt to reconstruct them below. */ - m = NULL; - res = PMAP_CAPLOADGEN_UNABLE; - goto out; + lock = NULL; + demoted = pmap_demote_l2_locked(pmap, l2, va, &lock); + if (lock != NULL) + rw_wunlock(lock); + if (!demoted) { + m = NULL; + res = PMAP_CAPLOADGEN_UNABLE; + goto out; + } + goto retry; } pte = pmap_l2_to_l3(l2, va); if (pte == NULL || ((oldpte = pmap_load(pte)) & PTE_V) == 0) { From 8a399c305350288c837fb04e5516fabcacc0f5d2 Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Thu, 12 Sep 2024 00:19:28 +0000 Subject: [PATCH 06/10] arm64: Rebuild ATTR_CONTIGUOUS mappings in pmap_caploadgen_update() The background scan opportunistically rebuilds L2 superpage mappings. Do the same for L3C mappings. Re-enable creation of read-only L3C mappings in pmap_enter_l3c(). Since that commit appeared, upstream has implemented L3C promotion, and pmap_caploadgen_update() now handles L3C mappings by demoting rather than going through the fault handler. --- sys/arm64/arm64/pmap.c | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c index 199d09709109..7aebd2068743 100644 --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -5868,18 +5868,6 @@ pmap_enter_l3c(pmap_t pmap, vm_offset_t va, pt_entry_t l3e, u_int flags, KASSERT(!VA_IS_CLEANMAP(va) || (l3e & ATTR_SW_MANAGED) == 0, ("pmap_enter_l3c: managed mapping within the clean submap")); -#ifdef CHERI_CAPREVOKE - if (!ADDR_IS_KERNEL(va)) { - /* - * pmap_caploadgen_update() currently does not handle L3C - * mappings. Avoid creating them at all on the basis that the - * pessimization of going through vm_fault() for capload faults - * is probably worse than not having L3C mappings at all. - */ - return (KERN_FAILURE); - } -#endif - /* * If the L3 PTP is not resident, we attempt to create it here. */ @@ -6677,15 +6665,17 @@ pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags) #if VM_NRESERVLEVEL > 0 /* * If we... - * are on the background scan (as indicated by EXCLUSIVE), + * are on the background scan (as indicated by NONEWMAPS), * are writing back a PTE (m != NULL), * have superpages enabled, - * are at the last page of an L2 entry, - * then see if we can put a superpage back together. + * try to restore an L3C mapping. If that succeeded, then see if we can + * put a superpage back together. */ if ((flags & PMAP_CAPLOADGEN_NONEWMAPS) && - (m != NULL) && pmap_ps_enabled(pmap) && - ((va & (L2_OFFSET - L3_OFFSET)) == (L2_OFFSET - L3_OFFSET))) { + m != NULL && pmap_ps_enabled(pmap) && + (va & L3C_OFFSET) == (PTE_TO_PHYS(tpte) & L3C_OFFSET) && + vm_reserv_is_populated(m, L3C_ENTRIES) && + pmap_promote_l3c(pmap, pte, va)) { KASSERT(lvl == 3, ("pmap_caploadgen_update superpage: lvl != 3")); KASSERT((m->flags & PG_FICTITIOUS) == 0, From b17f73ec3c290e6f61f53ec029b13e32b8f180ef Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Wed, 11 Sep 2024 14:01:58 +0000 Subject: [PATCH 07/10] caprevoke: Improve the vm_cheri_assert_consistent_clg() check - Use a sysctl to enable it, rather than requiring it be configured at compile-time - Skip over map entries that definitely do not contain capabilities, including the quarantine bitmap (OBJ_NOCAP) --- sys/kern/kern_cheri_revoke.c | 2 -- sys/vm/vm_cheri_revoke.c | 20 +++++++++++++++++--- sys/vm/vm_map.c | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/sys/kern/kern_cheri_revoke.c b/sys/kern/kern_cheri_revoke.c index 4a2ef177b6e9..8f5847d80eb2 100644 --- a/sys/kern/kern_cheri_revoke.c +++ b/sys/kern/kern_cheri_revoke.c @@ -724,9 +724,7 @@ kern_cheri_revoke(struct thread *td, int flags, /* OK, that's that. Where do we stand now? */ if (res == KERN_SUCCESS && myst == CHERI_REVOKE_ST_CLOSING) { -#ifdef DIAGNOSTIC vm_cheri_assert_consistent_clg(&vm->vm_map); -#endif /* Signal the end of this revocation epoch */ epoch++; crepochs.dequeue = epoch; diff --git a/sys/vm/vm_cheri_revoke.c b/sys/vm/vm_cheri_revoke.c index e1315298db39..6962d967232a 100644 --- a/sys/vm/vm_cheri_revoke.c +++ b/sys/vm/vm_cheri_revoke.c @@ -508,6 +508,15 @@ SYSCTL_BOOL(_vm_cheri_revoke, OID_AUTO, avoid_faults, CTLFLAG_RWTUN, &cheri_revoke_avoid_faults, 0, "Avoid faulting when the pager is known not to contain the page"); +#ifdef DIAGNOSTIC +static bool cheri_validate_clg = true; +#else +static bool cheri_validate_clg = false; +#endif +SYSCTL_BOOL(_vm_cheri_revoke, OID_AUTO, validate_clg, CTLFLAG_RWTUN, + &cheri_validate_clg, 0, + "Validate LCLGs against the pmap after forks and revocation scans"); + static bool vm_cheri_revoke_skip_fault(vm_object_t object) { @@ -1104,15 +1113,20 @@ vm_cheri_assert_consistent_clg(struct vm_map *map) { pmap_t pmap = map->pmap; vm_map_entry_t entry; + vm_object_t object; vm_offset_t addr; /* Called with map lock held */ + if (!cheri_validate_clg) + return; VM_MAP_ENTRY_FOREACH(entry, map) { - if ((entry->max_protection & VM_PROT_READ_CAP) == 0 || - entry->object.vm_object == NULL) { + if ((entry->max_protection & VM_PROT_READ_CAP) == 0) + continue; + object = entry->object.vm_object; + if (object == NULL || (object->flags & OBJ_HASCAP) == 0 || + (object->flags & OBJ_NOCAP) != 0) continue; - } for (addr = entry->start; addr < entry->end; addr += pagesizes[0]) { diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c index 62871a068eca..255d17e8bf21 100644 --- a/sys/vm/vm_map.c +++ b/sys/vm/vm_map.c @@ -5342,7 +5342,7 @@ vmspace_fork(struct vmspace *vm1, vm_ooffset_t *fork_charge) #endif } } -#if defined(CHERI_CAPREVOKE) && defined(DIAGNOSTIC) +#if defined(CHERI_CAPREVOKE) vm_cheri_assert_consistent_clg(new_map); #endif /* From a7686ba7ca93e14f24d9d57d2ab50776600f901b Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Fri, 13 Sep 2024 13:46:08 +0000 Subject: [PATCH 08/10] caprevoke: Avoid validating CLGs if revocation is in progress When forking, the GCLG and LCLGs may be out of sync if a new revocation epoch was opened (bumping the GCLG) but the background scan has not yet started (background scans block fork()). This can arise if a cheri_revoke() is called without either of the CHERI_REVOKE_LAST_PASS and CHERI_REVOKE_ASYNC flags. --- sys/vm/vm_cheri_revoke.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sys/vm/vm_cheri_revoke.c b/sys/vm/vm_cheri_revoke.c index 6962d967232a..114e5694fd37 100644 --- a/sys/vm/vm_cheri_revoke.c +++ b/sys/vm/vm_cheri_revoke.c @@ -1120,6 +1120,8 @@ vm_cheri_assert_consistent_clg(struct vm_map *map) if (!cheri_validate_clg) return; + if (cheri_revoke_st_is_revoking(map->vm_cheri_revoke_st)) + return; VM_MAP_ENTRY_FOREACH(entry, map) { if ((entry->max_protection & VM_PROT_READ_CAP) == 0) continue; From 136fa17b130d44016734c1426c8e0c14d66367c2 Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Fri, 13 Sep 2024 14:33:13 +0000 Subject: [PATCH 09/10] caprevoke: Fix a buggy vm_cheri_assert_consistent_clg() call vm_cheri_assert_consistent_clg() must be called with the VM map lock held. Also make sure to only call it after the epoch is closed, i.e., we have set the revocation state to CHERI_REVOKE_ST_NONE. --- sys/kern/kern_cheri_revoke.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sys/kern/kern_cheri_revoke.c b/sys/kern/kern_cheri_revoke.c index 8f5847d80eb2..c36258b617d5 100644 --- a/sys/kern/kern_cheri_revoke.c +++ b/sys/kern/kern_cheri_revoke.c @@ -724,7 +724,6 @@ kern_cheri_revoke(struct thread *td, int flags, /* OK, that's that. Where do we stand now? */ if (res == KERN_SUCCESS && myst == CHERI_REVOKE_ST_CLOSING) { - vm_cheri_assert_consistent_clg(&vm->vm_map); /* Signal the end of this revocation epoch */ epoch++; crepochs.dequeue = epoch; @@ -736,6 +735,8 @@ kern_cheri_revoke(struct thread *td, int flags, vm_map_lock(vmm); cheri_revoke_st_set(&vmm->vm_cheri_revoke_st, epoch, myst); + if (res == KERN_SUCCESS) + vm_cheri_assert_consistent_clg(&vm->vm_map); #ifdef CHERI_CAPREVOKE_STATS if (flags & CHERI_REVOKE_TAKE_STATS) { sx_xlock(&vmm->vm_cheri_revoke_stats_sx); From 9e7f3703138062f082f4b28632de81f98fa8c222 Mon Sep 17 00:00:00 2001 From: Mark Johnston Date: Wed, 9 Oct 2024 13:51:16 +0000 Subject: [PATCH 10/10] riscv: Grab pvh_global_lock in pmap_caploadgen_update() This is required when promoting or demoting superpages, both of which may be done here. --- sys/riscv/riscv/pmap.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sys/riscv/riscv/pmap.c b/sys/riscv/riscv/pmap.c index d136bee27996..574116037f9b 100644 --- a/sys/riscv/riscv/pmap.c +++ b/sys/riscv/riscv/pmap.c @@ -3880,6 +3880,7 @@ pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags) pt_entry_t *pte, oldpte = 0; vm_page_t m; + rw_rlock(&pvh_global_lock); PMAP_LOCK(pmap); KASSERT(!(csr_read(sccsr) & SCCSR_UGCLG) == !(pmap->flags.uclg), @@ -4010,6 +4011,7 @@ pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags) sfence_vma_page(va); } PMAP_UNLOCK(pmap); + rw_runlock(&pvh_global_lock); pmap_caploadgen_test_all_clean(m); m = NULL; goto out_unlocked; @@ -4133,6 +4135,7 @@ pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags) #endif /* VM_NRESERVLEVEL > 0 */ PMAP_UNLOCK(pmap); + rw_runlock(&pvh_global_lock); out_unlocked: if (*mp != NULL) {