Address caprevoke consistency check failures

sys/arm64/arm64/pmap.c

@@ -4973,21 +4973,6 @@ pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset_t va, vm_page_t mpte,
 			return (false);
 		}
 		all_l3e_AF &= oldl3;
-#if __has_feature(capabilities)
-		/*
-		 * Prohibit superpages involving CDBM-set SC-clear PTEs.  The
-		 * revoker creates these without TLB shootdown, and so there
-		 * may be a SC-set TLBE still in the system.  Thankfully,
-		 * these are ephemera: either they'll transition to CD-set
-		 * or CW-clear in the next revocation epoch.
-		 */
-		if ((oldl3 & (ATTR_CDBM | ATTR_SC)) == ATTR_CDBM) {
-			atomic_add_long(&pmap_l2_p_failures, 1);
-			CTR2(KTR_PMAP, "pmap_promote_l2: CDBM failure for va "
-			    "%#lx in pmap %p", va, pmap);
-			return (false);
-		}
-#endif
 		pa -= PAGE_SIZE;
 	}
 
@@ -5081,6 +5066,18 @@ pmap_promote_l3c(pmap_t pmap, pd_entry_t *l3p, vm_offset_t va)
 	 * to a read-only mapping.  See pmap_promote_l2() for the rationale.
 	 */
 set_first:
+#if __has_feature(capabilities)
+	/*
+	 * Prohibit superpages involving CDBM-set SC-clear PTEs.  See
+	 * pmap_promote_l2() for the rationale.
+	 */
+	if ((firstl3c & (ATTR_CDBM | ATTR_SC)) == ATTR_CDBM) {
+		CTR2(KTR_PMAP, "pmap_promote_l3c: CDBM failure for va "
+		    "%#lx in pmap %p", va, pmap);
+		return (false);
+	}
+#endif
+
 	if ((firstl3c & (ATTR_S1_AP_RW_BIT | ATTR_SW_DBM)) ==
 	    (ATTR_S1_AP(ATTR_S1_AP_RO) | ATTR_SW_DBM)) {
 		/*
@@ -5871,18 +5868,6 @@ pmap_enter_l3c(pmap_t pmap, vm_offset_t va, pt_entry_t l3e, u_int flags,
 	KASSERT(!VA_IS_CLEANMAP(va) || (l3e & ATTR_SW_MANAGED) == 0,
 	    ("pmap_enter_l3c: managed mapping within the clean submap"));
 
-#ifdef CHERI_CAPREVOKE
-	if (!ADDR_IS_KERNEL(va)) {
-		/*
-		 * pmap_caploadgen_update() currently does not handle L3C
-		 * mappings.  Avoid creating them at all on the basis that the
-		 * pessimization of going through vm_fault() for capload faults
-		 * is probably worse than not having L3C mappings at all. 
-		 */
-		return (KERN_FAILURE);
-	}
-#endif
-
 	/*
 	 * If the L3 PTP is not resident, we attempt to create it here.
 	 */
@@ -6440,6 +6425,7 @@ enum pmap_caploadgen_res
 pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags)
 {
 	enum pmap_caploadgen_res res;
+	struct rwlock *lock;
 #if VM_NRESERVLEVEL > 0
 	pd_entry_t *l2, l2e;
 #endif
@@ -6455,6 +6441,7 @@ pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags)
 	    ("pmap_caploadgen_update: pmap clg %d but CPU mismatch",
 	    (int)pmap->flags.uclg));
 
+retry:
 	pte = pmap_pte(pmap, va, &lvl);
 	if (pte == NULL) {
 		m = NULL;
@@ -6474,24 +6461,38 @@ pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags)
 		break;
 	}
 
-	switch(lvl) {
-	/*
-	 * Large page: bouncing out here means we'll take a VM fault to
-	 * find the page in question.
-	 *
-	 * XXX We'd rather just demote the pages right now, surely?
-	 */
+	switch (lvl) {
+	default:
+		__assert_unreachable();
 	case 1:
-	case 2:
+		/* XXX-MJ we can't really demote shm_largepage mappings */
 		m = NULL;
 		res = PMAP_CAPLOADGEN_UNABLE;
-		goto out;
-	case 3:
-		if ((tpte & ATTR_CONTIGUOUS) != 0) {
+		break;
+	case 2:
+		/*
+		 * Demote superpage and contiguous mappings.  When performing
+		 * load-side revocation, we don't want to pay the latency
+		 * penalty of scanning a large mapping.  Ideally, however, the
+		 * background scan could avoid breaking these mappings.  For
+		 * now, we attempt to reconstruct them below.
+		 */
+		lock = NULL;
+		pte = pmap_demote_l2_locked(pmap, pte, va, &lock);
+		if (lock != NULL)
+			rw_wunlock(lock);
+		if (pte == NULL) {
+			/* Demotion failed. */
 			m = NULL;
 			res = PMAP_CAPLOADGEN_UNABLE;
 			goto out;
 		}
+		goto retry;
+	case 3:
+		if ((tpte & ATTR_CONTIGUOUS) != 0) {
+			pmap_demote_l3c(pmap, pte, va);
+			goto retry;
+		}
 		break;
 	}
 
@@ -6664,15 +6665,17 @@ pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags)
 #if VM_NRESERVLEVEL > 0
 	/*
 	 * If we...
-	 *   are on the background scan (as indicated by EXCLUSIVE),
+	 *   are on the background scan (as indicated by NONEWMAPS),
 	 *   are writing back a PTE (m != NULL),
 	 *   have superpages enabled,
-	 *   are at the last page of an L2 entry,
-	 * then see if we can put a superpage back together.
+	 * try to restore an L3C mapping.  If that succeeded, then see if we can
+	 * put a superpage back together.
 	 */
 	if ((flags & PMAP_CAPLOADGEN_NONEWMAPS) &&
-	    (m != NULL) && pmap_ps_enabled(pmap) &&
-	    ((va & (L2_OFFSET - L3_OFFSET)) == (L2_OFFSET - L3_OFFSET))) {
+	    m != NULL && pmap_ps_enabled(pmap) &&
+	    (va & L3C_OFFSET) == (PTE_TO_PHYS(tpte) & L3C_OFFSET) &&
+	    vm_reserv_is_populated(m, L3C_ENTRIES) &&
+	    pmap_promote_l3c(pmap, pte, va)) {
 		KASSERT(lvl == 3,
 		    ("pmap_caploadgen_update superpage: lvl != 3"));
 		KASSERT((m->flags & PG_FICTITIOUS) == 0,
@@ -6719,7 +6722,7 @@ pmap_assert_consistent_clg(pmap_t pmap, vm_offset_t va)
 
 	pte = pmap_pte(pmap, va, &level);
 	if (pte == NULL)
-		return;	/* XXX: why does this happen? */
+		return;
 	tpte = pmap_load(pte);
 	if ((tpte & ATTR_SW_MANAGED) == 0 || !pmap_pte_capdirty(pmap, tpte))
 		return;
@@ -6729,10 +6732,12 @@ pmap_assert_consistent_clg(pmap_t pmap, vm_offset_t va)
 	case ATTR_LC_ENABLED:
 		panic("no clg");
 	case ATTR_LC_GEN0:
-		KASSERT(pmap->flags.uclg == 0, ("GEN0 LCLG with GEN1 GCLG"));
+		KASSERT(pmap->flags.uclg == 0,
+		    ("GEN0 LCLG with GEN1 GCLG (%#lx)", tpte));
 		break;
 	case ATTR_LC_GEN1:
-		KASSERT(pmap->flags.uclg == 1, ("GEN1 LCLG with GEN0 GCLG"));
+		KASSERT(pmap->flags.uclg == 1,
+		    ("GEN1 LCLG with GEN0 GCLG (%#lx)", tpte));
 		break;
 	default:
 		panic("impossible?");
@@ -9194,6 +9199,9 @@ pmap_demote_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va)
 		    (ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_SW_DBM))
 			mask = ATTR_S1_AP_RW_BIT;
 		nbits |= l3e & ATTR_AF;
+#if __has_feature(capabilities)
+		nbits |= l3e & ATTR_SC;
+#endif
 	}
 	if ((nbits & ATTR_AF) != 0) {
 		pmap_invalidate_range(pmap, va & ~L3C_OFFSET, (va + L3C_SIZE) &

sys/kern/kern_cheri_revoke.c

@@ -724,9 +724,6 @@ kern_cheri_revoke(struct thread *td, int flags,
 
 	/* OK, that's that.  Where do we stand now? */
 	if (res == KERN_SUCCESS && myst == CHERI_REVOKE_ST_CLOSING) {
-#ifdef DIAGNOSTIC
-		vm_cheri_assert_consistent_clg(&vm->vm_map);
-#endif
 		/* Signal the end of this revocation epoch */
 		epoch++;
 		crepochs.dequeue = epoch;
@@ -738,6 +735,8 @@ kern_cheri_revoke(struct thread *td, int flags,
 
 	vm_map_lock(vmm);
 	cheri_revoke_st_set(&vmm->vm_cheri_revoke_st, epoch, myst);
+	if (res == KERN_SUCCESS)
+		vm_cheri_assert_consistent_clg(&vm->vm_map);
 #ifdef CHERI_CAPREVOKE_STATS
 	if (flags & CHERI_REVOKE_TAKE_STATS) {
 		sx_xlock(&vmm->vm_cheri_revoke_stats_sx);

sys/riscv/riscv/pmap.c

@@ -3875,32 +3875,45 @@ enum pmap_caploadgen_res
 pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags)
 {
 	enum pmap_caploadgen_res res;
+	struct rwlock *lock;
 	pd_entry_t *l2, l2e;
 	pt_entry_t *pte, oldpte = 0;
 	vm_page_t m;
 
+	rw_rlock(&pvh_global_lock);
 	PMAP_LOCK(pmap);
 
 	KASSERT(!(csr_read(sccsr) & SCCSR_UGCLG) == !(pmap->flags.uclg),
 	    ("pmap_caploadgen_update: pmap crg %d but CPU mismatch",
 	    (int)pmap->flags.uclg));
 
+retry:
 	l2 = pmap_l2(pmap, va);
 	if (l2 == NULL || ((l2e = pmap_load(l2)) & PTE_V) == 0) {
 		m = NULL;
 		res = PMAP_CAPLOADGEN_UNABLE;
 		goto out;
 	}
 	if ((l2e & PTE_RWX) != 0) {
+		bool demoted;
+
 		/*
-		 * Large page: bouncing out here means we'll take a VM fault
-		 * to find the page in question.
-		 *
-		 * XXX We'd rather just demote the L2 page right now, surely?
+		 * Demote superpage and contiguous mappings.  When performing
+		 * load-side revocation, we don't want to pay the latency
+		 * penalty of scanning a large mapping.  Ideally, however, the
+		 * background scan could avoid breaking these mappings.  For
+		 * now, we attempt to reconstruct them below.
 		 */
-		m = NULL;
-		res = PMAP_CAPLOADGEN_UNABLE;
-		goto out;
+		lock = NULL;
+		demoted = pmap_demote_l2_locked(pmap, l2, va, &lock);
+		if (lock != NULL)
+			rw_wunlock(lock);
+		if (!demoted) {
+			m = NULL;
+			res = PMAP_CAPLOADGEN_UNABLE;
+			goto out;
+		}
+		goto retry;
 	}
 	pte = pmap_l2_to_l3(l2, va);
 	if (pte == NULL || ((oldpte = pmap_load(pte)) & PTE_V) == 0) {
@@ -3998,6 +4011,7 @@ pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags)
 						sfence_vma_page(va);
 					}
 					PMAP_UNLOCK(pmap);
+					rw_runlock(&pvh_global_lock);
 					pmap_caploadgen_test_all_clean(m);
 					m = NULL;
 					goto out_unlocked;
@@ -4121,6 +4135,7 @@ pmap_caploadgen_update(pmap_t pmap, vm_offset_t va, vm_page_t *mp, int flags)
 #endif /* VM_NRESERVLEVEL > 0 */
 
 	PMAP_UNLOCK(pmap);
+	rw_runlock(&pvh_global_lock);
 out_unlocked:
 
 	if (*mp != NULL) {
@@ -4156,9 +4171,11 @@ pmap_assert_consistent_clg(pmap_t pmap, vm_offset_t va)
 	if ((tpte & PTE_CW) == 0)
 		return;
 	if ((tpte & PTE_CRG) == 0) {
-		KASSERT(pmap->flags.uclg == 0, ("PTR_CRG unset, but GCLG set"));
+		KASSERT(pmap->flags.uclg == 0,
+		    ("PTR_CRG unset, but GCLG set (%#lx)", tpte));
 	} else {
-		KASSERT(pmap->flags.uclg == 1, ("PTR_CRG set, but GCLG unset"));
+		KASSERT(pmap->flags.uclg == 1,
+		    ("PTR_CRG set, but GCLG unset (%#lx)", tpte));
 	}
 }
 #endif /* CHERI_CAPREVOKE */

sys/vm/vm_cheri_revoke.c

@@ -508,6 +508,15 @@ SYSCTL_BOOL(_vm_cheri_revoke, OID_AUTO, avoid_faults, CTLFLAG_RWTUN,
     &cheri_revoke_avoid_faults, 0,
     "Avoid faulting when the pager is known not to contain the page");
 
+#ifdef DIAGNOSTIC
+static bool cheri_validate_clg = true;
+#else
+static bool cheri_validate_clg = false;
+#endif
+SYSCTL_BOOL(_vm_cheri_revoke, OID_AUTO, validate_clg, CTLFLAG_RWTUN,
+    &cheri_validate_clg, 0,
+    "Validate LCLGs against the pmap after forks and revocation scans");
+
 static bool
 vm_cheri_revoke_skip_fault(vm_object_t object)
 {
@@ -1104,15 +1113,22 @@ vm_cheri_assert_consistent_clg(struct vm_map *map)
 {
 	pmap_t pmap = map->pmap;
 	vm_map_entry_t entry;
+	vm_object_t object;
 	vm_offset_t addr;
 
 	/* Called with map lock held */
 
+	if (!cheri_validate_clg)
+		return;
+	if (cheri_revoke_st_is_revoking(map->vm_cheri_revoke_st))
+		return;
 	VM_MAP_ENTRY_FOREACH(entry, map) {
-		if ((entry->max_protection & VM_PROT_READ_CAP) == 0 ||
-		    entry->object.vm_object == NULL) {
+		if ((entry->max_protection & VM_PROT_READ_CAP) == 0)
+			continue;
+		object = entry->object.vm_object;
+		if (object == NULL || (object->flags & OBJ_HASCAP) == 0 ||
+		    (object->flags & OBJ_NOCAP) != 0)
 			continue;
-		}
 
 		for (addr = entry->start; addr < entry->end;
 		    addr += pagesizes[0]) {

sys/vm/vm_map.c

@@ -5342,10 +5342,10 @@
 #endif
 		}
 	}
-#if defined(CHERI_CAPREVOKE) && defined(DIAGNOSTIC)
+#if defined(CHERI_CAPREVOKE)
 	vm_cheri_assert_consistent_clg(new_map);
 #endif
 	/*
 	 * Use inlined vm_map_unlock() to postpone handling the deferred
 	 * map entries, which cannot be done until both old_map and
 	 * new_map locks are released.