Index: sys/amd64/amd64/pmap.c =================================================================== --- sys/amd64/amd64/pmap.c +++ sys/amd64/amd64/pmap.c @@ -474,11 +474,11 @@ static void pmap_update_pde_invalidate(pmap_t, vm_offset_t va, pd_entry_t pde); static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, - struct rwlock **lockp); + struct rw_priotracker *prio, struct rwlock **lockp); static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va, - struct rwlock **lockp); + struct rw_priotracker *prio, struct rwlock **lockp); static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, - struct rwlock **lockp); + struct rw_priotracker *prio, struct rwlock **lockp); static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free); @@ -888,7 +888,7 @@ /* * Initialize the global pv list lock. */ - rw_init(&pvh_global_lock, "pmap pv global"); + rw_init_flags(&pvh_global_lock, "pmap pv global", RW_SHARED_PRIO); /* * Reserve some special page table entries/VA space for temporary @@ -2293,7 +2293,8 @@ * race conditions. */ static vm_page_t -_pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) +_pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rw_priotracker *prio, + struct rwlock **lockp) { vm_page_t m, pdppg, pdpg; pt_entry_t PG_A, PG_M, PG_RW, PG_V; @@ -2313,9 +2314,9 @@ if (lockp != NULL) { RELEASE_PV_LIST_LOCK(lockp); PMAP_UNLOCK(pmap); - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, prio); VM_WAIT; - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, prio); PMAP_LOCK(pmap); } @@ -2356,7 +2357,7 @@ if ((*pml4 & PG_V) == 0) { /* Have to allocate a new pdp, recurse */ if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml4index, - lockp) == NULL) { + prio, lockp) == NULL) { --m->wire_count; atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); @@ -2389,7 +2390,7 @@ if ((*pml4 & PG_V) == 0) { /* Have to allocate a new pd, recurse */ if (_pmap_allocpte(pmap, NUPDE + pdpindex, - lockp) == NULL) { + prio, lockp) == NULL) { --m->wire_count; atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); @@ -2403,7 +2404,7 @@ if ((*pdp & PG_V) == 0) { /* Have to allocate a new pd, recurse */ if (_pmap_allocpte(pmap, NUPDE + pdpindex, - lockp) == NULL) { + prio, lockp) == NULL) { --m->wire_count; atomic_subtract_int(&vm_cnt.v_wire_count, 1); @@ -2429,7 +2430,8 @@ } static vm_page_t -pmap_allocpde(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) +pmap_allocpde(pmap_t pmap, vm_offset_t va, struct rw_priotracker *prio, + struct rwlock **lockp) { vm_pindex_t pdpindex, ptepindex; pdp_entry_t *pdpe, PG_V; @@ -2447,7 +2449,7 @@ /* Allocate a pd page. */ ptepindex = pmap_pde_pindex(va); pdpindex = ptepindex >> NPDPEPGSHIFT; - pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp); + pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, prio, lockp); if (pdpg == NULL && lockp != NULL) goto retry; } @@ -2455,7 +2457,8 @@ } static vm_page_t -pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) +pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rw_priotracker *prio, + struct rwlock **lockp) { vm_pindex_t ptepindex; pd_entry_t *pd, PG_V; @@ -2499,7 +2502,7 @@ * Here if the pte page isn't mapped, or if it has been * deallocated. */ - m = _pmap_allocpte(pmap, ptepindex, lockp); + m = _pmap_allocpte(pmap, ptepindex, prio, lockp); if (m == NULL && lockp != NULL) goto retry; } @@ -3594,6 +3597,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { struct rwlock *lock; + struct rw_priotracker tracker; vm_offset_t va, va_next; pml4_entry_t *pml4e; pdp_entry_t *pdpe; @@ -3614,7 +3618,7 @@ anyvalid = 0; SLIST_INIT(&free); - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); PMAP_LOCK(pmap); /* @@ -3729,7 +3733,7 @@ out: if (anyvalid) pmap_invalidate_all(pmap); - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); PMAP_UNLOCK(pmap); pmap_free_zero_pages(&free); } @@ -3859,6 +3863,7 @@ void pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { + struct rw_priotracker tracker; vm_offset_t va_next; pml4_entry_t *pml4e; pdp_entry_t *pdpe; @@ -3935,12 +3940,12 @@ } else { if (!pv_lists_locked) { pv_lists_locked = TRUE; - if (!rw_try_rlock(&pvh_global_lock)) { + if (!rw_try_rlock_prio(&pvh_global_lock, &tracker)) { if (anychanged) pmap_invalidate_all( pmap); PMAP_UNLOCK(pmap); - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); goto resume; } } @@ -3991,7 +3996,7 @@ if (anychanged) pmap_invalidate_all(pmap); if (pv_lists_locked) - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); PMAP_UNLOCK(pmap); } @@ -4141,6 +4146,7 @@ u_int flags, int8_t psind __unused) { struct rwlock *lock; + struct rw_priotracker tracker; pd_entry_t *pde; pt_entry_t *pte, PG_G, PG_A, PG_M, PG_RW, PG_V; pt_entry_t newpte, origpte; @@ -4196,7 +4202,7 @@ mpte = NULL; lock = NULL; - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); PMAP_LOCK(pmap); /* @@ -4218,12 +4224,12 @@ * deallocated. */ nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; - mpte = _pmap_allocpte(pmap, pmap_pde_pindex(va), + mpte = _pmap_allocpte(pmap, pmap_pde_pindex(va), &tracker, nosleep ? NULL : &lock); if (mpte == NULL && nosleep) { if (lock != NULL) rw_wunlock(lock); - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); PMAP_UNLOCK(pmap); return (KERN_RESOURCE_SHORTAGE); } @@ -4356,7 +4362,7 @@ if (lock != NULL) rw_wunlock(lock); - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } @@ -4380,7 +4386,7 @@ rw_assert(&pvh_global_lock, RA_LOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); - if ((mpde = pmap_allocpde(pmap, va, NULL)) == NULL) { + if ((mpde = pmap_allocpde(pmap, va, NULL, NULL)) == NULL) { CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" " in pmap %p", va, pmap); return (FALSE); @@ -4452,6 +4458,7 @@ pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, vm_page_t m_start, vm_prot_t prot) { + struct rw_priotracker tracker; struct rwlock *lock; vm_offset_t va; vm_page_t m, mpte; @@ -4463,7 +4470,7 @@ mpte = NULL; m = m_start; lock = NULL; - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); PMAP_LOCK(pmap); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { va = start + ptoa(diff); @@ -4478,7 +4485,7 @@ } if (lock != NULL) rw_wunlock(lock); - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); PMAP_UNLOCK(pmap); } @@ -4494,15 +4501,16 @@ void pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) { + struct rw_priotracker tracker; struct rwlock *lock; lock = NULL; - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); PMAP_LOCK(pmap); (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock); if (lock != NULL) rw_wunlock(lock); - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); PMAP_UNLOCK(pmap); } @@ -4557,7 +4565,8 @@ * Pass NULL instead of the PV list lock * pointer, because we don't intend to sleep. */ - mpte = _pmap_allocpte(pmap, ptepindex, NULL); + mpte = _pmap_allocpte(pmap, ptepindex, NULL, + NULL); if (mpte == NULL) return (mpte); } @@ -4691,7 +4700,7 @@ PMAP_LOCK(pmap); for (pa = ptepa | pmap_cache_bits(pmap, pat_mode, 1); pa < ptepa + size; pa += NBPDR) { - pdpg = pmap_allocpde(pmap, addr, NULL); + pdpg = pmap_allocpde(pmap, addr, NULL, NULL); if (pdpg == NULL) { /* * The creation of mappings below is only an @@ -4735,6 +4744,7 @@ void pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { + struct rw_priotracker tracker; vm_offset_t va_next; pml4_entry_t *pml4e; pdp_entry_t *pdpe; @@ -4784,9 +4794,9 @@ } else { if (!pv_lists_locked) { pv_lists_locked = TRUE; - if (!rw_try_rlock(&pvh_global_lock)) { + if (!rw_try_rlock_prio(&pvh_global_lock, &tracker)) { PMAP_UNLOCK(pmap); - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); /* Repeat sva. */ goto resume; } @@ -4815,7 +4825,7 @@ } } if (pv_lists_locked) - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); PMAP_UNLOCK(pmap); } @@ -4831,6 +4841,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr) { + struct rw_priotracker tracker; struct rwlock *lock; struct spglist free; vm_offset_t addr; @@ -4856,7 +4867,7 @@ return; lock = NULL; - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); if (dst_pmap < src_pmap) { PMAP_LOCK(dst_pmap); PMAP_LOCK(src_pmap); @@ -4907,7 +4918,7 @@ if (srcptepaddr & PG_PS) { if ((addr & PDRMASK) != 0 || addr + NBPDR > end_addr) continue; - dstmpde = pmap_allocpde(dst_pmap, addr, NULL); + dstmpde = pmap_allocpde(dst_pmap, addr, NULL, NULL); if (dstmpde == NULL) break; pde = (pd_entry_t *) @@ -4946,7 +4957,7 @@ dstmpte->pindex == pmap_pde_pindex(addr)) dstmpte->wire_count++; else if ((dstmpte = pmap_allocpte(dst_pmap, - addr, NULL)) == NULL) + addr, NULL, NULL)) == NULL) goto out; dst_pte = (pt_entry_t *) PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte)); @@ -4983,7 +4994,7 @@ out: if (lock != NULL) rw_wunlock(lock); - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); PMAP_UNLOCK(src_pmap); PMAP_UNLOCK(dst_pmap); } @@ -5087,6 +5098,7 @@ boolean_t pmap_page_exists_quick(pmap_t pmap, vm_page_t m) { + struct rw_priotracker tracker; struct md_page *pvh; struct rwlock *lock; pv_entry_t pv; @@ -5096,7 +5108,7 @@ KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_page_exists_quick: page %p is not managed", m)); rv = FALSE; - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_rlock(lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { @@ -5121,7 +5133,7 @@ } } rw_runlock(lock); - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); return (rv); } @@ -5134,6 +5146,7 @@ int pmap_page_wired_mappings(vm_page_t m) { + struct rw_priotracker tracker; struct rwlock *lock; struct md_page *pvh; pmap_t pmap; @@ -5143,7 +5156,7 @@ if ((m->oflags & VPO_UNMANAGED) != 0) return (0); - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_rlock(lock); restart: @@ -5188,7 +5201,7 @@ } } rw_runlock(lock); - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); return (count); } @@ -5199,19 +5212,20 @@ boolean_t pmap_page_is_mapped(vm_page_t m) { + struct rw_priotracker tracker; struct rwlock *lock; boolean_t rv; if ((m->oflags & VPO_UNMANAGED) != 0) return (FALSE); - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_rlock(lock); rv = !TAILQ_EMPTY(&m->md.pv_list) || ((m->flags & PG_FICTITIOUS) == 0 && !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list)); rw_runlock(lock); - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); return (rv); } @@ -5234,6 +5248,7 @@ void pmap_remove_pages(pmap_t pmap) { + struct rw_priotracker tracker; pd_entry_t ptepde; pt_entry_t *pte, tpte; pt_entry_t PG_M, PG_RW, PG_V; @@ -5274,7 +5289,7 @@ PG_RW = pmap_rw_bit(pmap); SLIST_INIT(&free); - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); PMAP_LOCK(pmap); TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { allfree = 1; @@ -5407,7 +5422,7 @@ if (lock != NULL) rw_wunlock(lock); pmap_invalidate_all(pmap); - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); PMAP_UNLOCK(pmap); pmap_free_zero_pages(&free); } @@ -5415,6 +5430,7 @@ static boolean_t pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) { + struct rw_priotracker tracker; struct rwlock *lock; pv_entry_t pv; struct md_page *pvh; @@ -5425,7 +5441,7 @@ boolean_t rv; rv = FALSE; - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_rlock(lock); restart: @@ -5494,7 +5510,7 @@ } out: rw_runlock(lock); - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); return (rv); } @@ -5568,6 +5584,7 @@ void pmap_remove_write(vm_page_t m) { + struct rw_priotracker tracker; struct md_page *pvh; pmap_t pmap; struct rwlock *lock; @@ -5588,7 +5605,7 @@ VM_OBJECT_ASSERT_WLOCKED(m->object); if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) return; - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); lock = VM_PAGE_TO_PV_LIST_LOCK(m); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); retry_pv_loop: @@ -5655,7 +5672,7 @@ } rw_wunlock(lock); vm_page_aflag_clear(m, PGA_WRITEABLE); - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); } static __inline boolean_t @@ -5702,6 +5719,7 @@ int pmap_ts_referenced(vm_page_t m) { + struct rw_priotracker tracker; struct md_page *pvh; pv_entry_t pv, pvf; pmap_t pmap; @@ -5721,7 +5739,7 @@ pa = VM_PAGE_TO_PHYS(m); lock = PHYS_TO_PV_LIST_LOCK(pa); pvh = pa_to_pvh(pa); - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); rw_wlock(lock); retry: not_cleared = 0; @@ -5881,7 +5899,7 @@ not_cleared < PMAP_TS_REFERENCED_MAX); out: rw_wunlock(lock); - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); pmap_free_zero_pages(&free); return (cleared + not_cleared); } @@ -5894,6 +5912,7 @@ void pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) { + struct rw_priotracker tracker; struct rwlock *lock; pml4_entry_t *pml4e; pdp_entry_t *pdpe; @@ -5952,11 +5971,11 @@ continue; if (!pv_lists_locked) { pv_lists_locked = TRUE; - if (!rw_try_rlock(&pvh_global_lock)) { + if (!rw_try_rlock_prio(&pvh_global_lock, &tracker)) { if (anychanged) pmap_invalidate_all(pmap); PMAP_UNLOCK(pmap); - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); goto resume; } } @@ -6020,7 +6039,7 @@ if (anychanged) pmap_invalidate_all(pmap); if (pv_lists_locked) - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); PMAP_UNLOCK(pmap); } @@ -6030,6 +6049,7 @@ void pmap_clear_modify(vm_page_t m) { + struct rw_priotracker tracker; struct md_page *pvh; pmap_t pmap; pv_entry_t next_pv, pv; @@ -6053,7 +6073,7 @@ if ((m->aflags & PGA_WRITEABLE) == 0) return; pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_wlock(lock); restart: @@ -6129,7 +6149,7 @@ PMAP_UNLOCK(pmap); } rw_wunlock(lock); - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); } /* @@ -6826,6 +6846,7 @@ int pmap_emulate_accessed_dirty(pmap_t pmap, vm_offset_t va, int ftype) { + struct rw_priotracker tracker; int rv; struct rwlock *lock; vm_page_t m, mpte; @@ -6900,9 +6921,9 @@ vm_reserv_level_iffullpop(m) == 0) { if (!pv_lists_locked) { pv_lists_locked = TRUE; - if (!rw_try_rlock(&pvh_global_lock)) { + if (!rw_try_rlock_prio(&pvh_global_lock, &tracker)) { PMAP_UNLOCK(pmap); - rw_rlock(&pvh_global_lock); + rw_rlock_prio(&pvh_global_lock, &tracker); goto retry; } } @@ -6922,7 +6943,7 @@ if (lock != NULL) rw_wunlock(lock); if (pv_lists_locked) - rw_runlock(&pvh_global_lock); + rw_runlock_prio(&pvh_global_lock, &tracker); PMAP_UNLOCK(pmap); return (rv); } Index: sys/kern/subr_witness.c =================================================================== --- sys/kern/subr_witness.c +++ sys/kern/subr_witness.c @@ -610,7 +610,7 @@ { "vm object", &lock_class_rw }, { "vm page", &lock_class_mtx_sleep }, { "vm page queue", &lock_class_mtx_sleep }, - { "pmap pv global", &lock_class_rw }, + { "pmap pv global", &lock_class_rw_shared_prio }, { "pmap", &lock_class_mtx_sleep }, { "pmap pv list", &lock_class_rw }, { "vm page free queue", &lock_class_mtx_sleep },