diff --git a/sys/sys/param.h b/sys/sys/param.h --- a/sys/sys/param.h +++ b/sys/sys/param.h @@ -73,7 +73,7 @@ * cannot include sys/param.h and should only be updated here. */ #undef __FreeBSD_version -#define __FreeBSD_version 1500045 +#define __FreeBSD_version 1500046 /* * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD, diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -229,7 +229,6 @@ void *zone; } uma; } plinks; - TAILQ_ENTRY(vm_page) listq; /* pages in same object (O) */ vm_object_t object; /* which object am I in (O) */ vm_pindex_t pindex; /* offset into object (O,P) */ vm_paddr_t phys_addr; /* physical address of page (C) */ diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -341,7 +341,7 @@ vm_domain_free_unlock(vmd); if (found) { vm_domain_freecnt_inc(vmd, -1); - TAILQ_INSERT_TAIL(&blacklist_head, m, listq); + TAILQ_INSERT_TAIL(&blacklist_head, m, plinks.q); if (verbose) printf("Skipping page with pa 0x%jx\n", (uintmax_t)pa); } @@ -411,7 +411,7 @@ if (error != 0) return (error); sbuf_new_for_sysctl(&sbuf, NULL, 128, req); - TAILQ_FOREACH(m, &blacklist_head, listq) { + TAILQ_FOREACH(m, &blacklist_head, plinks.q) { sbuf_printf(&sbuf, "%s%#jx", first ? "" : ",", (uintmax_t)m->phys_addr); first = 0; @@ -2470,6 +2470,13 @@ } found: + /* + * If the page comes from the free page cache, then it might still + * have a pending deferred dequeue. Specifically, when the page is + * imported from a different pool by vm_phys_alloc_npages(), the + * second, third, etc. pages in a non-zero order set could have + * pending deferred dequeues. + */ vm_page_dequeue(m); vm_page_alloc_check(m); @@ -2536,17 +2543,18 @@ return (NULL); } m->ref_count = count - 1; - TAILQ_INSERT_HEAD(&vmd->vmd_nofreeq, m, listq); + TAILQ_INSERT_HEAD(&vmd->vmd_nofreeq, m, plinks.q); VM_CNT_ADD(v_nofree_count, count); } m = TAILQ_FIRST(&vmd->vmd_nofreeq); - TAILQ_REMOVE(&vmd->vmd_nofreeq, m, listq); + TAILQ_REMOVE(&vmd->vmd_nofreeq, m, plinks.q); if (m->ref_count > 0) { vm_page_t m_next; m_next = &m[1]; + vm_page_dequeue(m_next); m_next->ref_count = m->ref_count - 1; - TAILQ_INSERT_HEAD(&vmd->vmd_nofreeq, m_next, listq); + TAILQ_INSERT_HEAD(&vmd->vmd_nofreeq, m_next, plinks.q); m->ref_count = 0; } vm_domain_free_unlock(vmd); @@ -2566,7 +2574,7 @@ { vm_domain_free_lock(vmd); MPASS(m->ref_count == 0); - TAILQ_INSERT_HEAD(&vmd->vmd_nofreeq, m, listq); + TAILQ_INSERT_HEAD(&vmd->vmd_nofreeq, m, plinks.q); vm_domain_free_unlock(vmd); VM_CNT_ADD(v_nofree_count, 1); } @@ -3971,7 +3979,7 @@ old = vm_page_astate_load(m); do { - if (old.queue == PQ_NONE) { + if (__predict_true(old.queue == PQ_NONE)) { KASSERT((old.flags & PGA_QUEUE_STATE_MASK) == 0, ("%s: page %p has unexpected queue state", __func__, m)); diff --git a/sys/vm/vm_phys.c b/sys/vm/vm_phys.c --- a/sys/vm/vm_phys.c +++ b/sys/vm/vm_phys.c @@ -393,13 +393,23 @@ vm_freelist_add(struct vm_freelist *fl, vm_page_t m, int order, int pool, int tail) { + /* + * The paging queues and the free page lists utilize the same field, + * plinks.q, within the vm_page structure. When a physical page is + * freed, it is lazily removed from the paging queues to reduce the + * cost of removal through batching. Here, we must ensure that any + * deferred dequeue on the physical page has completed before using + * its plinks.q field. + */ + if (__predict_false(vm_page_astate_load(m).queue != PQ_NONE)) + vm_page_dequeue(m); m->order = order; m->pool = pool; if (tail) - TAILQ_INSERT_TAIL(&fl[order].pl, m, listq); + TAILQ_INSERT_TAIL(&fl[order].pl, m, plinks.q); else - TAILQ_INSERT_HEAD(&fl[order].pl, m, listq); + TAILQ_INSERT_HEAD(&fl[order].pl, m, plinks.q); fl[order].lcnt++; } @@ -407,7 +417,7 @@ vm_freelist_rem(struct vm_freelist *fl, vm_page_t m, int order) { - TAILQ_REMOVE(&fl[order].pl, m, listq); + TAILQ_REMOVE(&fl[order].pl, m, plinks.q); fl[order].lcnt--; m->order = VM_NFREEORDER; } @@ -1582,7 +1592,7 @@ * check if there are enough free blocks starting at a properly aligned * block. Thus, no block is checked for free-ness more than twice. */ - TAILQ_FOREACH(m, &fl[max_order].pl, listq) { + TAILQ_FOREACH(m, &fl[max_order].pl, plinks.q) { /* * Skip m unless it is first in a sequence of free max page * blocks >= low in its segment. @@ -1655,7 +1665,7 @@ for (oind = order; oind < VM_NFREEORDER; oind++) { for (pind = vm_default_freepool; pind < VM_NFREEPOOL; pind++) { fl = (*queues)[pind]; - TAILQ_FOREACH(m_ret, &fl[oind].pl, listq) { + TAILQ_FOREACH(m_ret, &fl[oind].pl, plinks.q) { /* * Determine if the address range starting at pa * is within the given range, satisfies the