diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c --- a/sys/dev/nvme/nvme_ctrlr.c +++ b/sys/dev/nvme/nvme_ctrlr.c @@ -41,6 +41,13 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include #include "nvme_private.h" #include "nvme_linux.h" @@ -1268,6 +1275,44 @@ nvme_mmio_write_4(ctrlr, intmc, 1); } +#define NVME_MAX_PAGES (int)(1024 / sizeof(vm_page_t)) + +static int +nvme_npages(vm_offset_t addr, size_t len) +{ + return (atop(round_page(addr + len) - trunc_page(addr))); +} + +static struct nvme_request * +nvme_user_ioctl_req(vm_offset_t addr, size_t len, bool is_read, + vm_page_t *upages, int max_pages, + nvme_cb_fn_t cb_fn, void *cb_arg) +{ + vm_prot_t prot = VM_PROT_READ; + struct nvme_request *req; + int npages __diagused; + + MPASS(max_pages <= NVME_MAX_PAGES); + MPASS(nvme_npages(addr, len) == max_pages); + if (is_read) + prot |= VM_PROT_WRITE; /* Less backwards than it looks */ + npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, + addr, len, prot, upages, max_pages); + if (npages < 0) + return (NULL); + MPASS(max_pages == npages); /* We've computed exactly */ + req = nvme_allocate_request_null(M_WAITOK, cb_fn, cb_arg); + req->payload = memdesc_vmpages(upages, len, addr & PAGE_MASK); + req->payload_valid = true; + return (req); +} + +static void +nvme_user_ioctl_free(vm_page_t *pages, int npage) +{ + vm_page_unhold_pages(pages, npage); +} + static void nvme_pt_done(void *arg, const struct nvme_completion *cpl) { @@ -1290,30 +1335,35 @@ int nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr, - struct nvme_pt_command *pt, uint32_t nsid, int is_user_buffer, + struct nvme_pt_command *pt, uint32_t nsid, int is_user, int is_admin_cmd) { - struct nvme_request *req; - struct mtx *mtx; - struct buf *buf = NULL; - int ret = 0; + struct nvme_request *req; + struct mtx *mtx; + int ret = 0; + int max_pages = nvme_npages((vm_offset_t)pt->buf, pt->len); + vm_page_t upages[is_user ? min(max_pages, NVME_MAX_PAGES) : 1]; + bool release = false; if (pt->len > 0) { if (pt->len > ctrlr->max_xfer_size) { nvme_printf(ctrlr, "pt->len (%d) " "exceeds max_xfer_size (%d)\n", pt->len, ctrlr->max_xfer_size); - return EIO; + return (EIO); } - if (is_user_buffer) { - buf = uma_zalloc(pbuf_zone, M_WAITOK); - buf->b_iocmd = pt->is_read ? BIO_READ : BIO_WRITE; - if (vmapbuf(buf, pt->buf, pt->len, 0) < 0) { - ret = EFAULT; - goto err; + if (is_user) { + if (max_pages > NVME_MAX_PAGES) { + nvme_printf(ctrlr, "pt->len (%d) " + "exceeds max_user pages (%d)\n", pt->len, + ptoa(NVME_MAX_PAGES)); + return (EIO); } - req = nvme_allocate_request_buf(buf, M_WAITOK, - nvme_pt_done, pt); + req = nvme_user_ioctl_req((vm_offset_t)pt->buf, pt->len, + pt->is_read, upages, max_pages, nvme_pt_done, pt); + if (req == NULL) + return (EFAULT); + release = true; } else req = nvme_allocate_request_vaddr(pt->buf, pt->len, M_WAITOK, nvme_pt_done, pt); @@ -1347,11 +1397,8 @@ mtx_sleep(pt, mtx, PRIBIO, "nvme_pt", 0); mtx_unlock(mtx); - if (buf != NULL) { - vunmapbuf(buf); -err: - uma_zfree(pbuf_zone, buf); - } + if (release) + nvme_user_ioctl_free(upages, max_pages); return (ret); } @@ -1377,8 +1424,10 @@ { struct nvme_request *req; struct mtx *mtx; - struct buf *buf = NULL; int ret = 0; + int max_pages = nvme_npages(npc->addr, npc->data_len); + vm_page_t upages[is_user ? min(max_pages, NVME_MAX_PAGES) : 1]; + bool release = false; /* * We don't support metadata. @@ -1402,15 +1451,17 @@ if ((npc->opcode & 0x3) == 3) return (EINVAL); if (is_user) { - buf = uma_zalloc(pbuf_zone, M_WAITOK); - buf->b_iocmd = npc->opcode & 1 ? BIO_WRITE : BIO_READ; - if (vmapbuf(buf, (void *)(uintptr_t)npc->addr, - npc->data_len, 0) < 0) { - ret = EFAULT; - goto err; + if (max_pages > NVME_MAX_PAGES) { + nvme_printf(ctrlr, "data_len (%d) " + "exceeds max_user pages (%d)\n", npc->data_len, + ptoa(NVME_MAX_PAGES)); + return (EIO); } - req = nvme_allocate_request_buf(buf, M_WAITOK, - nvme_npc_done, npc); + req = nvme_user_ioctl_req(npc->addr, npc->data_len, npc->opcode & 0x1, + upages, max_pages, nvme_npc_done, npc); + if (req == NULL) + return (EFAULT); + release = true; } else req = nvme_allocate_request_vaddr( (void *)(uintptr_t)npc->addr, npc->data_len, @@ -1445,11 +1496,8 @@ mtx_sleep(npc, mtx, PRIBIO, "nvme_npc", 0); mtx_unlock(mtx); - if (buf != NULL) { - vunmapbuf(buf); -err: - uma_zfree(pbuf_zone, buf); - } + if (release) + nvme_user_ioctl_free(upages, max_pages); return (ret); }