diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c --- a/sys/dev/nvme/nvme_ctrlr.c +++ b/sys/dev/nvme/nvme_ctrlr.c @@ -41,6 +41,9 @@ #include #include #include +#include +#include +#include #include "nvme_private.h" #include "nvme_linux.h" @@ -1268,6 +1271,46 @@ nvme_mmio_write_4(ctrlr, intmc, 1); } +#define NVME_MAX_PAGES (int)(1024 / sizeof(vm_page_t)) + +static int +nvme_npages(vm_offset_t addr, size_t len) +{ + return (atop(round_page(addr + len) - trunc_page(addr))); +} + +static struct nvme_request * +nvme_user_ioctl_req(vm_offset_t addr, size_t len, bool is_read, + vm_page_t *upages, int *max_pages, + nvme_cb_fn_t cb_fn, void *cb_arg) +{ + vm_prot_t prot = VM_PROT_READ; + struct nvme_request *req; + + /* + * Make sure we have enough pages, vm_fault_quick_hold_pages will panic + * if there's not enough room. + */ + if (*max_pages < nvme_npages(addr, len)) + return (NULL); + if (is_read) + prot |= VM_PROT_WRITE; /* Less backwards than it looks */ + *max_pages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, + addr, len, prot, upages, *max_pages); + if (*max_pages < 0) + return (NULL); + req = nvme_allocate_request_null(M_WAITOK, cb_fn, cb_arg); + req->payload = memdesc_vmpages(upages, len, addr & PAGE_MASK); + req->payload_valid = true; + return (req); +} + +static void +nvme_user_ioctl_free(vm_page_t *pages, int npage) +{ + vm_page_unhold_pages(pages, npage); +} + static void nvme_pt_done(void *arg, const struct nvme_completion *cpl) { @@ -1290,30 +1333,31 @@ int nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr, - struct nvme_pt_command *pt, uint32_t nsid, int is_user_buffer, + struct nvme_pt_command *pt, uint32_t nsid, int is_user, int is_admin_cmd) { - struct nvme_request *req; - struct mtx *mtx; - struct buf *buf = NULL; - int ret = 0; + struct nvme_request *req; + struct mtx *mtx; + int ret = 0; + int npages = 0; + vm_page_t upages[NVME_MAX_PAGES]; if (pt->len > 0) { - if (pt->len > ctrlr->max_xfer_size) { - nvme_printf(ctrlr, "pt->len (%d) " - "exceeds max_xfer_size (%d)\n", pt->len, - ctrlr->max_xfer_size); - return EIO; + if (pt->len > ctrlr->max_xfer_size || + (is_user && + nvme_npages((vm_offset_t)pt->buf, pt->len) > NVME_MAX_PAGES)) { + nvme_printf(ctrlr, + "len (%d) exceeds max_xfer_size (%d) or pagelimit (%d)\n", + pt->len, ctrlr->max_xfer_size, + (int)ptoa(NVME_MAX_PAGES)); + return (EIO); } - if (is_user_buffer) { - buf = uma_zalloc(pbuf_zone, M_WAITOK); - buf->b_iocmd = pt->is_read ? BIO_READ : BIO_WRITE; - if (vmapbuf(buf, pt->buf, pt->len, 1) < 0) { - ret = EFAULT; - goto err; - } - req = nvme_allocate_request_vaddr(buf->b_data, pt->len, - M_WAITOK, nvme_pt_done, pt); + if (is_user) { + npages = NVME_MAX_PAGES; + req = nvme_user_ioctl_req((vm_offset_t)pt->buf, pt->len, + pt->is_read, upages, &npages, nvme_pt_done, pt); + if (req == NULL) + return (EFAULT); } else req = nvme_allocate_request_vaddr(pt->buf, pt->len, M_WAITOK, nvme_pt_done, pt); @@ -1347,11 +1391,8 @@ mtx_sleep(pt, mtx, PRIBIO, "nvme_pt", 0); mtx_unlock(mtx); - if (buf != NULL) { - vunmapbuf(buf); -err: - uma_zfree(pbuf_zone, buf); - } + if (npages > 0) + nvme_user_ioctl_free(upages, npages); return (ret); } @@ -1377,8 +1418,9 @@ { struct nvme_request *req; struct mtx *mtx; - struct buf *buf = NULL; int ret = 0; + int npages = 0; + vm_page_t upages[NVME_MAX_PAGES]; /* * We don't support metadata. @@ -1387,10 +1429,13 @@ return (EIO); if (npc->data_len > 0 && npc->addr != 0) { - if (npc->data_len > ctrlr->max_xfer_size) { + if (npc->data_len > ctrlr->max_xfer_size || + (is_user && + nvme_npages(npc->addr, npc->data_len) > NVME_MAX_PAGES)) { nvme_printf(ctrlr, - "npc->data_len (%d) exceeds max_xfer_size (%d)\n", - npc->data_len, ctrlr->max_xfer_size); + "data_len (%d) exceeds max_xfer_size (%d) or pagelimit (%d)\n", + npc->data_len, ctrlr->max_xfer_size, + (int)ptoa(NVME_MAX_PAGES)); return (EIO); } /* @@ -1402,15 +1447,11 @@ if ((npc->opcode & 0x3) == 3) return (EINVAL); if (is_user) { - buf = uma_zalloc(pbuf_zone, M_WAITOK); - buf->b_iocmd = npc->opcode & 1 ? BIO_WRITE : BIO_READ; - if (vmapbuf(buf, (void *)(uintptr_t)npc->addr, - npc->data_len, 1) < 0) { - ret = EFAULT; - goto err; - } - req = nvme_allocate_request_vaddr(buf->b_data, - npc->data_len, M_WAITOK, nvme_npc_done, npc); + npages = NVME_MAX_PAGES; + req = nvme_user_ioctl_req(npc->addr, npc->data_len, npc->opcode & 0x1, + upages, &npages, nvme_npc_done, npc); + if (req == NULL) + return (EFAULT); } else req = nvme_allocate_request_vaddr( (void *)(uintptr_t)npc->addr, npc->data_len, @@ -1445,11 +1486,8 @@ mtx_sleep(npc, mtx, PRIBIO, "nvme_npc", 0); mtx_unlock(mtx); - if (buf != NULL) { - vunmapbuf(buf); -err: - uma_zfree(pbuf_zone, buf); - } + if (npages > 0) + nvme_user_ioctl_free(upages, npages); return (ret); }