Index: sys/dev/dmaengine/ptdma/ptdma.h =================================================================== --- /dev/null +++ sys/dev/dmaengine/ptdma/ptdma.h @@ -0,0 +1,276 @@ +/*- + * Copyright (c) 2020 Advanced Micro Devices Inc., + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Contact Information : + * Rajesh Kumar + * Arpan Palit + */ + +#ifndef _PTDMA_H_ +#define _PTDMA_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +extern unsigned int g_ptdma_dbg_lvl; + +#define ptdma_info(v, ...) do { \ + if ((v) <= g_ptdma_dbg_lvl) { \ + device_printf(pt->dev, __VA_ARGS__); \ + } \ +} while (0) + +#define ptdma_error(...) do { \ + device_printf(pt->dev, __VA_ARGS__); \ +} while (0) + +#define PTDMA_VENDOR_ID 0x1022 +#define PTDMA_DEVICE_ID 0x1498 + +#define PTDMA_MAX_CHANNELS 32 +#define PTDMA_MSIX_CNT 1 +#define PTDMA_MAX_NAME_LEN 64 +#define PTDMA_ENGINE_PASSTHRU 5 + +#define PTDMA_VSIZE 16 +#define PTDMA_VMASK ((unsigned int)((1 << PTDMA_VSIZE) - 1)) +#define PTDMA_VERSION(v, r) ((unsigned int)((v << PTDMA_VSIZE) \ + | (r & PTDMA_VMASK))) + +#define PTDMA_DETACH 0x0001 +#define PTDMA_EXIT 0x0002 + +typedef void *bus_dmaengine_t; +typedef void (*bus_dmaengine_callback_t)(void *arg, int error); + +/* + * HW descriptor for PTDMA commands + * 8 32-bit words: + * word 0: function; engine; control bits + * word 1: length of source data + * word 2: low 32 bits of source pointer + * word 3: upper 16 bits of source pointer; source memory type + * word 4: low 32 bits of destination pointer + * word 5: upper 16 bits of destination pointer; destination memory type + * word 6: reserved 32 bits + * word 7: reserved 32 bits + */ +struct ptdma_cmd_desc { + struct { + unsigned int hoc : 1; + unsigned int ioc : 1; + unsigned int unused0 : 1; + unsigned int som : 1; + unsigned int eom : 1; + unsigned int function : 15; + unsigned int engine : 4; + unsigned int unused1 : 8; + } dword0; + struct { + uint32_t length; + } dword1; + struct { + uint32_t src_low; + } dword2; + struct { + unsigned int src_high : 16; + unsigned int src_mem : 2; + unsigned int lsb_context : 8; + unsigned int unused0 : 5; + unsigned int fixed : 1; + } dword3; + struct { + uint32_t dest_low; + } dword4; + struct { + unsigned int dest_high : 16; + unsigned int dest_mem : 2; + unsigned int unused0 : 13; + unsigned int fixed : 1; + } dword5; + struct { + unsigned int unused0 : 32; + } dword6; + struct { + unsigned int unused0 : 32; + } dword7; +}; + +enum cmd_status { + CMD_IDLE = 0, + CMD_BUSY, + CMD_PROCESS, + CMD_ERROR, + CMD_BACKLOG, +}; + +struct ptdma_work_cmd { + TAILQ_ENTRY(ptdma_work_cmd) entry; + + struct ptdma_softc *pt; + struct ptdma_work_request *work_req; + + bus_addr_t src; + bus_addr_t dest; + bus_size_t len; + + enum cmd_status status; + int cmd_no; + bool completion; +}; + +enum req_status { + REQ_IDLE = 0, + REQ_BUSY, + REQ_PROCESS, + REQ_ERROR, +}; + +struct ptdma_work_request { + TAILQ_ENTRY(ptdma_work_request) entry; + + struct ptdma_softc *pt; + + //XXX - do we need seperate lock for both queue + struct mtx req_cmd_lock; + TAILQ_HEAD(, ptdma_work_cmd) cmd_work_q; + TAILQ_HEAD(, ptdma_work_cmd) cmd_back_q; + + bus_addr_t src; + bus_addr_t dest; + bus_size_t len; + uint16_t ncmds; + + enum req_status status; + + bus_dmaengine_callback_t callback_fn; + void *callback_arg; + int req_no; +}; + +/* Structure to hold PT device data */ +struct pt_dev_vdata { + const unsigned int bar[2]; + const unsigned int version; +}; + +struct pt_irq_info { + int irq_rid; + struct resource *irq_res; + void *irq_tag; +}; + +struct ptdma_softc { + /* dmaengine.c context. Do not move! Must go first! */ + void *dmaengine_store; + + TAILQ_ENTRY(ptdma_softc) entry; + bus_dmaengine_t dmaengine; + +#define to_ptdma_softc(_dmaeng) \ +({ \ + bus_dmaengine_t *_p = (_dmaeng); \ + (struct ptdma_softc *)((char *)_p - \ + offsetof(struct ptdma_softc, dmaengine)); \ +}) + device_t dev; + uint8_t chan_idx; + char name[PTDMA_MAX_NAME_LEN]; + struct pt_dev_vdata *dev_vdata; + uint32_t refcnt; + uint32_t cmd_count; + uint32_t req_count; + uint32_t intr_count; + uint32_t comp_count; + + /* PCI map details */ + bus_space_tag_t pci_bus_tag[2]; + bus_space_handle_t pci_bus_handle[2]; + struct resource *pci_resource[2]; + int pci_resource_id[2]; + vm_paddr_t paddr[2]; + vm_size_t bar_size[2]; + caddr_t vaddr[2]; + + /* IRQ details */ + struct pt_irq_info irq_info[PTDMA_MSIX_CNT]; + uint8_t msix_vector_count; + uint32_t irq_allocated; + struct taskqueue *intr_workqueue; + struct task intr_task; + + /* QUEUE Buffer details */ + size_t ringsz; + uint32_t ringlen; + bus_dma_tag_t hw_desc_tag; + bus_dmamap_t hw_desc_map; + struct ptdma_cmd_desc *hw_desc_vaddr; + bus_addr_t hw_desc_paddr; + uint32_t qidx; + + //XXX - should we need seperate lock for each queue. + struct mtx pt_req_lock; + TAILQ_HEAD(, ptdma_work_request) req_free_q; + TAILQ_HEAD(, ptdma_work_request) req_work_q; + + struct mtx pt_cmd_lock; + struct mtx cmd_comp_lock; + TAILQ_HEAD(, ptdma_work_cmd) cmd_free_q; + uint32_t cmd_error; + struct ptdma_work_cmd *curr_cmd; + + struct mtx thr_lock; + struct proc *procp; + uint8_t thr_flags; + + void *src_vaddr; + bus_addr_t src_paddr; +}; + +/* API's exposed to other drivers */ +extern int ptdma_get_chan_idx(ssize_t); +extern void ptdma_put_chan_idx(int); +extern bus_dmaengine_t ptdma_get_dmaengine(uint32_t chan_idx, int flags); +extern void ptdma_put_dmaengine(bus_dmaengine_t dmaengine); +extern void ptdma_acquire(bus_dmaengine_t dmaengine); +extern void ptdma_release(bus_dmaengine_t dmaengine); +extern int ptdma_copy(bus_dmaengine_t dmaengine, bus_addr_t dst, + bus_addr_t src, bus_size_t len, bus_dmaengine_callback_t callback_fn, + void *callback_arg, uint32_t flags); + +#endif Index: sys/dev/dmaengine/ptdma/ptdma.c =================================================================== --- /dev/null +++ sys/dev/dmaengine/ptdma/ptdma.c @@ -0,0 +1,1054 @@ +/*- + * Copyright (c) 2020 Advanced Micro Devices Inc., + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Contact Information : + * Rajesh Kumar + * Arpan Palit + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "ptdma.h" +#include "ptdma_hw.h" + +/* Globals */ +MALLOC_DEFINE(M_PTDMA, "ptdma", "ptdma driver memory allocations"); + +/* Store all the device instance in a list */ +TAILQ_HEAD(pt_channel, ptdma_softc) pt_channel; + +unsigned int n_ptdma_dev = 0; +unsigned int g_ptdma_dbg_lvl = 0; + +struct mtx pt_channel_lock; + +SYSCTL_NODE(_hw, OID_AUTO, pt, CTLFLAG_RD, 0, "pt node"); +SYSCTL_INT(_hw_pt, OID_AUTO, debug_level, CTLFLAG_RWTUN, &g_ptdma_dbg_lvl, + 0, "Set log level (0-3) for ptdma(4). Higher is more verbose."); + +/* Function prototypes */ +static int ptdma_probe(device_t); +static int ptdma_attach(device_t); +static int ptdma_detach(device_t); +int ptdma_get_chan_idx(ssize_t); +void ptdma_put_chan_idx(int); +bus_dmaengine_t ptdma_get_dmaengine(uint32_t, int); +void ptdma_put_dmaengine(bus_dmaengine_t); +int ptdma_copy(bus_dmaengine_t, bus_addr_t, bus_addr_t, + bus_size_t, bus_dmaengine_callback_t, void *, uint32_t); + +/* Externs */ +static void ptdma_process_request(struct ptdma_softc *pt); + +static const struct pt_dev_vdata dev_vdata[] = { + { .bar = {2, 5}, + .version = PTDMA_VERSION(5, 0), + }, +}; + +static const struct pci_device_table ptdma_devs[] = { + { PCI_DEV(PTDMA_VENDOR_ID, PTDMA_DEVICE_ID), + .driver_data = (uintptr_t)&dev_vdata[0], + PCI_DESCR("Pass-through Direct Memory Access Driver") }, +}; + +static inline uint32_t ptdma_low_address(unsigned long addr) +{ + return (uint64_t)addr & 0xffffffff; +} + +static inline uint32_t ptdma_high_address(unsigned long addr) +{ + return ((uint64_t)addr >> 32) & 0x0000ffff; +} + +/* API's exposed to client applications/drivers */ +bus_dmaengine_t ptdma_get_dmaengine(uint32_t chan_idx, int flags) +{ + struct ptdma_softc *pt, *pt_tmp; + + mtx_lock(&pt_channel_lock); + + TAILQ_FOREACH_SAFE(pt, &pt_channel, entry, pt_tmp) { + if (chan_idx == pt->chan_idx) + break; + } + + mtx_unlock(&pt_channel_lock); + + return (&pt->dmaengine); +} + +void ptdma_put_dmaengine(bus_dmaengine_t dmaengine) +{ + struct ptdma_softc *pt = to_ptdma_softc(dmaengine); + + if (pt == NULL) { + ptdma_info(1, "no ptdma device found\n"); + return; + } + + mtx_lock(&pt_channel_lock); + ptdma_info(0, "%s: req_cnt %d cmd_cnt %d intr_cnt %d comp_cnt %d\n", + __func__, pt->req_count, pt->cmd_count, pt->intr_count, + pt->comp_count); + while (pt->req_count != 0){ + atomic_subtract_int(&pt->req_count, 1); + atomic_subtract_int(&pt->cmd_count, 1); + atomic_subtract_int(&pt->intr_count, 1); + atomic_subtract_int(&pt->comp_count, 1); + } + ptdma_info(2, "%s: Requesting to put chan_idx %d ref_cnt %d\n", + __func__, pt->chan_idx, pt->refcnt); + if (pt->refcnt == 1) + atomic_subtract_int(&pt->refcnt, 1); + + mtx_unlock(&pt_channel_lock); + + ptdma_info(2, "%s: Freed ref_cnt %d for ptdma channel\n", + __func__, pt->refcnt); + + return; +} + +/* + * Change the free chan_idx logic based time-sharing policy + * of hardware for getting better performance with less number of thread + */ +int ptdma_get_chan_idx(ssize_t tcnt) +{ + struct ptdma_softc *pt, *pt_tmp; + + mtx_lock(&pt_channel_lock); + + TAILQ_FOREACH_SAFE(pt, &pt_channel, entry, pt_tmp) { + if (pt->refcnt == 0) { + ptdma_info(0, "%s: Got free chan_idx %d thread_count " + "%ld\n", __func__, pt->chan_idx, tcnt); + atomic_add_int(&pt->refcnt, 1); + atomic_set_int(&pt->req_count, 0); + atomic_set_int(&pt->cmd_count, 0); + atomic_set_int(&pt->intr_count, 0); + atomic_set_int(&pt->comp_count, 0); + mtx_unlock(&pt_channel_lock); + return pt->chan_idx; + } + } + + mtx_unlock(&pt_channel_lock); + + return (-ENODEV); +} + +void ptdma_put_chan_idx(int chan_idx) +{ + struct ptdma_softc *pt, *pt_tmp; + + mtx_lock(&pt_channel_lock); + + TAILQ_FOREACH_SAFE(pt, &pt_channel, entry, pt_tmp) { + if (chan_idx == pt->chan_idx) + break; + } + + ptdma_info(2, "%s: Requesting to put chan_idx %d ref_cnt %d\n", + __func__, pt->chan_idx, pt->refcnt); + if (pt->refcnt == 1) + atomic_subtract_int(&pt->refcnt, 1); + + ptdma_info(2, "%s: Freed ref_cnt %d for ptdma channel\n", + __func__, pt->refcnt); + mtx_unlock(&pt_channel_lock); +} + +static int +ptdma_get_work_cmds(struct ptdma_work_request *req) +{ + struct ptdma_softc *pt = req->pt; + struct ptdma_work_cmd *cmd; + uint16_t offset = 0, total_len = 0; + int i, cmdlen = PAGE_SIZE; + + mtx_lock(&pt->pt_cmd_lock); + + ptdma_info(2, "%s: ncmds %d\n", __func__, req->ncmds); + + for (i = 0 ; i < req->ncmds ; i++) { + + cmd = malloc(sizeof(*cmd), M_PTDMA, M_ZERO); + if (cmd == NULL) { + mtx_unlock(&pt->pt_cmd_lock); + return(-ENOMEM); + } + + cmd->pt = pt; + cmd->status = CMD_IDLE; + cmd->src = req->src + offset; + cmd->dest = req->dest + offset; + + if (i == (req->ncmds - 1)) + cmd->len = req->len - total_len; + else { + cmd->len = cmdlen; + total_len += cmdlen; + } + + offset += cmd->len; + + if (i <= 100) { + cmd->status = CMD_BUSY; + TAILQ_INSERT_TAIL(&req->cmd_work_q, cmd, entry); + } else { + cmd->status = CMD_BACKLOG; + TAILQ_INSERT_TAIL(&req->cmd_back_q, cmd, entry); + } + + ptdma_info(1, "%s: src 0x%zx dest 0x%zx len 0x%x offset 0x%x " + "status 0x%x\n", __func__, (unsigned long)cmd->src, + (unsigned long)cmd->dest, (unsigned int)cmd->len, + (unsigned int)offset, (unsigned int)cmd->status); + + atomic_add_int(&pt->cmd_count, 1); + cmd->cmd_no = pt->cmd_count; + } + + mtx_unlock(&pt->pt_cmd_lock); + + return (0); +} + +static struct ptdma_work_request * +ptdma_get_work_request(struct ptdma_softc *pt, bus_addr_t src, + bus_addr_t dest, bus_size_t len) +{ + struct ptdma_work_request *req = NULL; + //int ncmds = 0, total_len = len, cmdlen = PAGE_SIZE; + + mtx_lock(&pt->pt_req_lock); + + req = malloc(sizeof(*req), M_PTDMA, M_WAITOK | M_ZERO); + if (req == NULL) { + mtx_unlock(&pt->pt_req_lock); + return NULL; + } + + req->pt = pt; + req->status = REQ_IDLE; + mtx_init(&req->req_cmd_lock, "work request cmd lock", NULL, MTX_SPIN); + + TAILQ_INIT(&req->cmd_work_q); + TAILQ_INIT(&req->cmd_back_q); + + req->src = src; + req->dest = dest; + req->len = len; + req->ncmds = 1; + + ptdma_info(2, "%s: src 0x%zx dest 0x%zx len 0x%x ncmds %d\n", + __func__, (unsigned long)req->src, (unsigned long)req->dest, + (unsigned int)req->len, req->ncmds); + + atomic_add_int(&pt->req_count, 1); + req->req_no = pt->req_count; + mtx_unlock(&pt->pt_req_lock); + + return (req); +} + +static void ptdma_put_work_request(struct ptdma_work_request *req) +{ + struct ptdma_softc *pt = req->pt; + + mtx_lock(&pt->pt_req_lock); + + if (req->status != REQ_IDLE) + TAILQ_REMOVE(&pt->req_work_q, req, entry); + + req->src = 0; + req->dest = 0; + req->len = 0; + req->status = REQ_IDLE; + req->req_no = 0; + + TAILQ_INIT(&req->cmd_work_q); + TAILQ_INIT(&req->cmd_back_q); + free(req, M_PTDMA); + + mtx_unlock(&pt->pt_req_lock); +} + +int +ptdma_copy(bus_dmaengine_t dmaengine, bus_addr_t dest, bus_addr_t src, + bus_size_t len, bus_dmaengine_callback_t callback_fn, + void *callback_arg, uint32_t flags) +{ + struct ptdma_softc *pt = to_ptdma_softc(dmaengine); + struct ptdma_work_request *req; + int ret = 0; + + ptdma_info(2, "%s: src 0x%zx dest 0x%zx len 0x%x\n", __func__, + (unsigned long)src, (unsigned long)dest, (unsigned int)len); + + req = ptdma_get_work_request(pt, src, dest, len); + if (req == NULL) { + ptdma_error("%s: fail work_req\n", __func__); + return (-ENOMEM); + } + + ret = ptdma_get_work_cmds(req); + if (ret < 0) { + ptdma_error("%s: fail work_cmd %d\n", __func__, ret); + goto copy_err; + } + + ptdma_info(1, "%s: get_work_cmds pass\n", __func__); + + mtx_lock(&pt->pt_req_lock); + + req->callback_fn = callback_fn; + req->callback_arg = callback_arg; + + req->status = REQ_BUSY; + TAILQ_INSERT_TAIL(&pt->req_work_q, req, entry); + mtx_unlock(&pt->pt_req_lock); + + ptdma_info(1, "%s: wakeup thread\n", __func__); + wakeup(pt); + +copy_err: + + if (ret < 0) + ptdma_put_work_request(req); + + ptdma_info(1, "%s: ret %d\n", __func__, ret); + return (ret); +} + +static int ptdma_execute_cmd(struct ptdma_work_cmd *cmd) +{ + struct ptdma_softc *pt = cmd->pt; + struct ptdma_cmd_desc *desc; + uint32_t ctrl_reg; + + desc = &pt->hw_desc_vaddr[pt->qidx]; + memset(desc, 0, sizeof(*desc)); + ptdma_info(2, "%s: qidx %d desc %p-%p\n", __func__, pt->qidx, desc, + desc+pt->qidx); + + desc->dword0.hoc = 0; + desc->dword0.ioc = 1; + desc->dword0.eom = 1; + desc->dword0.som = 1; + desc->dword0.function = 0; + desc->dword0.engine = PTDMA_ENGINE_PASSTHRU; + + desc->dword1.length = cmd->len; + + desc->dword2.src_low = ptdma_low_address(cmd->src); + desc->dword3.src_high = ptdma_high_address(cmd->src); + desc->dword3.src_mem = 0; + + desc->dword4.dest_low = ptdma_low_address(cmd->dest); + desc->dword5.dest_high = ptdma_high_address(cmd->dest); + desc->dword5.dest_mem = 0; + + pt->qidx = ((pt->qidx + 1) % COMMANDS_PER_QUEUE); + + ptdma_info(2, "%s: qidx %d paddr 0x%x-0x%zx\n", + __func__, pt->qidx, (uint32_t)pt->hw_desc_paddr, + ((uint32_t)(pt->hw_desc_paddr) + (pt->qidx * sizeof(*desc)))); + wmb(); + + ptdma_write(pt, PTDMA_Q_REG_TAIL_LO, + ((uint32_t)pt->hw_desc_paddr) + (pt->qidx * sizeof(*desc))); + + ctrl_reg = ptdma_read(pt, PTDMA_Q_CONTROL_REG); + if (!(ctrl_reg & PTDMA_VQ_CTRL_RUN)) { + ctrl_reg |= PTDMA_VQ_CTRL_RUN; + ptdma_write(pt, PTDMA_Q_CONTROL_REG, ctrl_reg); + } + rmb(); + + ptdma_info(2, "%s: pt->qidx %d\n", __func__, pt->qidx); + + return (0); +} + +static void dump_reg_status(struct ptdma_softc *pt) +{ + ptdma_info(0, "CMD_INTR_STATUS: 0x%x\n", + ptdma_read(pt, PTDMA_Q_REG_INTR_STATUS)); + ptdma_info(0, "CMD_STATUS: 0x%x\n", + ptdma_read(pt, PTDMA_Q_REG_STATUS)); + ptdma_info(0, "CMD_QINTR_STATUS: 0x%x\n", + ptdma_read(pt, PTDMA_Q_REG_INT_STATUS)); + ptdma_info(0, "CMD_QDMA_STATUS: 0x%x\n", + ptdma_read(pt, PTDMA_Q_REG_DMA_STATUS)); + ptdma_info(0, "CMD_QDMA_READ_STATUS: 0x%x\n", + ptdma_read(pt, PTDMA_Q_REG_DMA_READ_STATUS)); + ptdma_info(0, "CMD_QDMA_WRITE_STATUS: 0x%x\n", + ptdma_read(pt, PTDMA_Q_REG_DMA_WRITE_STATUS)); +} + +static void ptdma_intr_work(void *data, int pending __unused) +{ + struct ptdma_softc *pt = (struct ptdma_softc *)data; + uint32_t status, reg_int_status, reg_status; + + mtx_lock_spin(&pt->cmd_comp_lock); + /* Disable interrupts */ + ptdma_write(pt, PTDMA_Q_REG_INT_ENABLE, 0); + + status = ptdma_read(pt, PTDMA_Q_REG_INTR_STATUS); + ptdma_info(2, "%s: status 0x%x\n", __func__, status); + if (status) { + /* Read status registers */ + reg_int_status = ptdma_read(pt, PTDMA_Q_REG_INT_STATUS); + reg_status = ptdma_read(pt, PTDMA_Q_REG_STATUS); + ptdma_info(2, "%s: int_status 0x%x status 0x%x\n", __func__, + reg_int_status, reg_status); + + atomic_add_int(&pt->intr_count, 1); + /* On error, only save the first error value */ + if ((status & PTDMA_INT_ERROR) && !pt->cmd_error) { + pt->cmd_error = (reg_status & 0x0000003f); + ptdma_error("%s: cmd_no %d cmd error %d\n", __func__, + pt->curr_cmd->cmd_no, pt->cmd_error); + dump_reg_status(pt); + } + + wmb(); + ptdma_write(pt, PTDMA_Q_REG_HEAD_LO, + ((uint32_t)pt->hw_desc_paddr) + + (pt->qidx * sizeof(struct ptdma_cmd_desc))); + + /* Acknowledge the interrupt and wake the thread*/ + ptdma_write(pt, PTDMA_Q_REG_INTR_STATUS, status); + if (pt->curr_cmd == NULL) + ptdma_info(0, "%s: status 0x%x for cur_cmd is null\n", + __func__, status); + else + ptdma_info(2, "%s: status 0x%x curr_cmd %p cmd_no %d\n", + __func__, status, pt->curr_cmd, pt->curr_cmd->cmd_no); + + pt->curr_cmd->completion = true; + wakeup_one(pt->curr_cmd); + } + + /* Enable Interrupts */ + ptdma_write(pt, PTDMA_Q_REG_INT_ENABLE, PTDMA_SUPP_INTERRUPTS); + mtx_unlock_spin(&pt->cmd_comp_lock); +} + +static void pt_interrupt_handler(void *arg) +{ + struct ptdma_softc *pt = (struct ptdma_softc *)arg; + + taskqueue_enqueue(pt->intr_workqueue, &pt->intr_task); + + return; +} + +static void ptdma_process_request(struct ptdma_softc *pt) +{ + struct ptdma_work_request *req; + struct ptdma_work_cmd *cmd, *cmd_tmp; + int ret = 0, err = 0; + + if (TAILQ_EMPTY(&pt->req_work_q)) { + ptdma_error("%s: pt->req_work_q empty\n", __func__); + return; + } + + req = TAILQ_FIRST(&pt->req_work_q); + req->status = REQ_PROCESS; + + TAILQ_FOREACH_SAFE(cmd, &req->cmd_work_q, entry, cmd_tmp) { + + cmd->status = CMD_PROCESS; + + if (req->status != REQ_ERROR) { + mtx_lock_spin(&pt->cmd_comp_lock); + pt->curr_cmd = cmd; + pt->curr_cmd->completion = false; + ret = ptdma_execute_cmd(cmd); + if (ret == 0) { + while (!pt->curr_cmd->completion && err == 0) + err = msleep_spin(pt->curr_cmd, + &pt->cmd_comp_lock, "cmd_xfer", hz); + + if (err || (err == EWOULDBLOCK)) { + ptdma_error("%s: failed curr_cmd %p" + " cmd_no %d completion %d err %d\n", + __func__, pt->curr_cmd, + pt->curr_cmd->cmd_no, + pt->curr_cmd->completion, err); + dump_reg_status(pt); + pt->curr_cmd = NULL; + mtx_unlock_spin(&pt->cmd_comp_lock); + ret = -EWOULDBLOCK; + goto cmd_fail; + } + atomic_add_int(&pt->comp_count, 1); + pt->curr_cmd = NULL; + mtx_unlock_spin(&pt->cmd_comp_lock); + } else + ptdma_error("%s: execute_cmd fail %d\n", + __func__, ret); + } +cmd_fail: + + if (ret || pt->cmd_error) { + ptdma_error("%s: cmd error ret %d cmd_err %d\n", + __func__, ret, pt->cmd_error); + cmd->status = CMD_ERROR; + req->status = REQ_ERROR; + } + + mtx_lock(&pt->pt_cmd_lock); + + TAILQ_REMOVE(&req->cmd_work_q, cmd, entry); + cmd->status = CMD_IDLE; + cmd->src = 0; + cmd->dest = 0; + cmd->len = 0; + cmd->cmd_no = 0; + free(cmd, M_PTDMA); + + if (!TAILQ_EMPTY(&req->cmd_back_q)) { + cmd_tmp = TAILQ_FIRST(&req->cmd_back_q); + TAILQ_REMOVE(&req->cmd_back_q, cmd_tmp, entry); + cmd_tmp->status = CMD_BUSY; + TAILQ_INSERT_TAIL(&req->cmd_work_q, cmd_tmp, entry); + } + + mtx_unlock(&pt->pt_cmd_lock); + } + + if (TAILQ_EMPTY(&req->cmd_work_q)) + ptdma_info(3, "%s: req cmd work queue empty\n", __func__); + + if (TAILQ_EMPTY(&req->cmd_back_q)) + ptdma_info(3, "%s: req cmd back queue empty\n", __func__); + + req->callback_fn(req->callback_arg, + ((req->status == REQ_ERROR)? req->status : 0)); + + ptdma_put_work_request(req); +} + +static void ptdma_thread_work(void *data) +{ + struct ptdma_softc *pt = (struct ptdma_softc *)data; + + ptdma_info(2, "%s: THREAD_ENTRY\n", __func__); + + while (1) { + + mtx_lock(&pt->thr_lock); + + if (pt->thr_flags & PTDMA_DETACH) { + pt->thr_flags |= PTDMA_EXIT; + mtx_unlock(&pt->thr_lock); + kproc_exit(0); + } + + if (TAILQ_EMPTY(&pt->req_work_q)) { + msleep(pt, &pt->thr_lock, 0, "workwait", 0); + mtx_unlock(&pt->thr_lock); + continue; + } + + mtx_unlock(&pt->thr_lock); + + ptdma_process_request(pt); + } +} + +static int ptdma_init_work_queue(struct ptdma_softc *pt) +{ + int ret; + + mtx_init(&pt->pt_req_lock, "ptdma req lock", NULL, MTX_SPIN); + TAILQ_INIT(&pt->req_work_q); + + mtx_init(&pt->pt_cmd_lock, "ptdma cmd lock", NULL, MTX_SPIN); + mtx_init(&pt->cmd_comp_lock, "ptdma cmd completion lock", NULL, MTX_SPIN); + + mtx_init(&pt->thr_lock, "thread lock", NULL, MTX_DEF); + pt->thr_flags = 0; + ret = kproc_create(ptdma_thread_work, pt, &pt->procp, 0, 0,"%s", pt->name); + if (ret) + ptdma_error("ptdma_thread_work failed %d\n", ret); + + return (ret); +} + +static void ptdma_destroy_work_queue(struct ptdma_softc *pt) +{ + mtx_lock(&pt->thr_lock); + + TAILQ_INIT(&pt->req_work_q); + pt->thr_flags |= PTDMA_DETACH; + wakeup(pt); + while (!(pt->thr_flags & PTDMA_EXIT)) + msleep(pt, &pt->thr_lock, 0, "ptdmadestroy", hz/10); + mtx_unlock(&pt->thr_lock); + pt->thr_flags = 0; + +} + +static void +ptdma_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) +{ + bus_addr_t *baddr; + + KASSERT(error == 0, ("%s: error:%d", __func__, error)); + baddr = arg; + *baddr = segs->ds_addr; + + return; +} + +static int ptdma_queue_init(struct ptdma_softc *pt) +{ + uint32_t ctrl_reg, int_status, status; + int ret; + + pt->ringlen = COMMANDS_PER_QUEUE; + pt->ringsz = sizeof(struct ptdma_cmd_desc) * pt->ringlen; + ptdma_info(1, "%s: ring len %d sz %zu\n", __func__, pt->ringlen, + pt->ringsz); + + ret = bus_dma_tag_create(bus_get_dma_tag(pt->dev), + PAGE_SIZE, (bus_addr_t)1 << 32, BUS_SPACE_MAXADDR_48BIT, + BUS_SPACE_MAXADDR, NULL, NULL, pt->ringsz, 1, pt->ringsz, 0, NULL, + NULL, &pt->hw_desc_tag); + if (ret != 0) + return (ret); + + ret = bus_dmamem_alloc(pt->hw_desc_tag, (void **)&pt->hw_desc_vaddr, + BUS_DMA_NOWAIT | BUS_DMA_COHERENT | BUS_DMA_ZERO, &pt->hw_desc_map); + if (ret != 0) + return (ret); + + ret = bus_dmamap_load(pt->hw_desc_tag, pt->hw_desc_map, pt->hw_desc_vaddr, + pt->ringsz, ptdma_dmamap_cb, &pt->hw_desc_paddr, BUS_DMA_NOWAIT); + if (ret) + return (ret); + + ptdma_info(1, "%s: vaddr %p paddr 0x%lx\n", __func__, pt->hw_desc_vaddr, + pt->hw_desc_paddr); + + /* Init Registers */ + ctrl_reg = 0; + ptdma_write(pt, PTDMA_Q_CONTROL_REG, ctrl_reg); + ptdma_info(1, "%s: ctrl_reg1 0x%x\n", __func__, ctrl_reg); + + /* Disable interrupts */ + ptdma_write(pt, PTDMA_Q_REG_INT_ENABLE, 0); + ptdma_info(1, "%s: int_enable 0x%x\n", __func__, + ptdma_read(pt, PTDMA_Q_REG_INT_ENABLE)); + + /* Read status registers */ + int_status = ptdma_read(pt, PTDMA_Q_REG_INT_STATUS); + status = ptdma_read(pt, PTDMA_Q_REG_STATUS); + ptdma_info(1, "%s: int_status 0x%x status 0x%x\n", __func__, int_status, + status); + + ptdma_write(pt, PTDMA_CMD_PRIO_OFFSET, 0x6); + ptdma_write(pt, PTDMA_CMD_REQID_CFG_OFFSET, 0x0); + ptdma_write(pt, PTDMA_CMD_TIMEOUT_OFFSET, 0x0); + ptdma_write(pt, PTDMA_CMD_CONFIG_OFFSET, 0x1); + ptdma_write(pt, PTDMA_CMD_CLK_GATE_CTL_OFFSET, CMD_CLK_GATE_CONFIG); + + ptdma_write(pt, PTDMA_Q_REG_TAIL_LO, (uint32_t)pt->hw_desc_paddr); + ptdma_write(pt, PTDMA_Q_REG_HEAD_LO, (uint32_t)pt->hw_desc_paddr); + ptdma_info(1, "%s: tail 0x%x head 0x%x\n", __func__, + ptdma_read(pt, PTDMA_Q_REG_TAIL_LO), + ptdma_read(pt, PTDMA_Q_REG_HEAD_LO)); + + ptdma_info(1, "%s: ctrl_reg2 0x%x\n", __func__, ctrl_reg); + ctrl_reg |= ((ffs(COMMANDS_PER_QUEUE) - 2) << PTDMA_VQ_CTRL_QSIZE_SHIFT); + ctrl_reg |= (pt->hw_desc_paddr >> 32) << 16; + ctrl_reg |= PTDMA_VQ_CTRL_RUN; + ptdma_write(pt, PTDMA_Q_CONTROL_REG, ctrl_reg); + ptdma_info(1, "%s: ctrl_reg3 0x%x reg 0x%x\n", __func__, ctrl_reg, + ptdma_read(pt, PTDMA_Q_CONTROL_REG)); + + /* Clear Interrupt status and enable interrupts */ + ptdma_write(pt, PTDMA_Q_REG_INTR_STATUS, 0xF); + ptdma_write(pt, PTDMA_Q_REG_INT_ENABLE, PTDMA_SUPP_INTERRUPTS); + ptdma_info(1, "%s: intr_status 0x%x int_enable 0x%x\n", __func__, + ptdma_read(pt, PTDMA_Q_REG_INTR_STATUS), + ptdma_read(pt, PTDMA_Q_REG_INT_ENABLE)); + + return(0); +} + +static void +ptdma_queue_free(struct ptdma_softc *pt) +{ + uint32_t ctrl_reg, int_status, status; + + /* Disable interrupts */ + ptdma_write(pt, PTDMA_Q_REG_INT_ENABLE, 0); + ptdma_info(1, "%s: int_enable 0x%x\n", __func__, + ptdma_read(pt, PTDMA_Q_REG_INT_ENABLE)); + + /* Turn off the Run Bit */ + ctrl_reg = ptdma_read(pt, PTDMA_Q_CONTROL_REG); + ctrl_reg &= ~PTDMA_VQ_CTRL_RUN; + ptdma_write(pt, PTDMA_Q_CONTROL_REG, ctrl_reg); + ptdma_info(1, "%s: ctrl_reg1 0x%x\n", __func__, ctrl_reg); + + /* Clear the interrupt status */ + ptdma_write(pt, PTDMA_Q_REG_INTR_STATUS, 0xFFFFFFFF); + ptdma_info(1, "%s: intr_status 0x%x\n", __func__, + ptdma_read(pt, PTDMA_Q_REG_INTR_STATUS)); + + /* Read status registers */ + int_status = ptdma_read(pt, PTDMA_Q_REG_INT_STATUS); + status = ptdma_read(pt, PTDMA_Q_REG_STATUS); + ptdma_info(1, "%s: int_status 0x%x status 0x%x\n", __func__, int_status, + status); + + if (pt->hw_desc_paddr != 0) + bus_dmamap_unload(pt->hw_desc_tag, pt->hw_desc_map); + + if (pt->hw_desc_vaddr != NULL) + bus_dmamem_free(pt->hw_desc_tag, pt->hw_desc_vaddr, + pt->hw_desc_map); + + if (pt->hw_desc_tag != NULL) + bus_dma_tag_destroy(pt->hw_desc_tag); +} + +static int ptdma_setup_irq(struct ptdma_softc *pt) +{ + boolean_t use_msi = false, f_legacy = false; + uint32_t supp_count = 0, vector_count = 0; + int i, ret = 0, flags = 0; + + supp_count = pci_msix_count(pt->dev); + if (!supp_count || (supp_count < PTDMA_MSIX_CNT)) { + ptdma_error("MSIX supp count %d\n", supp_count); + use_msi = true; + goto enable_msi_vec; + } + + vector_count = min(PTDMA_MSIX_CNT, supp_count); + ptdma_info(1, "%s: MSIX supp %d vector %d\n", __func__, supp_count, + vector_count); + ret = pci_alloc_msix(pt->dev, &vector_count); + if (ret != 0) { + ptdma_error("Error allocating msix vectors: %d\n", ret); + use_msi = true; + } + +enable_msi_vec: + + if(use_msi) { + pci_release_msi(pt->dev); + supp_count = pci_msi_count(pt->dev); + if (!supp_count) { + ptdma_error("MSI supp count %d\n", supp_count); + use_msi = false; + f_legacy = true; + goto enable_legacy_intr; + } + + vector_count = 1; + ptdma_info(0, "%s: MSI supp %d vector %d\n", __func__, + supp_count, vector_count); + ret = pci_alloc_msi(pt->dev, &vector_count); + if (ret != 0) { + ptdma_error("Error allocating msi vector: %d\n", ret); + use_msi = false; + f_legacy = true; + } + } + +enable_legacy_intr: + + if (f_legacy) { + pt->msix_vector_count = 0; + vector_count = 1; + } else + pt->msix_vector_count = vector_count; + ptdma_info(1, "%s: msix/msi vector %d\n", __func__, + pt->msix_vector_count); + + flags |= RF_ACTIVE; + if (f_legacy) + flags |= RF_SHAREABLE; + + for (i = 0; i < vector_count; i++) { + + /* RID should be 0 for legacy intr */ + if (f_legacy) + pt->irq_info[i].irq_rid = i; + else + pt->irq_info[i].irq_rid = i + 1; + + ptdma_info(1, "%s: i %d irq_rid %d\n", __func__, i, + pt->irq_info[i].irq_rid); + pt->irq_info[i].irq_res = bus_alloc_resource_any(pt->dev, + SYS_RES_IRQ, &pt->irq_info[i].irq_rid, flags); + if (pt->irq_info[i].irq_res == NULL) { + ptdma_error(0, "bus alloc resource failed\n"); + return (-ENOMEM); + } + + pt->irq_info[i].irq_tag = NULL; + pt->irq_allocated++; + + ret = bus_setup_intr(pt->dev, pt->irq_info[i].irq_res, + INTR_MPSAFE | INTR_TYPE_MISC, NULL, pt_interrupt_handler, + pt, &pt->irq_info[i].irq_tag); + if (ret != 0) { + ptdma_error("bus setup intr failed \n"); + return (-ret); + } + } + + TASK_INIT(&pt->intr_task, 0, ptdma_intr_work, pt); + + pt->intr_workqueue = taskqueue_create("ptdma_intr_wq", + M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &pt->intr_workqueue); + + taskqueue_start_threads(&pt->intr_workqueue, 1, PI_DISK, + "ptdma_intr_work_thread"); + + return (0); +} + +static int ptdma_teardown_intr(struct ptdma_softc *pt) +{ + struct pt_irq_info *irq_info; + int i; + + taskqueue_drain(pt->intr_workqueue, &pt->intr_task); + taskqueue_free(pt->intr_workqueue); + + for (i = 0; i < pt->irq_allocated; i++) { + irq_info = &pt->irq_info[i]; + if (irq_info[i].irq_tag != NULL) + bus_teardown_intr(pt->dev, irq_info[i].irq_res, + irq_info[i].irq_tag); + + if (pt->irq_info[i].irq_res != NULL) + bus_release_resource(pt->dev, SYS_RES_IRQ, + rman_get_rid(irq_info[i].irq_res), + irq_info[i].irq_res); + } + + if (pt->msix_vector_count) { + pci_release_msi(pt->dev); + pt->msix_vector_count = 0; + } + + return(0); +} + +static int ptdma_map_pci_bar(struct ptdma_softc *pt) +{ + int i; + + for (i = 0; i < 2 ; i++) { + + pt->pci_resource_id[i] = PCIR_BAR(pt->dev_vdata->bar[i]); + ptdma_info(1, "pci resource_id %d\n", pt->pci_resource_id[i]); + + pt->pci_resource[i] = bus_alloc_resource_any(pt->dev, + SYS_RES_MEMORY, &pt->pci_resource_id[i], RF_ACTIVE); + if (pt->pci_resource[i] == NULL) { + ptdma_error("unable to allocate pci resource\n"); + return (-ENODEV); + } + + pt->pci_bus_tag[i] = rman_get_bustag(pt->pci_resource[i]); + pt->pci_bus_handle[i] = rman_get_bushandle(pt->pci_resource[i]); + pt->paddr[i] = rman_get_start(pt->pci_resource[i]); + pt->vaddr[i] = rman_get_virtual(pt->pci_resource[i]); + pt->bar_size[i] = rman_get_size(pt->pci_resource[i]); + } + + ptdma_info(1, "PCI bar map successful\n"); + + return (0); +} + +/* + * PTDMA Driver linkage functions to system + */ +static int ptdma_probe(device_t dev) +{ + struct ptdma_softc *pt = device_get_softc(dev); + const struct pci_device_table *id; + + pt->dev = dev; + id = PCI_MATCH(dev, ptdma_devs); + if (id == NULL) + return (ENXIO); + + pt->dev_vdata = (struct pt_dev_vdata *)id->driver_data; + device_set_desc(dev, id->descr); + + return (BUS_PROBE_GENERIC); +} + +static int ptdma_attach(device_t dev) +{ + struct ptdma_softc *pt = device_get_softc(dev); + int ret = 0; + + if (n_ptdma_dev == 0) + mtx_init(&pt_channel_lock, "PTDMA Channels", 0, MTX_DEF); + + mtx_lock(&pt_channel_lock); + pt->chan_idx = n_ptdma_dev; + snprintf(pt->name, PTDMA_MAX_NAME_LEN, "%s", device_get_nameunit(dev)); + + if (pt_channel.tqh_last == NULL) + TAILQ_INIT(&pt_channel); + atomic_set_int(&pt->refcnt, 0); + atomic_add_int(&n_ptdma_dev, 1); + TAILQ_INSERT_TAIL(&pt_channel, pt, entry); + mtx_unlock(&pt_channel_lock); + + ret = pci_enable_busmaster(dev); + if (ret != 0) + return (ret); + + ret = ptdma_map_pci_bar(pt); + if (ret) + return (ret); + + ret = ptdma_setup_irq(pt); + if (ret) + goto err; + + ret = ptdma_queue_init(pt); + if (ret) + goto err; + + ret = ptdma_init_work_queue(pt); + if (ret != 0) + goto err;; + + device_printf(pt->dev, "Driver attach successful\n"); + +err: + if (ret != 0) + ptdma_detach(dev); + + return ret; +} + +static int ptdma_detach(device_t dev) +{ + struct ptdma_softc *pt = device_get_softc(dev), *pt_chan, *pt_chan_tmp; + int i; + + pci_disable_busmaster(dev); + + ptdma_destroy_work_queue(pt); + ptdma_info(1, "PTDMA work queue released\n"); + + ptdma_queue_free(pt); + + ptdma_teardown_intr(pt); + ptdma_info(1, "PT IRQ released\n"); + + for (i = 0 ; i < 2 ; i++) { + if (pt->pci_resource[i] != NULL) + bus_release_resource(dev, SYS_RES_MEMORY, + pt->pci_resource_id[i], pt->pci_resource[i]); + } + + mtx_lock(&pt_channel_lock); + ptdma_info(1, "%s: name %s id %d num_dev %d\n", __func__, pt->name, + pt->chan_idx, n_ptdma_dev); + TAILQ_FOREACH_SAFE(pt_chan, &pt_channel, entry, pt_chan_tmp) { + if (pt->chan_idx == pt_chan->chan_idx) { + TAILQ_REMOVE(&pt_channel, pt_chan, entry); + } + } + pt->chan_idx = 0; + atomic_subtract_int(&n_ptdma_dev, 1); + mtx_unlock(&pt_channel_lock); + + ptdma_info(1, "PTDMA Device Detached\n"); + + return (0); +} + +/* + * PTDMA Driver interface structures to system + */ +static device_method_t ptdma_pci_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, ptdma_probe), + DEVMETHOD(device_attach, ptdma_attach), + DEVMETHOD(device_detach, ptdma_detach), + DEVMETHOD_END +}; + +static driver_t ptdma_pci_driver = { + "ptdma", + ptdma_pci_methods, + sizeof(struct ptdma_softc), +}; + +static devclass_t ptdma_devclass; +DRIVER_MODULE(ptdma, pci, ptdma_pci_driver, ptdma_devclass, 0, 0); +MODULE_VERSION(ptdma, 1); Index: sys/dev/dmaengine/ptdma/ptdma_hw.h =================================================================== --- /dev/null +++ sys/dev/dmaengine/ptdma/ptdma_hw.h @@ -0,0 +1,151 @@ +/*- + * Copyright (c) 2020 Advanced Micro Devices Inc., + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Contact Information : + * Rajesh Kumar + * Arpan Palit + */ + +#ifndef __PT_INTERNAL_H__ +#define __PT_INTERNAL_H__ + +#define cpu_to_le32 htole32 + +#define ptdma_read(pt, offset) \ + bus_read_4((pt)->pci_resource[0], (offset)) + +#define ptdma_write(pt, offset, value) \ + bus_write_4((pt)->pci_resource[0], (offset), (value)) + +/* Register Masks, Shifts, and Bitfields. */ +#define PTDMA_CMD_PRIO_MASK 0x0003 +#define PTDMA_CMD_PRIO_DISABLE 0x0007 +#define PTDMA_CMD_PRIO_HIGHEST 0x0006 +#define PTDMA_CMD_PRIO_LOWEST 0x0000 + +#define PTDMA_REQID_CFG_MASK 0x0003 +#define PTDMA_REQID_CFG_MAX 0x0005 +#define PTDMA_REQID_CFG_MIN 0x0001 + +#define PTDMA_VQ_CTRL_RUN_MASK 0x0001 +#define PTDMA_VQ_CTRL_RUN 0x0001 +#define PTDMA_VQ_CTRL_HALTED_MASK 0x0002 +#define PTDMA_VQ_CTRL_HALTED 0x0002 +#define PTDMA_VQ_CTRL_MEM_MASK 0x0004 +#define PTDMA_VQ_CTRL_QSIZE_MASK 0x0078 +#define PTDMA_VQ_CTRL_QSIZE_SHIFT 0x0003 +#define PTDMA_VQ_CTRL_HPTR_MASK 0xFFFF0000 +#define PTDMA_VQ_CTRL_HPTR_SHIFT 0x0008 + +#define PTDMA_CMD_INT_IOC 0x0001 +#define PTDMA_CMD_INT_ERR 0x0002 +#define PTDMA_CMD_INT_QSTOP 0x0004 +#define PTDMA_CMD_INT_QEMPTY 0x0008 + +#define PTDMA_CMD_DMA_STATUS_ERR_MASK 0x003F +#define PTDMA_CMD_DMA_STATUS_JOBSTATUS_MASK 0x0380 +#define PTDMA_CMD_DMA_STATUS_ERRSRC_MASK 0x0C00 +#define PTDMA_CMD_DMA_STATUS_VLSBBLK_MASK 0x7000 + +#define PTDMA_CMD_DMA_INT_STATUS_CNT_MASK 0x007F + +#define PTDMA_CMD_DMA_STATUS_WR_MASK 0x00003FF +#define PTDMA_CMD_DMA_STATUS_WR_SHIFT 0x0000000 +#define PTDMA_CMD_DMA_STATUS_WR_MAX 0x0000400 +#define PTDMA_CMD_DMA_STATUS_RD_MASK 0x007FC00 +#define PTDMA_CMD_DMA_STATUS_RD_SHIFT 0x0000008 +#define PTDMA_CMD_DMA_STATUS_RD_MAX 0x0000100 +#define PTDMA_CMD_DMA_STATUS_STAT_MASK 0x0380000 +#define PTDMA_CMD_DMA_STATUS_STAT_SHIFT 0x0000013 +#define PTDMA_CMD_DMA_STATUS_STAT_IDLE 0x0000000 +#define PTDMA_CMD_DMA_STATUS_STAT_WAIT_BUF 0x0000001 +#define PTDMA_CMD_DMA_STATUS_STAT_ARB_REQ 0x0000002 +#define PTDMA_CMD_DMA_STATUS_STAT_ISSUE_REQ 0x0000003 +#define PTDMA_CMD_DMA_STATUS_STAT_WAIT_DATA 0x0000004 + +#define PTDMA_CMD_DMA_ABORT_OFFSET_MASK 0x003FFFF +#define PTDMA_CMD_DMA_ABORT_OFFSET_VALID_MASK 0x8000000 + +#define PTDMA_CMD_AXCACHE_WCACHE_MASK 0x0000F +#define PTDMA_CMD_AXCACHE_RCACHE_MASK 0x000F0 +#define PTDMA_CMD_AXCACHE_RCACHE_SHIFT 0x00004 +#define PTDMA_CMD_AXCACHE_VQCACHE_MASK 0x00F00 +#define PTDMA_CMD_AXCACHE_VQCACHE_SHIFT 0x00008 + +#define PTDMA_CMD_VLSB_LSB0_MASK 0x00000000F +#define PTDMA_CMD_VLSB_LSB1_MASK 0x0000000F0 +#define PTDMA_CMD_VLSB_LSB2_MASK 0x000000F00 +#define PTDMA_CMD_VLSB_LSB3_MASK 0x00000F000 +#define PTDMA_CMD_VLSB_LSB4_MASK 0x0000F0000 +#define PTDMA_CMD_VLSB_LSB5_MASK 0x000F00000 +#define PTDMA_CMD_VLSB_LSB6_MASK 0x00F000000 +#define PTDMA_CMD_VLSB_LSB7_MASK 0x0F0000000 + +#define PTDMA_CFG_ENABLE_MASK 0x0001 + +/* Register addresses for queue */ +#define PTDMA_CMD_PRIO_OFFSET 0x0000 +#define PTDMA_CMD_REQID_CFG_OFFSET 0x0004 +#define PTDMA_CMD_TIMEOUT_OFFSET 0x0008 +#define PTDMA_CMD_PT_VERSION 0x0010 + +#define PTDMA_Q_CONTROL_REG 0x1000 +#define PTDMA_Q_REG_TAIL_LO 0x1004 +#define PTDMA_Q_REG_HEAD_LO 0x1008 +#define PTDMA_Q_REG_INT_ENABLE 0x100c +#define PTDMA_Q_REG_INTR_STATUS 0x1010 +#define PTDMA_Q_REG_STATUS 0x1100 +#define PTDMA_Q_REG_INT_STATUS 0x1104 +#define PTDMA_Q_REG_DMA_STATUS 0x1108 +#define PTDMA_Q_REG_DMA_READ_STATUS 0x110c +#define PTDMA_Q_REG_DMA_WRITE_STATUS 0x1110 +#define PTDMA_Q_REG_ABORT_BASE 0x1114 +#define PTDMA_Q_REG_AXCACHE_BASE 0x1118 +#define PTDMA_CMD_CONFIG_OFFSET 0x1120 +#define PTDMA_CMD_CLK_GATE_CTL_OFFSET 0x6004 + +/* Bit masks */ +#define CMD_CLK_DYN_GATING_EN 0x1 +#define CMD_CLK_DYN_GATING_DIS 0x0 +#define CMD_CLK_HW_GATE_MODE 0x1 +#define CMD_CLK_SW_GATE_MODE 0x0 +#define CMD_CLK_GATE_ON_DELAY 0x1000 +#define CMD_CLK_GATE_CTL 0x0 +#define CMD_CLK_GATE_OFF_DELAY 0x1000 + +#define CMD_CLK_GATE_CONFIG (CMD_CLK_DYN_GATING_EN | \ + CMD_CLK_HW_GATE_MODE | CMD_CLK_GATE_ON_DELAY | \ + CMD_CLK_GATE_CTL | CMD_CLK_GATE_OFF_DELAY) + +#define COMMANDS_PER_QUEUE 16 + +#define PTDMA_INT_COMPLETION 0x1 +#define PTDMA_INT_ERROR 0x2 +#define PTDMA_INT_QUEUE_STOPPED 0x4 +#define PTDMA_INT_EMPTY_QUEUE 0x8 + +#define PTDMA_SUPP_INTERRUPTS (PTDMA_INT_COMPLETION | PTDMA_INT_ERROR | \ + PTDMA_INT_QUEUE_STOPPED | PTDMA_INT_EMPTY_QUEUE) +#endif Index: sys/modules/dmaengine/Makefile =================================================================== --- /dev/null +++ sys/modules/dmaengine/Makefile @@ -0,0 +1,5 @@ +# $FreeBSD$ + +SUBDIR= ptdma + +.include Index: sys/modules/dmaengine/ptdma/Makefile =================================================================== --- /dev/null +++ sys/modules/dmaengine/ptdma/Makefile @@ -0,0 +1,9 @@ +# $FreeBSD$ + +.PATH: ${SRCTOP}/sys/dev/dmaengine/ptdma + +KMOD = ptdma +SRCS = ptdma.c +SRCS += device_if.h bus_if.h pci_if.h + +.include