Index: contrib/ofed/libbnxtre/abi.h =================================================================== --- contrib/ofed/libbnxtre/abi.h +++ contrib/ofed/libbnxtre/abi.h @@ -529,6 +529,7 @@ struct bnxt_re_srq_resp { struct ibv_create_srq_resp resp; __u32 srqid; + __u64 srq_page; } __attribute__((packed)); struct bnxt_re_srqe { Index: contrib/ofed/libbnxtre/db.c =================================================================== --- contrib/ofed/libbnxtre/db.c +++ contrib/ofed/libbnxtre/db.c @@ -38,7 +38,7 @@ #define BNXT_RE_MAX_FIFO_DEPTH_P5 0x2c00 #define BNXT_RE_DB_FIFO_ROOM_MASK_P7 0x3FFF8000 -#define BNXT_RE_MAX_FIFO_DEPTH_P7 0x8000 +#define BNXT_RE_MAX_FIFO_DEPTH_P7 0x7fff #define BNXT_RE_DB_FIFO_ROOM_SHIFT 15 #define BNXT_RE_DB_THRESHOLD 20 @@ -248,10 +248,15 @@ void bnxt_re_ring_srq_arm(struct bnxt_re_srq *srq) { struct bnxt_re_db_hdr hdr; + uint32_t toggle = 0; + + if (srq->srq_page) + toggle = *(uint32_t *)srq->srq_page; if (bnxt_re_do_pacing(srq->uctx, &srq->rand)) return; - bnxt_re_init_db_hdr(&hdr, srq->cap.srq_limit, 0, srq->srqid, + bnxt_re_init_db_hdr(&hdr, srq->cap.srq_limit, + toggle << BNXT_RE_DB_TOGGLE_SHIFT, srq->srqid, BNXT_RE_QUE_TYPE_SRQ_ARM); bnxt_re_ring_db(srq->udpi, hdr.typ_qid_indx, &srq->shadow_db_key, &srq->dbr_lock); Index: contrib/ofed/libbnxtre/main.h =================================================================== --- contrib/ofed/libbnxtre/main.h +++ contrib/ofed/libbnxtre/main.h @@ -189,6 +189,7 @@ struct xorshift32_state rand; uint8_t dbr_lock; bool arm_req; + void *srq_page; }; struct bnxt_re_joint_queue { Index: contrib/ofed/libbnxtre/verbs.c =================================================================== --- contrib/ofed/libbnxtre/verbs.c +++ contrib/ofed/libbnxtre/verbs.c @@ -342,9 +342,9 @@ } if (resp.comp_mask & BNXT_RE_COMP_MASK_CQ_HAS_CQ_PAGE) { - cq->cq_page = mmap(NULL, dev->pg_size, PROT_WRITE, MAP_SHARED, + cq->cq_page = mmap(NULL, dev->pg_size, PROT_READ | PROT_WRITE, MAP_SHARED, ibvctx->cmd_fd, resp.cq_page); - if (!cq->cq_page) + if (cq->cq_page == MAP_FAILED) fprintf(stderr, DEV "Valid cq_page not mapped\n"); } @@ -2347,6 +2347,13 @@ goto fail; srq->srqid = resp.srqid; + if (resp.srq_page) { + srq->srq_page = mmap(NULL, uctx->rdev->pg_size, PROT_READ, MAP_SHARED, + ibvpd->context->cmd_fd, resp.srq_page); + if (srq->srq_page == MAP_FAILED) + srq->srq_page = NULL; + } + srq->udpi = &uctx->udpi; srq->cap.max_wr = srq->srqq->depth; srq->cap.max_sge = attr->attr.max_sge; @@ -2395,6 +2402,8 @@ bnxt_re_list_del_node(&srq->dbnode, &srq->uctx->srq_dbr_res.head); pthread_spin_unlock(&srq->uctx->srq_dbr_res.lock); } + if (srq->srq_page) + munmap(srq->srq_page, srq->uctx->rdev->pg_size); ret = ibv_cmd_destroy_srq(ibvsrq); if (ret) { if (_is_db_drop_recovery_enable(srq->uctx)) { Index: sys/dev/bnxt/bnxt_en/bnxt.h =================================================================== --- sys/dev/bnxt/bnxt_en/bnxt.h +++ sys/dev/bnxt/bnxt_en/bnxt.h @@ -1167,6 +1167,7 @@ struct iflib_dma_info def_nq_ring_mem; struct task def_cp_task; int db_size; + int db_offset; int legacy_db_size; struct bnxt_doorbell_ops db_ops; Index: sys/dev/bnxt/bnxt_en/bnxt_hwrm.c =================================================================== --- sys/dev/bnxt/bnxt_en/bnxt_hwrm.c +++ sys/dev/bnxt/bnxt_en/bnxt_hwrm.c @@ -1309,6 +1309,7 @@ goto end; softc->legacy_db_size = le16_to_cpu(resp->legacy_l2_db_size_kb) * 1024; + softc->db_offset = le16toh(resp->legacy_l2_db_size_kb) * 1024; if (BNXT_CHIP_P5(softc)) { if (BNXT_PF(softc)) @@ -1316,6 +1317,7 @@ else min_db_offset = DB_VF_OFFSET_P5; softc->legacy_db_size = min_db_offset; + softc->db_offset = min_db_offset; } softc->db_size = roundup2(le16_to_cpu(resp->l2_doorbell_bar_size_kb) * Index: sys/dev/bnxt/bnxt_en/bnxt_ulp.h =================================================================== --- sys/dev/bnxt/bnxt_en/bnxt_ulp.h +++ sys/dev/bnxt/bnxt_en/bnxt_ulp.h @@ -94,6 +94,9 @@ #define BNXT_EN_MH(edev) ((edev)->flags & BNXT_EN_FLAG_MULTI_HOST) const struct bnxt_en_ops *en_ops; struct bnxt_ulp ulp_tbl[BNXT_MAX_ULP]; + int l2_db_offset; /* Doorbell BAR offset + * of non-cacheable. + */ int l2_db_size; /* Doorbell BAR size in * bytes mapped by L2 * driver. Index: sys/dev/bnxt/bnxt_en/bnxt_ulp.c =================================================================== --- sys/dev/bnxt/bnxt_en/bnxt_ulp.c +++ sys/dev/bnxt/bnxt_en/bnxt_ulp.c @@ -125,7 +125,7 @@ ent[i].vector = bp->irq_tbl[idx + i].vector; ent[i].ring_idx = idx + i; if (BNXT_CHIP_P5_PLUS(bp)) - ent[i].db_offset = DB_PF_OFFSET_P5; + ent[i].db_offset = bp->db_offset; else ent[i].db_offset = (idx + i) * 0x80; @@ -449,6 +449,7 @@ edev->pdev = bp->pdev; edev->softc = bp; edev->l2_db_size = bp->db_size; + edev->l2_db_offset = bp->db_offset; mtx_init(&bp->en_ops_lock, "Ethernet ops lock", NULL, MTX_DEF); if (bp->flags & BNXT_FLAG_ROCEV1_CAP) Index: sys/dev/bnxt/bnxt_re/ib_verbs.h =================================================================== --- sys/dev/bnxt/bnxt_re/ib_verbs.h +++ sys/dev/bnxt/bnxt_re/ib_verbs.h @@ -77,10 +77,18 @@ #define SPEED_200000 200000 #endif +#ifndef SPEED_400000 +#define SPEED_400000 400000 +#endif + #ifndef IB_SPEED_HDR #define IB_SPEED_HDR 64 #endif +#ifndef IB_SPEED_NDR +#define IB_SPEED_NDR 128 +#endif + #define RDMA_NETWORK_IPV4 1 #define RDMA_NETWORK_IPV6 2 Index: sys/dev/bnxt/bnxt_re/ib_verbs.c =================================================================== --- sys/dev/bnxt/bnxt_re/ib_verbs.c +++ sys/dev/bnxt/bnxt_re/ib_verbs.c @@ -276,6 +276,10 @@ *speed = IB_SPEED_HDR; *width = IB_WIDTH_4X; break; + case SPEED_400000: + *speed = IB_SPEED_NDR; + *width = IB_WIDTH_4X; + break; default: *speed = IB_SPEED_SDR; *width = IB_WIDTH_1X; @@ -1666,23 +1670,24 @@ return rc; bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); + bytes = PAGE_ALIGN(bytes); /* Consider mapping PSN search memory only for RC QPs. */ if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { psn_sz = _is_chip_gen_p5_p7(rdev->chip_ctx) ? sizeof(struct sq_psn_search_ext) : sizeof(struct sq_psn_search); - if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) + if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2)) psn_sz = sizeof(struct sq_msn_search); psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? qplib_qp->sq.max_wqe : ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / sizeof(struct bnxt_qplib_sge)); - if (BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) + if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2)) psn_nume = roundup_pow_of_two(psn_nume); bytes += (psn_nume * psn_sz); + bytes = PAGE_ALIGN(bytes); } - bytes = PAGE_ALIGN(bytes); umem = ib_umem_get_compat(rdev, context, udata, ureq.qpsva, bytes, IB_ACCESS_LOCAL_WRITE, 1); if (IS_ERR(umem)) { @@ -5294,11 +5299,9 @@ } genp5 = _is_chip_gen_p5_p7(cctx); - if (BNXT_RE_ABI_VERSION > 5) { - resp.modes = genp5 ? cctx->modes.wqe_mode : 0; - if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) - resp.comp_mask = BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED; - } + resp.modes = genp5 ? cctx->modes.wqe_mode : 0; + if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2)) + resp.comp_mask = BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED; resp.pg_size = PAGE_SIZE; resp.cqe_sz = sizeof(struct cq_base); Index: sys/dev/bnxt/bnxt_re/main.c =================================================================== --- sys/dev/bnxt/bnxt_re/main.c +++ sys/dev/bnxt/bnxt_re/main.c @@ -1411,12 +1411,14 @@ dev_info(rdev_to_dev(rdev), "Couldn't get DB bar size, Low latency framework is disabled\n"); /* set register offsets for both UC and WC */ - if (_is_chip_p7(cctx)) - res->dpi_tbl.ucreg.offset = offset; - else + if (_is_chip_p7(cctx)) { + res->dpi_tbl.ucreg.offset = en_dev->l2_db_offset; + res->dpi_tbl.wcreg.offset = en_dev->l2_db_size; + } else { res->dpi_tbl.ucreg.offset = res->is_vf ? BNXT_QPLIB_DBR_VF_DB_OFFSET : BNXT_QPLIB_DBR_PF_DB_OFFSET; - res->dpi_tbl.wcreg.offset = res->dpi_tbl.ucreg.offset; + res->dpi_tbl.wcreg.offset = res->dpi_tbl.ucreg.offset; + } /* If WC mapping is disabled by L2 driver then en_dev->l2_db_size * is equal to the DB-Bar actual size. This indicates that L2 @@ -3592,19 +3594,24 @@ goto release_rtnl; } + set_bit(BNXT_RE_FLAG_NET_RING_ALLOC, &rdev->flags); + if (!rdev->chip_ctx) goto release_rtnl; - /* Program the NQ ID for DBQ notification */ - if (rdev->chip_ctx->modes.dbr_pacing_v0 || - bnxt_qplib_dbr_pacing_en(rdev->chip_ctx) || - bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { - rc = bnxt_re_initialize_dbr_pacing(rdev); - if (!rc) - rdev->dbr_pacing = true; - else - rdev->dbr_pacing = false; - dev_dbg(rdev_to_dev(rdev), "%s: initialize db pacing ret %d\n", - __func__, rc); + + if (!(_is_chip_p7(rdev->chip_ctx))) { + /* Program the NQ ID for DBQ notification */ + if (rdev->chip_ctx->modes.dbr_pacing_v0 || + bnxt_qplib_dbr_pacing_en(rdev->chip_ctx) || + bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { + rc = bnxt_re_initialize_dbr_pacing(rdev); + if (!rc) + rdev->dbr_pacing = true; + else + rdev->dbr_pacing = false; + dev_dbg(rdev_to_dev(rdev), "%s: initialize db pacing ret %d\n", + __func__, rc); + } } vec = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].vector; Index: sys/dev/bnxt/bnxt_re/qplib_fp.h =================================================================== --- sys/dev/bnxt/bnxt_re/qplib_fp.h +++ sys/dev/bnxt/bnxt_re/qplib_fp.h @@ -390,6 +390,7 @@ u32 msn_tbl_sz; /* get devflags in PI code */ u16 dev_cap_flags; + bool is_host_msn_tbl; }; Index: sys/dev/bnxt/bnxt_re/qplib_fp.c =================================================================== --- sys/dev/bnxt/bnxt_re/qplib_fp.c +++ sys/dev/bnxt/bnxt_re/qplib_fp.c @@ -388,10 +388,14 @@ struct bnxt_qplib_srq *srq; struct nq_srq_event *nqsrqe = (struct nq_srq_event *)nqe; + u8 toggle; q_handle = le32_to_cpu(nqsrqe->srq_handle_low); q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high) << 32; srq = (struct bnxt_qplib_srq *)q_handle; + toggle = (le16_to_cpu(nqe->info10_type) & NQ_CN_TOGGLE_MASK) + >> NQ_CN_TOGGLE_SFT; + srq->dbinfo.toggle = toggle; bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA); if (!nq->srqn_handler(nq, @@ -1075,8 +1079,11 @@ u32 sqsz; qp->cctx = res->cctx; - if (res->dattr) + if (res->dattr) { qp->dev_cap_flags = res->dattr->dev_cap_flags; + qp->is_host_msn_tbl = _is_host_msn_table(res->dattr->dev_cap_ext_flags2); + } + /* General */ req.type = qp->type; req.dpi = cpu_to_le32(qp->dpi->dpi); @@ -1087,7 +1094,7 @@ psn_sz = _is_chip_gen_p5_p7(qp->cctx) ? sizeof(struct sq_psn_search_ext) : sizeof(struct sq_psn_search); - if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { + if (qp->is_host_msn_tbl) { psn_sz = sizeof(struct sq_msn_search); qp->msn = 0; } @@ -1103,7 +1110,7 @@ hwq_attr.aux_depth = (psn_sz) ? _set_sq_size(sq, qp->wqe_mode) : 0; /* Update msn tbl size */ - if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) { + if (qp->is_host_msn_tbl && psn_sz) { if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode)); else @@ -1131,8 +1138,7 @@ req.sq_pg_size_sq_lvl = pg_sz_lvl; req.sq_fwo_sq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP_SQ_FWO_SFT) & CMDQ_CREATE_QP_SQ_FWO_MASK) | - ((BNXT_RE_HW_RETX(qp->dev_cap_flags)) ? - BNXT_MSN_TBLE_SGE : sq->max_sge & + (sq->max_sge & CMDQ_CREATE_QP_SQ_SGE_MASK)); req.scq_cid = cpu_to_le32(qp->scq->id); @@ -1764,7 +1770,7 @@ return; /* Handle MSN differently on cap flags */ - if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { + if (qp->is_host_msn_tbl) { bnxt_qplib_fill_msn_search(qp, wqe, swq); return; } @@ -1897,7 +1903,7 @@ } static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_qp *qp, struct bnxt_qplib_q *sq, - struct bnxt_qplib_swq *swq, bool hw_retx) + struct bnxt_qplib_swq *swq, bool is_host_msn_tbl) { struct bnxt_qplib_hwq *sq_hwq; u32 pg_num, pg_indx; @@ -1909,8 +1915,11 @@ return; tail = swq->slot_idx / sq->dbinfo.max_slot; - if (hw_retx) + if (is_host_msn_tbl) { + /* For HW retx use qp msn index */ + tail = qp->msn; tail %= qp->msn_tbl_sz; + } pg_num = (tail + sq_hwq->pad_pgofft) / (PAGE_SIZE / sq_hwq->pad_stride); pg_indx = (tail + sq_hwq->pad_pgofft) % (PAGE_SIZE / sq_hwq->pad_stride); buff = (void *)(sq_hwq->pad_pg[pg_num] + pg_indx * sq_hwq->pad_stride); @@ -1935,6 +1944,7 @@ struct bnxt_qplib_swq *swq; bool sch_handler = false; u16 slots_needed; + bool msn_update; void *base_hdr; void *ext_hdr; __le32 temp32; @@ -1976,7 +1986,7 @@ sw_prod = sq_hwq->prod; swq = bnxt_qplib_get_swqe(sq, &wqe_idx); swq->slot_idx = sw_prod; - bnxt_qplib_pull_psn_buff(qp, sq, swq, BNXT_RE_HW_RETX(qp->dev_cap_flags)); + bnxt_qplib_pull_psn_buff(qp, sq, swq, qp->is_host_msn_tbl); swq->wr_id = wqe->wr_id; swq->type = wqe->type; @@ -2010,6 +2020,9 @@ wqe->num_sge, &sw_prod); if (data_len < 0) goto queue_err; + /* Make sure we update MSN table only for wired wqes */ + msn_update = true; + /* Specifics */ switch (wqe->type) { case BNXT_QPLIB_SWQE_TYPE_SEND: @@ -2064,6 +2077,7 @@ ext_sqe->avid = cpu_to_le32(wqe->send.avid & SQ_SEND_AVID_MASK); sq->psn = (sq->psn + 1) & BTH_PSN_MASK; + msn_update = false; } else { sqe->length = cpu_to_le32(data_len); if (qp->mtu) @@ -2157,6 +2171,7 @@ "\tflags = 0x%x\n" "\tinv_l_key = 0x%x\n", sqe->wqe_type, sqe->flags, sqe->inv_l_key); + msn_update = false; break; } case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR: @@ -2207,6 +2222,7 @@ sqe->zero_based_page_size_log, sqe->l_key, *(u32 *)sqe->length, sqe->numlevels_pbl_page_size_log, ext_sqe->pblptr, ext_sqe->va); + msn_update = false; break; } case BNXT_QPLIB_SWQE_TYPE_BIND_MW: @@ -2236,6 +2252,7 @@ sqe->wqe_type, sqe->flags, sqe->access_cntl, sqe->mw_type_zero_based, sqe->parent_l_key, sqe->l_key, sqe->va, ext_sqe->length_lo); + msn_update = false; break; } default: @@ -2243,8 +2260,10 @@ rc = -EINVAL; goto done; } - swq->next_psn = sq->psn & BTH_PSN_MASK; - bnxt_qplib_fill_psn_search(qp, wqe, swq); + if (!qp->is_host_msn_tbl || msn_update) { + swq->next_psn = sq->psn & BTH_PSN_MASK; + bnxt_qplib_fill_psn_search(qp, wqe, swq); + } queue_err: bnxt_qplib_swq_mod_start(sq, wqe_idx); Index: sys/dev/bnxt/bnxt_re/qplib_res.h =================================================================== --- sys/dev/bnxt/bnxt_re/qplib_res.h +++ sys/dev/bnxt/bnxt_re/qplib_res.h @@ -616,6 +616,12 @@ /* Disable HW_RETX */ #define BNXT_RE_HW_RETX(a) _is_hw_retx_supported((a)) +static inline bool _is_host_msn_table(u16 dev_cap_ext_flags2) +{ + return (dev_cap_ext_flags2 & CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_MASK) == + CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_MSN_TABLE; +} + static inline bool _is_cqe_v2_supported(u16 dev_cap_flags) { return dev_cap_flags & @@ -650,7 +656,7 @@ #define BNXT_QPLIB_INIT_DBHDR(xid, type, indx, toggle) \ (((u64)(((xid) & DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE | \ (type) | BNXT_QPLIB_DBR_VALID) << 32) | (indx) | \ - ((toggle) << (BNXT_QPLIB_DBR_TOGGLE_SHIFT))) + (((u32)(toggle)) << (BNXT_QPLIB_DBR_TOGGLE_SHIFT))) static inline void bnxt_qplib_write_db(struct bnxt_qplib_db_info *info, u64 key, void __iomem *db, @@ -724,7 +730,7 @@ u64 key = 0; u8 toggle = 0; - if (type == DBC_DBC_TYPE_CQ_ARMENA) + if (type == DBC_DBC_TYPE_CQ_ARMENA || type == DBC_DBC_TYPE_SRQ_ARMENA) toggle = info->toggle; /* Index always at 0 */ key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, 0, toggle); @@ -746,7 +752,7 @@ u64 key = 0; /* Index always at 0 */ - key = BNXT_QPLIB_INIT_DBHDR(info->xid, DBC_DBC_TYPE_SRQ_ARM, 0, 0); + key = BNXT_QPLIB_INIT_DBHDR(info->xid, DBC_DBC_TYPE_SRQ_ARM, 0, info->toggle); bnxt_qplib_write_db(info, key, info->priv_db, &info->shadow_key); } Index: sys/dev/bnxt/bnxt_re/qplib_res.c =================================================================== --- sys/dev/bnxt/bnxt_re/qplib_res.c +++ sys/dev/bnxt/bnxt_re/qplib_res.c @@ -1139,7 +1139,8 @@ ucreg->bar_id = RCFW_DBR_PCI_BAR_REGION; ucreg->bar_base = pci_resource_start(res->pdev, ucreg->bar_id); - ucreg->offset = 65536; + if (_is_chip_gen_p5(res->cctx)) + ucreg->offset = 65536; ucreg->len = ucreg->offset + PAGE_SIZE; Index: sys/dev/bnxt/bnxt_re/qplib_sp.h =================================================================== --- sys/dev/bnxt/bnxt_re/qplib_sp.h +++ sys/dev/bnxt/bnxt_re/qplib_sp.h @@ -71,6 +71,7 @@ u32 l2_db_size; u8 tqm_alloc_reqs[MAX_TQM_ALLOC_REQ]; u8 is_atomic; + u16 dev_cap_ext_flags2; u16 dev_cap_flags; u64 page_size_cap; u32 max_dpi; Index: sys/dev/bnxt/bnxt_re/qplib_sp.c =================================================================== --- sys/dev/bnxt/bnxt_re/qplib_sp.c +++ sys/dev/bnxt/bnxt_re/qplib_sp.c @@ -185,6 +185,7 @@ attr->page_size_cap = BIT_ULL(28) | BIT_ULL(21) | BIT_ULL(12); bnxt_qplib_query_version(rcfw, attr->fw_ver); + attr->dev_cap_ext_flags2 = le16_to_cpu(sb->dev_cap_ext_flags_2); for (i = 0; i < MAX_TQM_ALLOC_REQ / 4; i++) { temp = le32_to_cpu(sb->tqm_alloc_reqs[i]);