diff --git a/sys/dev/cxgbe/cxgbei/cxgbei.h b/sys/dev/cxgbe/cxgbei/cxgbei.h --- a/sys/dev/cxgbe/cxgbei/cxgbei.h +++ b/sys/dev/cxgbe/cxgbei/cxgbei.h @@ -119,7 +119,7 @@ /* cxgbei.c */ u_int cxgbei_select_worker_thread(struct icl_cxgbei_conn *); void cwt_queue_for_tx(struct icl_cxgbei_conn *); -void parse_pdus(struct icl_cxgbei_conn *, struct sockbuf *); +void parse_pdus(struct icl_cxgbei_conn *); /* icl_cxgbei.c */ void cwt_tx_main(void *); diff --git a/sys/dev/cxgbe/cxgbei/cxgbei.c b/sys/dev/cxgbe/cxgbei/cxgbei.c --- a/sys/dev/cxgbe/cxgbei/cxgbei.c +++ b/sys/dev/cxgbe/cxgbei/cxgbei.c @@ -403,15 +403,16 @@ } void -parse_pdus(struct icl_cxgbei_conn *icc, struct sockbuf *sb) +parse_pdus(struct icl_cxgbei_conn *icc) { struct icl_conn *ic = &icc->ic; struct socket *so = ic->ic_socket; + struct sockbuf *sb = &so->so_rcv; struct toepcb *toep = icc->toep; struct icl_pdu *ip, *lastip; u_int total_len; - SOCKBUF_LOCK_ASSERT(sb); + SOCK_RECVBUF_LOCK_ASSERT(so); CTR3(KTR_CXGBE, "%s: tid %u, %u bytes in so_rcv", __func__, toep->tid, sbused(sb)); @@ -419,13 +420,13 @@ lastip = NULL; while (sbused(sb) != 0 && (sb->sb_state & SBS_CANTRCVMORE) == 0) { total_len = sbused(sb); - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); ip = parse_pdu(so, toep, icc, sb, total_len); if (ip == NULL) { ic->ic_error(ic); - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_UNLOCK(so); return; } @@ -436,7 +437,7 @@ ip_next); lastip = ip; - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_UNLOCK(so); } } @@ -525,14 +526,14 @@ so = inp->inp_socket; sb = &so->so_rcv; - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_LOCK(so); icc = toep->ulpcb; if (__predict_false(icc == NULL || sb->sb_state & SBS_CANTRCVMORE)) { CTR5(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes), icc %p, sb_state 0x%x", __func__, tid, pdu_len, icc, sb->sb_state); - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); INP_WUNLOCK(inp); CURVNET_SET(so->so_vnet); @@ -552,7 +553,7 @@ ic = &icc->ic; if ((val & (F_DDP_PADDING_ERR | F_DDP_HDRCRC_ERR | F_DDP_DATACRC_ERR)) != 0) { - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); INP_WUNLOCK(inp); icl_cxgbei_conn_pdu_free(NULL, ip); @@ -568,7 +569,7 @@ icc->rx_active = true; wakeup(&icc->rx_active); } - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); INP_WUNLOCK(inp); toep->ulpcb2 = NULL; @@ -780,12 +781,12 @@ so = inp->inp_socket; sb = &so->so_rcv; - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_LOCK(so); if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { CTR5(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes), icc %p, sb_state 0x%x", __func__, tid, pdu_len, icc, sb->sb_state); - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); INP_WUNLOCK(inp); CURVNET_SET(so->so_vnet); @@ -811,7 +812,7 @@ icc->rx_active = true; wakeup(&icc->rx_active); } - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); INP_WUNLOCK(inp); toep->ulpcb2 = NULL; diff --git a/sys/dev/cxgbe/cxgbei/icl_cxgbei.c b/sys/dev/cxgbe/cxgbei/icl_cxgbei.c --- a/sys/dev/cxgbe/cxgbei/icl_cxgbei.c +++ b/sys/dev/cxgbe/cxgbei/icl_cxgbei.c @@ -500,17 +500,18 @@ { struct icl_cxgbei_conn *icc = arg; struct icl_conn *ic = &icc->ic; + struct socket *so = ic->ic_socket; + struct sockbuf *sb = &so->so_rcv; struct icl_pdu *ip; - struct sockbuf *sb; STAILQ_HEAD(, icl_pdu) rx_pdus = STAILQ_HEAD_INITIALIZER(rx_pdus); bool cantrcvmore; - sb = &ic->ic_socket->so_rcv; - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_LOCK(so); while (__predict_true(!ic->ic_disconnecting)) { while (STAILQ_EMPTY(&icc->rcvd_pdus)) { icc->rx_active = false; - mtx_sleep(&icc->rx_active, SOCKBUF_MTX(sb), 0, "-", 0); + mtx_sleep(&icc->rx_active, SOCK_RECVBUF_MTX(so), + 0, "-", 0); if (__predict_false(ic->ic_disconnecting)) goto out; MPASS(icc->rx_active); @@ -523,12 +524,12 @@ * them to icl_cxgbei_pdus and insert * them into the head of rcvd_pdus. */ - parse_pdus(icc, sb); + parse_pdus(icc); } cantrcvmore = (sb->sb_state & SBS_CANTRCVMORE) != 0; MPASS(STAILQ_EMPTY(&rx_pdus)); STAILQ_SWAP(&icc->rcvd_pdus, &rx_pdus, icl_pdu); - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); /* Hand over PDUs to ICL. */ while ((ip = STAILQ_FIRST(&rx_pdus)) != NULL) { @@ -539,18 +540,18 @@ ic->ic_receive(ip); } - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_LOCK(so); } out: /* - * Since ic_disconnecting is set before the SOCKBUF_MTX is + * Since ic_disconnecting is set before the SOCK_RECVBUF_MTX(so) is * locked in icl_cxgbei_conn_close, the loop above can exit - * before icl_cxgbei_conn_close can lock SOCKBUF_MTX and block + * before icl_cxgbei_conn_close can lock SOCK_RECVBUF_MTX(so) and block * waiting for the thread exit. */ while (!icc->rx_exiting) - mtx_sleep(&icc->rx_active, SOCKBUF_MTX(sb), 0, "-", 0); - SOCKBUF_UNLOCK(sb); + mtx_sleep(&icc->rx_active, SOCK_RECVBUF_MTX(so), 0, "-", 0); + SOCK_RECVBUF_LOCK(so); kthread_exit(); } @@ -953,12 +954,12 @@ error = soreserve(so, ss, rs); if (error != 0) return (error); - SOCKBUF_LOCK(&so->so_snd); + SOCK_SENDBUF_LOCK(so); so->so_snd.sb_flags |= SB_AUTOSIZE; - SOCKBUF_UNLOCK(&so->so_snd); - SOCKBUF_LOCK(&so->so_rcv); + SOCK_SENDBUF_UNLOCK(so); + SOCK_RECVBUF_LOCK(so); so->so_rcv.sb_flags |= SB_AUTOSIZE; - SOCKBUF_UNLOCK(&so->so_rcv); + SOCK_RECVBUF_UNLOCK(so); /* * Disable Nagle. @@ -1231,7 +1232,6 @@ struct icl_cxgbei_conn *icc = ic_to_icc(ic); struct icl_pdu *ip; struct socket *so; - struct sockbuf *sb; struct inpcb *inp; struct toepcb *toep = icc->toep; @@ -1275,17 +1275,16 @@ ICL_CONN_UNLOCK(ic); inp = sotoinpcb(so); - sb = &so->so_rcv; /* * Wait for the receive thread to stop processing this * connection. */ - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_LOCK(so); if (icc->rx_thread != NULL) { icc->rx_exiting = true; wakeup(&icc->rx_active); - mtx_sleep(icc->rx_thread, SOCKBUF_MTX(sb), 0, "conclo", 0); + mtx_sleep(icc->rx_thread, SOCK_RECVBUF_MTX(so), 0, "conclo", 0); } /* @@ -1296,7 +1295,7 @@ STAILQ_REMOVE_HEAD(&icc->rcvd_pdus, ip_next); icl_cxgbei_pdu_done(ip, ENOTCONN); } - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); INP_WLOCK(inp); if (toep != NULL) { /* NULL if connection was never offloaded. */ diff --git a/sys/dev/cxgbe/iw_cxgbe/cm.c b/sys/dev/cxgbe/iw_cxgbe/cm.c --- a/sys/dev/cxgbe/iw_cxgbe/cm.c +++ b/sys/dev/cxgbe/iw_cxgbe/cm.c @@ -844,10 +844,10 @@ so->so_state |= SS_NBIO; SOLISTEN_UNLOCK(so); } else { - SOCKBUF_LOCK(&so->so_rcv); + SOCK_RECVBUF_LOCK(so); soupcall_set(so, SO_RCV, c4iw_so_upcall, arg); so->so_state |= SS_NBIO; - SOCKBUF_UNLOCK(&so->so_rcv); + SOCK_RECVBUF_UNLOCK(so); } } @@ -859,9 +859,9 @@ solisten_upcall_set(so, NULL, NULL); SOLISTEN_UNLOCK(so); } else { - SOCKBUF_LOCK(&so->so_rcv); + SOCK_RECVBUF_LOCK(so); soupcall_clear(so, SO_RCV); - SOCKBUF_UNLOCK(&so->so_rcv); + SOCK_RECVBUF_UNLOCK(so); } } diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c --- a/sys/dev/cxgbe/tom/t4_cpl_io.c +++ b/sys/dev/cxgbe/tom/t4_cpl_io.c @@ -419,7 +419,7 @@ int rx_credits; INP_WLOCK_ASSERT(inp); - SOCKBUF_LOCK_ASSERT(sb); + SOCK_RECVBUF_LOCK_ASSERT(so); rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0; if (rx_credits > 0 && @@ -437,11 +437,10 @@ { struct inpcb *inp = tptoinpcb(tp); struct socket *so = inp->inp_socket; - struct sockbuf *sb = &so->so_rcv; - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_LOCK(so); t4_rcvd_locked(tod, tp); - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); } /* @@ -688,7 +687,7 @@ max_imm = max_imm_payload(tx_credits, 0); max_nsegs = max_dsgl_nsegs(tx_credits, 0); - SOCKBUF_LOCK(sb); + SOCK_SENDBUF_LOCK(so); sowwakeup = drop; if (drop) { sbdrop_locked(sb, drop); @@ -710,7 +709,7 @@ if (m->m_epg_tls != NULL) { toep->flags |= TPF_KTLS; if (plen == 0) { - SOCKBUF_UNLOCK(sb); + SOCK_SENDBUF_UNLOCK(so); t4_push_ktls(sc, toep, 0); return; } @@ -739,8 +738,8 @@ toep); sowwakeup_locked(so); } else - SOCKBUF_UNLOCK(sb); - SOCKBUF_UNLOCK_ASSERT(sb); + SOCK_SENDBUF_UNLOCK(so); + SOCK_SENDBUF_UNLOCK_ASSERT(so); return; } break; @@ -782,8 +781,8 @@ t4_aiotx_queue_toep(so, toep); sowwakeup_locked(so); } else - SOCKBUF_UNLOCK(sb); - SOCKBUF_UNLOCK_ASSERT(sb); + SOCK_SENDBUF_UNLOCK(so); + SOCK_SENDBUF_UNLOCK_ASSERT(so); /* nothing to send */ if (plen == 0) { @@ -860,10 +859,10 @@ tp->snd_nxt += plen; tp->snd_max += plen; - SOCKBUF_LOCK(sb); + SOCK_SENDBUF_LOCK(so); KASSERT(sb_sndptr, ("%s: sb_sndptr is NULL", __func__)); sb->sb_sndptr = sb_sndptr; - SOCKBUF_UNLOCK(sb); + SOCK_SENDBUF_UNLOCK(so); toep->flags |= TPF_TX_DATA_SENT; if (toep->tx_credits < MIN_OFLD_TX_CREDITS) @@ -1157,7 +1156,7 @@ * stay zero. */ if (__predict_false(sbused(sb)) > 0) { - SOCKBUF_LOCK(sb); + SOCK_SENDBUF_LOCK(so); sbu = sbused(sb); if (sbu > 0) { /* @@ -1705,13 +1704,13 @@ DDP_LOCK(toep); so = inp_inpcbtosocket(inp); sb = &so->so_rcv; - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_LOCK(so); if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) { CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)", __func__, tid, len); m_freem(m); - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); if (ulp_mode(toep) == ULP_MODE_TCPDDP) DDP_UNLOCK(toep); INP_WUNLOCK(inp); @@ -1790,9 +1789,9 @@ ddp_queue_toep(toep); } if (toep->flags & TPF_TLS_STARTING) - tls_received_starting_data(sc, toep, sb, len); + tls_received_starting_data(sc, toep, so, len); sorwakeup_locked(so); - SOCKBUF_UNLOCK_ASSERT(sb); + SOCK_RECVBUF_UNLOCK_ASSERT(so); if (ulp_mode(toep) == ULP_MODE_TCPDDP) DDP_UNLOCK(toep); @@ -1905,7 +1904,7 @@ struct sockbuf *sb = &so->so_snd; int sbu; - SOCKBUF_LOCK(sb); + SOCK_SENDBUF_LOCK(so); sbu = sbused(sb); if (ulp_mode(toep) == ULP_MODE_ISCSI) { if (__predict_false(sbu > 0)) { @@ -1930,7 +1929,7 @@ t4_aiotx_queue_toep(so, toep); sowwakeup_locked(so); /* unlocks so_snd */ } - SOCKBUF_UNLOCK_ASSERT(sb); + SOCK_SENDBUF_UNLOCK_ASSERT(so); } INP_WUNLOCK(inp); @@ -2180,7 +2179,7 @@ bool moretocome, sendmore; sb = &so->so_snd; - SOCKBUF_UNLOCK(sb); + SOCK_SENDBUF_UNLOCK(so); m = NULL; #ifdef MAC @@ -2195,9 +2194,9 @@ MPASS(error == 0); sendanother: - SOCKBUF_LOCK(sb); + SOCK_SENDBUF_LOCK(so); if (so->so_snd.sb_state & SBS_CANTSENDMORE) { - SOCKBUF_UNLOCK(sb); + SOCK_SENDBUF_UNLOCK(so); SOCK_IO_SEND_UNLOCK(so); if ((so->so_options & SO_NOSIGPIPE) == 0) { PROC_LOCK(job->userproc); @@ -2210,12 +2209,12 @@ if (so->so_error) { error = so->so_error; so->so_error = 0; - SOCKBUF_UNLOCK(sb); + SOCK_SENDBUF_UNLOCK(so); SOCK_IO_SEND_UNLOCK(so); goto out; } if ((so->so_state & SS_ISCONNECTED) == 0) { - SOCKBUF_UNLOCK(sb); + SOCK_SENDBUF_UNLOCK(so); SOCK_IO_SEND_UNLOCK(so); error = ENOTCONN; goto out; @@ -2228,13 +2227,13 @@ * buffer. Instead, requeue the request. */ if (!aio_set_cancel_function(job, t4_aiotx_cancel)) { - SOCKBUF_UNLOCK(sb); + SOCK_SENDBUF_UNLOCK(so); SOCK_IO_SEND_UNLOCK(so); error = ECANCELED; goto out; } TAILQ_INSERT_HEAD(&toep->aiotx_jobq, job, list); - SOCKBUF_UNLOCK(sb); + SOCK_SENDBUF_UNLOCK(so); SOCK_IO_SEND_UNLOCK(so); goto out; } @@ -2257,7 +2256,7 @@ if (!TAILQ_EMPTY(&toep->aiotx_jobq)) moretocome = true; - SOCKBUF_UNLOCK(sb); + SOCK_SENDBUF_UNLOCK(so); MPASS(len != 0); m = alloc_aiotx_mbuf(job, len); @@ -2315,9 +2314,9 @@ */ if (job->aio_sent < job->uaiocb.aio_nbytes && !(so->so_state & SS_NBIO)) { - SOCKBUF_LOCK(sb); + SOCK_SENDBUF_LOCK(so); if (!aio_set_cancel_function(job, t4_aiotx_cancel)) { - SOCKBUF_UNLOCK(sb); + SOCK_SENDBUF_UNLOCK(so); error = ECANCELED; goto out; } @@ -2341,7 +2340,7 @@ aiotx_free_job(job); } m_freem(m); - SOCKBUF_LOCK(sb); + SOCK_SENDBUF_LOCK(so); } static void @@ -2355,7 +2354,7 @@ so = toep->aiotx_so; CURVNET_SET(toep->vnet); NET_EPOCH_ENTER(et); - SOCKBUF_LOCK(&so->so_snd); + SOCK_SENDBUF_LOCK(so); while (!TAILQ_EMPTY(&toep->aiotx_jobq) && sowriteable(so)) { job = TAILQ_FIRST(&toep->aiotx_jobq); TAILQ_REMOVE(&toep->aiotx_jobq, job, list); @@ -2365,7 +2364,7 @@ t4_aiotx_process_job(toep, so, job); } toep->aiotx_so = NULL; - SOCKBUF_UNLOCK(&so->so_snd); + SOCK_SENDBUF_UNLOCK(so); NET_EPOCH_EXIT(et); free_toepcb(toep); @@ -2377,7 +2376,7 @@ t4_aiotx_queue_toep(struct socket *so, struct toepcb *toep) { - SOCKBUF_LOCK_ASSERT(&toep->inp->inp_socket->so_snd); + SOCK_SENDBUF_LOCK_ASSERT(toep->inp->inp_socket); #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: queueing aiotx task for tid %d, active = %s", __func__, toep->tid, toep->aiotx_so != NULL ? "true" : "false"); @@ -2394,7 +2393,6 @@ t4_aiotx_cancel(struct kaiocb *job) { struct socket *so; - struct sockbuf *sb; struct tcpcb *tp; struct toepcb *toep; @@ -2402,12 +2400,11 @@ tp = sototcpcb(so); toep = tp->t_toe; MPASS(job->uaiocb.aio_lio_opcode == LIO_WRITE); - sb = &so->so_snd; - SOCKBUF_LOCK(sb); + SOCK_SENDBUF_LOCK(so); if (!aio_cancel_cleared(job)) TAILQ_REMOVE(&toep->aiotx_jobq, job, list); - SOCKBUF_UNLOCK(sb); + SOCK_SENDBUF_UNLOCK(so); job->aio_error = (void *)(intptr_t)ECANCELED; aiotx_free_job(job); @@ -2430,7 +2427,7 @@ if (tls_tx_key(toep)) return (EOPNOTSUPP); - SOCKBUF_LOCK(&so->so_snd); + SOCK_SENDBUF_LOCK(so); #ifdef VERBOSE_TRACES CTR3(KTR_CXGBE, "%s: queueing %p for tid %u", __func__, job, toep->tid); #endif @@ -2440,7 +2437,7 @@ TAILQ_INSERT_TAIL(&toep->aiotx_jobq, job, list); if (sowriteable(so)) t4_aiotx_queue_toep(so, toep); - SOCKBUF_UNLOCK(&so->so_snd); + SOCK_SENDBUF_UNLOCK(so); return (0); } diff --git a/sys/dev/cxgbe/tom/t4_ddp.c b/sys/dev/cxgbe/tom/t4_ddp.c --- a/sys/dev/cxgbe/tom/t4_ddp.c +++ b/sys/dev/cxgbe/tom/t4_ddp.c @@ -430,7 +430,6 @@ queue_ddp_rcvbuf_mbuf(struct toepcb *toep, u_int db_idx, u_int len) { struct inpcb *inp = toep->inp; - struct sockbuf *sb; struct ddp_buffer *db; struct ddp_rcv_buffer *drb; struct mbuf *m; @@ -449,9 +448,8 @@ m->m_pkthdr.len = len; m->m_len = len; - sb = &inp->inp_socket->so_rcv; - SOCKBUF_LOCK_ASSERT(sb); - sbappendstream_locked(sb, m, 0); + SOCK_RECVBUF_LOCK_ASSERT(so); + sbappendstream_locked(&inp->inp_socket->so_rcv, m, 0); db->placed += len; toep->ofld_rxq->rx_toe_ddp_octets += len; @@ -736,7 +734,7 @@ /* receive buffer autosize */ MPASS(toep->vnet == so->so_vnet); CURVNET_SET(toep->vnet); - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_LOCK(so); if (sb->sb_flags & SB_AUTOSIZE && V_tcp_do_autorcvbuf && sb->sb_hiwat < V_tcp_autorcvbuf_max && @@ -749,7 +747,7 @@ if (!sbreserve_locked(so, SO_RCV, newsize, NULL)) sb->sb_flags &= ~SB_AUTOSIZE; } - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); CURVNET_RESTORE(); job->msgrcv = 1; @@ -953,7 +951,7 @@ /* receive buffer autosize */ MPASS(toep->vnet == so->so_vnet); CURVNET_SET(toep->vnet); - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_LOCK(so); if (sb->sb_flags & SB_AUTOSIZE && V_tcp_do_autorcvbuf && sb->sb_hiwat < V_tcp_autorcvbuf_max && @@ -972,7 +970,7 @@ t4_rcvd_locked(&toep->td->tod, tp); } sorwakeup_locked(so); - SOCKBUF_UNLOCK_ASSERT(sb); + SOCK_RECVBUF_UNLOCK_ASSERT(so); CURVNET_RESTORE(); if (invalidated) @@ -1121,7 +1119,6 @@ handle_ddp_close(struct toepcb *toep, struct tcpcb *tp, __be32 rcv_nxt) { struct socket *so = toep->inp->inp_socket; - struct sockbuf *sb = &so->so_rcv; struct ddp_buffer *db; struct kaiocb *job; long copied; @@ -1156,10 +1153,10 @@ if (placed > db->drb->len - db->placed) placed = db->drb->len - db->placed; if (placed != 0) { - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_LOCK(so); queue_ddp_rcvbuf_mbuf(toep, db_idx, placed); sorwakeup_locked(so); - SOCKBUF_UNLOCK_ASSERT(sb); + SOCK_RECVBUF_UNLOCK_ASSERT(so); } complete_ddp_buffer(toep, db, db_idx); len -= placed; @@ -2380,11 +2377,11 @@ job = TAILQ_FIRST(&toep->ddp.aiojobq); so = job->fd_file->f_data; sb = &so->so_rcv; - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_LOCK(so); /* We will never get anything unless we are or were connected. */ if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) { - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); ddp_complete_all(toep, ENOTCONN); return; } @@ -2398,7 +2395,7 @@ toep->ddp.waiting_count--; TAILQ_REMOVE(&toep->ddp.aiojobq, job, list); if (!aio_clear_cancel_function(job)) { - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); goto restart; } @@ -2409,13 +2406,13 @@ */ copied = job->aio_received; if (copied != 0) { - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); aio_complete(job, copied, 0); goto restart; } error = so->so_error; so->so_error = 0; - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); aio_complete(job, -1, error); goto restart; } @@ -2426,7 +2423,7 @@ * to complete. Once they have completed, return EOF reads. */ if (sb->sb_state & SBS_CANTRCVMORE && sbavail(sb) == 0) { - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); if (toep->ddp.active_count != 0) return; ddp_complete_all(toep, 0); @@ -2438,7 +2435,7 @@ * data, try to enable DDP. */ if (sbavail(sb) == 0 && (toep->ddp.flags & DDP_ON) == 0) { - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); /* * Wait for the card to ACK that DDP is enabled before @@ -2455,7 +2452,7 @@ enable_ddp(sc, toep); return; } - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); /* * If another thread is queueing a buffer for DDP, let it @@ -2479,11 +2476,11 @@ goto restart; } - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_LOCK(so); if (so->so_error && sbavail(sb) == 0) { copied = job->aio_received; if (copied != 0) { - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); recycle_pageset(toep, ps); aio_complete(job, copied, 0); toep->ddp.queueing = NULL; @@ -2492,7 +2489,7 @@ error = so->so_error; so->so_error = 0; - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); recycle_pageset(toep, ps); aio_complete(job, -1, error); toep->ddp.queueing = NULL; @@ -2500,7 +2497,7 @@ } if (sb->sb_state & SBS_CANTRCVMORE && sbavail(sb) == 0) { - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); recycle_pageset(toep, ps); if (toep->ddp.active_count != 0) { /* @@ -2578,11 +2575,11 @@ * Our caller has a reference on the 'toep' that * keeps it stable. */ - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); DDP_UNLOCK(toep); INP_WLOCK(inp); DDP_LOCK(toep); - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_LOCK(so); /* * If the socket has been closed, we should detect @@ -2599,7 +2596,7 @@ * is being shut down, so complete the * request. */ - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); recycle_pageset(toep, ps); aio_complete(job, copied, 0); toep->ddp.queueing = NULL; @@ -2612,7 +2609,7 @@ * arrive on the socket buffer. */ if ((toep->ddp.flags & (DDP_ON | DDP_SC_REQ)) != DDP_ON) { - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); recycle_pageset(toep, ps); aio_ddp_requeue_one(toep, job); toep->ddp.queueing = NULL; @@ -2627,7 +2624,7 @@ if (sbavail(sb) != 0) goto sbcopy; } - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); if (prep_pageset(sc, toep, ps) == 0) { recycle_pageset(toep, ps); @@ -2928,20 +2925,20 @@ } /* We will never get anything unless we are or were connected. */ - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_LOCK(so); if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) { - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); recycle_ddp_rcv_buffer(toep, drb); return; } /* Abort if socket has reported problems or is closed. */ if (so->so_error != 0 || (sb->sb_state & SBS_CANTRCVMORE) != 0) { - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); recycle_ddp_rcv_buffer(toep, drb); return; } - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); if (!queue_ddp_rcvbuf(toep, drb)) { /* diff --git a/sys/dev/cxgbe/tom/t4_tls.c b/sys/dev/cxgbe/tom/t4_tls.c --- a/sys/dev/cxgbe/tom/t4_tls.c +++ b/sys/dev/cxgbe/tom/t4_tls.c @@ -1208,16 +1208,16 @@ */ static void tls_check_rx_sockbuf(struct adapter *sc, struct toepcb *toep, - struct sockbuf *sb) + struct socket *so) { uint64_t seqno; size_t resid; bool have_header; - SOCKBUF_LOCK_ASSERT(sb); + SOCK_RECVBUF_LOCK_ASSERT(so); MPASS(toep->tls.rx_resid == 0); - have_header = ktls_pending_rx_info(sb, &seqno, &resid); + have_header = ktls_pending_rx_info(so, &seqno, &resid); CTR5(KTR_CXGBE, "%s: tid %d have_header %d seqno %lu resid %zu", __func__, toep->tid, have_header, seqno, resid); @@ -1239,7 +1239,7 @@ void tls_received_starting_data(struct adapter *sc, struct toepcb *toep, - struct sockbuf *sb, int len) + struct socket *so, int len) { MPASS(toep->flags & TPF_TLS_STARTING); @@ -1266,7 +1266,7 @@ if (toep->tls.rx_resid != 0) return; - tls_check_rx_sockbuf(sc, toep, sb); + tls_check_rx_sockbuf(sc, toep, so); } static int @@ -1278,7 +1278,6 @@ struct toepcb *toep; struct inpcb *inp; struct socket *so; - struct sockbuf *sb; if (cpl->status != CPL_ERR_NONE) panic("XXX: tcp_rpl failed: %d", cpl->status); @@ -1297,10 +1296,9 @@ toep->flags |= TPF_TLS_RX_QUIESCED; so = inp->inp_socket; - sb = &so->so_rcv; - SOCKBUF_LOCK(sb); - tls_check_rx_sockbuf(sc, toep, sb); - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_LOCK(so); + tls_check_rx_sockbuf(sc, toep, so); + SOCK_RECVBUF_UNLOCK(so); INP_WUNLOCK(inp); break; default: diff --git a/sys/dev/cxgbe/tom/t4_tom.h b/sys/dev/cxgbe/tom/t4_tom.h --- a/sys/dev/cxgbe/tom/t4_tom.h +++ b/sys/dev/cxgbe/tom/t4_tom.h @@ -551,7 +551,7 @@ void do_rx_data_tls(const struct cpl_rx_data *, struct toepcb *, struct mbuf *); void t4_push_ktls(struct adapter *, struct toepcb *, int); void tls_received_starting_data(struct adapter *, struct toepcb *, - struct sockbuf *, int); + struct socket *, int); void t4_tls_mod_load(void); void t4_tls_mod_unload(void); void tls_init_toep(struct toepcb *); diff --git a/sys/dev/cxgbe/tom/t4_tom.c b/sys/dev/cxgbe/tom/t4_tom.c --- a/sys/dev/cxgbe/tom/t4_tom.c +++ b/sys/dev/cxgbe/tom/t4_tom.c @@ -229,17 +229,17 @@ /* Update socket */ sb = &so->so_snd; - SOCKBUF_LOCK(sb); + SOCK_SENDBUF_LOCK(so); sb->sb_flags |= SB_NOCOALESCE; - SOCKBUF_UNLOCK(sb); + SOCK_SENDBUF_UNLOCK(so); sb = &so->so_rcv; - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_LOCK(so); sb->sb_flags |= SB_NOCOALESCE; if (inp->inp_vflag & INP_IPV6) so->so_proto = &toe6_protosw; else so->so_proto = &toe_protosw; - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); /* Update TCP PCB */ tp->tod = &td->tod; @@ -279,14 +279,14 @@ INP_WLOCK_ASSERT(inp); sb = &so->so_snd; - SOCKBUF_LOCK(sb); + SOCK_SENDBUF_LOCK(so); sb->sb_flags &= ~SB_NOCOALESCE; - SOCKBUF_UNLOCK(sb); + SOCK_SENDBUF_UNLOCK(so); sb = &so->so_rcv; - SOCKBUF_LOCK(sb); + SOCK_RECVBUF_LOCK(so); sb->sb_flags &= ~SB_NOCOALESCE; restore_so_proto(so, inp->inp_vflag & INP_IPV6); - SOCKBUF_UNLOCK(sb); + SOCK_RECVBUF_UNLOCK(so); tp->tod = NULL; tp->t_toe = NULL; @@ -1110,7 +1110,7 @@ { unsigned long wnd; - SOCKBUF_LOCK_ASSERT(&so->so_rcv); + SOCK_RECVBUF_LOCK_ASSERT(so); wnd = sbspace(&so->so_rcv); if (wnd < MIN_RCV_WND) @@ -1430,21 +1430,21 @@ else cp->ecn = 0; - SOCKBUF_LOCK(&so->so_rcv); + SOCK_RECVBUF_LOCK(so); wnd = max(select_rcv_wnd(so), MIN_RCV_WND); - SOCKBUF_UNLOCK(&so->so_rcv); + SOCK_RECVBUF_UNLOCK(so); cp->opt0_bufsize = min(wnd >> 10, M_RCV_BUFSIZ); if (tt->sndbuf > 0) cp->sndbuf = tt->sndbuf; else { - SOCKBUF_LOCK(&so->so_snd); + SOCK_SENDBUF_LOCK(so); if (so->so_snd.sb_flags & SB_AUTOSIZE && V_tcp_do_autosndbuf) cp->sndbuf = 256 * 1024; else cp->sndbuf = so->so_snd.sb_hiwat; - SOCKBUF_UNLOCK(&so->so_snd); + SOCK_SENDBUF_UNLOCK(so); } } diff --git a/sys/kern/uipc_ktls.c b/sys/kern/uipc_ktls.c --- a/sys/kern/uipc_ktls.c +++ b/sys/kern/uipc_ktls.c @@ -1229,16 +1229,18 @@ * is the number of bytes needed to complete the TLS header. */ bool -ktls_pending_rx_info(struct sockbuf *sb, uint64_t *seqnop, size_t *residp) +ktls_pending_rx_info(struct socket *so, uint64_t *seqnop, size_t *residp) { + struct sockbuf *sb = &so->so_rcv; struct tls_record_layer hdr; struct mbuf *m; uint64_t seqno; size_t resid; u_int offset, record_len; - SOCKBUF_LOCK_ASSERT(sb); + SOCK_RECVBUF_LOCK_ASSERT(so); MPASS(sb->sb_flags & SB_TLS_RX); + seqno = sb->sb_tls_seqno; resid = sb->sb_tlscc; m = sb->sb_mtls; diff --git a/sys/sys/ktls.h b/sys/sys/ktls.h --- a/sys/sys/ktls.h +++ b/sys/sys/ktls.h @@ -233,7 +233,7 @@ int ktls_modify_txrtlmt(struct ktls_session *tls, uint64_t max_pacing_rate); #endif int ktls_output_eagain(struct inpcb *inp, struct ktls_session *tls); -bool ktls_pending_rx_info(struct sockbuf *sb, uint64_t *seqnop, size_t *residp); +bool ktls_pending_rx_info(struct socket *so, uint64_t *seqnop, size_t *residp); bool ktls_permit_empty_frames(struct ktls_session *tls); void ktls_seq(struct sockbuf *sb, struct mbuf *m); int ktls_set_tx_mode(struct socket *so, int mode);