Lines Matching +full:ps +full:- +full:seq +full:- +full:loop
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
100 * reuse of buffers (avoiding the need to re-fault in pages, hold
104 * is a trade-off for performance.
106 * If an application ping-pongs two buffers for a connection via
113 free_pageset(struct tom_data *td, struct pageset *ps)
118 if (ps->prsv.prsv_nppods > 0)
119 t4_free_page_pods(&ps->prsv);
121 for (i = 0; i < ps->npages; i++) {
122 p = ps->pages[i];
126 TAILQ_INSERT_TAIL(&ddp_orphan_pagesets, ps, link);
134 struct pageset *ps;
138 ps = TAILQ_FIRST(&ddp_orphan_pagesets);
139 TAILQ_REMOVE(&ddp_orphan_pagesets, ps, link);
141 if (ps->vm)
142 vmspace_free(ps->vm);
143 free(ps, M_CXGBE);
150 recycle_pageset(struct toepcb *toep, struct pageset *ps)
154 if (!(toep->ddp.flags & DDP_DEAD)) {
155 KASSERT(toep->ddp.cached_count + toep->ddp.active_count <
156 nitems(toep->ddp.db), ("too many wired pagesets"));
157 TAILQ_INSERT_HEAD(&toep->ddp.cached_pagesets, ps, link);
158 toep->ddp.cached_count++;
160 free_pageset(toep->td, ps);
173 copied = job->aio_received;
177 aio_complete(job, -1, error);
183 t4_free_page_pods(&drb->prsv);
184 free(drb->buf, M_CXGBE);
186 counter_u64_add(toep->ofld_rxq->ddp_buffer_free, 1);
194 if (!(toep->ddp.flags & DDP_DEAD) &&
195 toep->ddp.cached_count < t4_ddp_rcvbuf_cache) {
196 TAILQ_INSERT_HEAD(&toep->ddp.cached_buffers, drb, link);
197 toep->ddp.cached_count++;
211 if (!TAILQ_EMPTY(&toep->ddp.cached_buffers)) {
212 drb = TAILQ_FIRST(&toep->ddp.cached_buffers);
213 TAILQ_REMOVE(&toep->ddp.cached_buffers, drb, link);
214 toep->ddp.cached_count--;
215 counter_u64_add(toep->ofld_rxq->ddp_buffer_reuse, 1);
225 struct tom_data *td = toep->td;
234 drb->buf = contigmalloc(t4_ddp_rcvbuf_len, M_CXGBE, how, 0, ~0,
236 if (drb->buf == NULL) {
240 drb->len = t4_ddp_rcvbuf_len;
241 drb->refs = 1;
243 error = t4_alloc_page_pods_for_rcvbuf(&td->pr, drb);
245 free(drb->buf, M_CXGBE);
250 error = t4_write_page_pods_for_rcvbuf(sc, toep->ctrlq, toep->tid, drb);
252 t4_free_page_pods(&drb->prsv);
253 free(drb->buf, M_CXGBE);
259 counter_u64_add(toep->ofld_rxq->ddp_buffer_alloc, 1);
266 if ((toep->ddp.flags & DDP_RCVBUF) != 0) {
267 if (db->drb != NULL)
268 free_ddp_rcv_buffer(toep, db->drb);
270 db->drb = NULL;
275 if (db->job) {
277 * XXX: If we are un-offloading the socket then we
282 if (!aio_clear_cancel_function(db->job))
283 ddp_complete_one(db->job, 0);
285 db->job = NULL;
289 if (db->ps) {
290 free_pageset(toep->td, db->ps);
292 db->ps = NULL;
301 toep->ddp.flags = DDP_OK;
302 toep->ddp.active_id = -1;
303 mtx_init(&toep->ddp.lock, "t4 ddp", NULL, MTX_DEF);
304 mtx_init(&toep->ddp.cache_lock, "t4 ddp cache", NULL, MTX_DEF);
311 mtx_destroy(&toep->ddp.lock);
312 mtx_destroy(&toep->ddp.cache_lock);
319 struct pageset *ps;
324 toep->ddp.flags |= DDP_DEAD;
326 for (i = 0; i < nitems(toep->ddp.db); i++) {
327 free_ddp_buffer(toep, &toep->ddp.db[i]);
329 if ((toep->ddp.flags & DDP_AIO) != 0) {
330 while ((ps = TAILQ_FIRST(&toep->ddp.cached_pagesets)) != NULL) {
331 TAILQ_REMOVE(&toep->ddp.cached_pagesets, ps, link);
332 free_pageset(toep->td, ps);
336 if ((toep->ddp.flags & DDP_RCVBUF) != 0) {
338 while ((drb = TAILQ_FIRST(&toep->ddp.cached_buffers)) != NULL) {
339 TAILQ_REMOVE(&toep->ddp.cached_buffers, drb, link);
353 MPASS((toep->ddp.flags & (DDP_TASK_ACTIVE | DDP_DEAD)) != DDP_TASK_ACTIVE);
354 for (i = 0; i < nitems(toep->ddp.db); i++) {
355 if ((toep->ddp.flags & DDP_AIO) != 0) {
356 MPASS(toep->ddp.db[i].job == NULL);
357 MPASS(toep->ddp.db[i].ps == NULL);
359 MPASS(toep->ddp.db[i].drb == NULL);
361 if ((toep->ddp.flags & DDP_AIO) != 0) {
362 MPASS(TAILQ_EMPTY(&toep->ddp.cached_pagesets));
363 MPASS(TAILQ_EMPTY(&toep->ddp.aiojobq));
365 if ((toep->ddp.flags & DDP_RCVBUF) != 0)
366 MPASS(TAILQ_EMPTY(&toep->ddp.cached_buffers));
377 toep->ddp.active_count--;
378 if (toep->ddp.active_id == db_idx) {
379 if (toep->ddp.active_count == 0) {
380 if ((toep->ddp.flags & DDP_AIO) != 0)
381 KASSERT(toep->ddp.db[db_idx ^ 1].job == NULL,
384 KASSERT(toep->ddp.db[db_idx ^ 1].drb == NULL,
386 toep->ddp.active_id = -1;
388 toep->ddp.active_id ^= 1;
391 toep->tid, toep->ddp.active_id);
394 KASSERT(toep->ddp.active_count != 0 &&
395 toep->ddp.active_id != -1,
399 if ((toep->ddp.flags & DDP_AIO) != 0) {
400 db->cancel_pending = 0;
401 db->job = NULL;
402 recycle_pageset(toep, db->ps);
403 db->ps = NULL;
405 drb = db->drb;
406 if (atomic_fetchadd_int(&drb->refs, -1) == 1)
408 db->drb = NULL;
409 db->placed = 0;
413 KASSERT(toep->ddp.flags & db_flag,
415 __func__, toep, toep->ddp.flags));
416 toep->ddp.flags &= ~db_flag;
423 struct toepcb *toep = m->m_ext.ext_arg1;
424 struct ddp_rcv_buffer *drb = m->m_ext.ext_arg2;
432 struct inpcb *inp = toep->inp;
443 m->m_pkthdr.rcvif = toep->vi->ifp;
445 db = &toep->ddp.db[db_idx];
446 drb = db->drb;
447 m_extaddref(m, (char *)drb->buf + db->placed, len, &drb->refs,
449 m->m_pkthdr.len = len;
450 m->m_len = len;
452 sb = &inp->inp_socket->so_rcv;
456 db->placed += len;
457 toep->ofld_rxq->rx_toe_ddp_octets += len;
464 struct inpcb *inp = toep->inp;
479 ddp_rcvbuf = (toep->ddp.flags & DDP_RCVBUF) != 0;
480 tp->rcv_nxt += n;
482 KASSERT(tp->rcv_wnd >= n, ("%s: negative window size", __func__));
483 tp->rcv_wnd -= n;
487 while (toep->ddp.active_count > 0) {
488 MPASS(toep->ddp.active_id != -1);
489 db_idx = toep->ddp.active_id;
493 MPASS((toep->ddp.flags & db_flag) != 0);
494 db = &toep->ddp.db[db_idx];
497 if (placed > db->drb->len - db->placed)
498 placed = db->drb->len - db->placed;
502 n -= placed;
505 job = db->job;
506 copied = job->aio_received;
508 if (placed > job->uaiocb.aio_nbytes - copied)
509 placed = job->uaiocb.aio_nbytes - copied;
511 job->msgrcv = 1;
512 toep->ofld_rxq->rx_aio_ddp_jobs++;
514 toep->ofld_rxq->rx_aio_ddp_octets += placed;
521 job->aio_received += placed;
529 TAILQ_INSERT_HEAD(&toep->ddp.aiojobq, job, list);
530 toep->ddp.waiting_count++;
533 n -= placed;
554 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
555 ulpmc->len = htobe32(howmany(LEN__RX_DATA_ACK_ULP, 16));
558 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
559 ulpsc->len = htobe32(sizeof(*req));
562 OPCODE_TID(req) = htobe32(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tid));
563 req->credit_dack = htobe32(F_RX_MODULATE_RX);
567 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
568 ulpsc->len = htobe32(0);
598 wr = alloc_wrqe(wrlen, toep->ctrlq);
606 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid,
609 V_TCB_RX_DDP_BUF0_TAG(prsv->prsv_tag));
613 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid,
620 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid,
628 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_RX_DDP_FLAGS,
643 struct inpcb *inp = toep->inp;
658 sb = &so->so_rcv;
661 KASSERT(toep->ddp.active_id == db_idx,
663 toep->ddp.active_id, toep->tid));
664 db = &toep->ddp.db[db_idx];
665 job = db->job;
667 if (__predict_false(inp->inp_flags & INP_DROPPED)) {
672 CTR5(KTR_CXGBE, "%s: tid %u, seq 0x%x, len %d, inp_flags 0x%x",
673 __func__, toep->tid, be32toh(rcv_nxt), len, inp->inp_flags);
687 * For RX_DATA_DDP, len might be non-zero, but it is only the
693 len += be32toh(rcv_nxt) - tp->rcv_nxt;
694 tp->rcv_nxt += len;
695 tp->t_rcvtime = ticks;
697 KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__));
698 tp->rcv_wnd -= len;
702 toep->tid, db_idx, len, report);
706 MPASS(toep->vnet == so->so_vnet);
707 CURVNET_SET(toep->vnet);
709 if (sb->sb_flags & SB_AUTOSIZE &&
711 sb->sb_hiwat < V_tcp_autorcvbuf_max &&
713 struct adapter *sc = td_adapter(toep->td);
714 unsigned int hiwat = sb->sb_hiwat;
715 unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc,
719 sb->sb_flags &= ~SB_AUTOSIZE;
724 job->msgrcv = 1;
725 toep->ofld_rxq->rx_aio_ddp_jobs++;
726 toep->ofld_rxq->rx_aio_ddp_octets += len;
727 if (db->cancel_pending) {
732 job->aio_received += len;
739 job->aio_received += len;
741 copied = job->aio_received;
745 __func__, toep->tid, job, copied, len);
748 t4_rcvd(&toep->td->tod, tp);
753 if (toep->ddp.waiting_count > 0)
765 struct adapter *sc = td_adapter(toep->td);
773 KASSERT((toep->ddp.flags & DDP_DEAD) == 0, ("%s: DDP_DEAD", __func__));
774 KASSERT(toep->ddp.active_count < nitems(toep->ddp.db),
778 if (toep->ddp.db[0].drb == NULL) {
781 MPASS(toep->ddp.db[1].drb == NULL);
809 MPASS((toep->ddp.flags & buf_flag) == 0);
810 if ((toep->ddp.flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE)) == 0) {
812 MPASS(toep->ddp.active_id == -1);
813 MPASS(toep->ddp.active_count == 0);
822 wr = mk_update_tcb_for_ddp(sc, toep, db_idx, &drb->prsv, 0, drb->len,
833 toep->tid, db_idx, ddp_flags, ddp_flags_mask);
839 drb->refs = 1;
841 /* Give the chip the go-ahead. */
843 db = &toep->ddp.db[db_idx];
844 db->drb = drb;
845 toep->ddp.flags |= buf_flag;
846 toep->ddp.active_count++;
847 if (toep->ddp.active_count == 1) {
848 MPASS(toep->ddp.active_id == -1);
849 toep->ddp.active_id = db_idx;
851 toep->ddp.active_id);
861 struct inpcb *inp = toep->inp;
876 sb = &so->so_rcv;
879 KASSERT(toep->ddp.active_id == db_idx,
881 toep->ddp.active_id, toep->tid));
882 db = &toep->ddp.db[db_idx];
884 if (__predict_false(inp->inp_flags & INP_DROPPED)) {
889 CTR5(KTR_CXGBE, "%s: tid %u, seq 0x%x, len %d, inp_flags 0x%x",
890 __func__, toep->tid, be32toh(rcv_nxt), len, inp->inp_flags);
904 * For RX_DATA_DDP, len might be non-zero, but it is only the
910 len += be32toh(rcv_nxt) - tp->rcv_nxt;
911 tp->rcv_nxt += len;
912 tp->t_rcvtime = ticks;
914 KASSERT(tp->rcv_wnd >= len, ("%s: negative window size", __func__));
915 tp->rcv_wnd -= len;
919 toep->tid, db_idx, len, report);
923 MPASS(toep->vnet == so->so_vnet);
924 CURVNET_SET(toep->vnet);
926 if (sb->sb_flags & SB_AUTOSIZE &&
928 sb->sb_hiwat < V_tcp_autorcvbuf_max &&
930 struct adapter *sc = td_adapter(toep->td);
931 unsigned int hiwat = sb->sb_hiwat;
932 unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc,
936 sb->sb_flags &= ~SB_AUTOSIZE;
941 t4_rcvd_locked(&toep->td->tod, tp);
950 KASSERT(db->placed < db->drb->len,
953 if (toep->ddp.active_count != nitems(toep->ddp.db)) {
975 if ((toep->ddp.flags & DDP_RCVBUF) != 0)
986 if ((toep->ddp.flags & DDP_RCVBUF) != 0) {
995 MPASS(toep->ddp.active_count == 0);
996 MPASS((toep->ddp.flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE)) == 0);
997 if (toep->ddp.waiting_count == 0) {
1007 toep->tid, toep->ddp.waiting_count);
1016 struct adapter *sc = iq->adapter;
1026 if (cpl->status != CPL_ERR_NONE)
1027 panic("XXX: tcp_rpl failed: %d", cpl->status);
1030 inp = toep->inp;
1031 switch (cpl->cookie) {
1037 KASSERT((toep->ddp.flags & DDP_AIO) != 0,
1039 db_idx = G_COOKIE(cpl->cookie) - CPL_COOKIE_DDP0;
1040 MPASS(db_idx < nitems(toep->ddp.db));
1043 db = &toep->ddp.db[db_idx];
1050 MPASS(db->job != NULL);
1051 MPASS(db->cancel_pending);
1063 job = db->job;
1064 copied = job->aio_received;
1072 t4_rcvd(&toep->td->tod, intotcpcb(inp));
1076 if (toep->ddp.waiting_count > 0)
1083 G_WORD(cpl->cookie), G_COOKIE(cpl->cookie));
1092 struct socket *so = toep->inp->inp_socket;
1093 struct sockbuf *sb = &so->so_rcv;
1104 INP_WLOCK_ASSERT(toep->inp);
1107 ddp_rcvbuf = (toep->ddp.flags & DDP_RCVBUF) != 0;
1109 /* - 1 is to ignore the byte for FIN */
1110 len = be32toh(rcv_nxt) - tp->rcv_nxt - 1;
1111 tp->rcv_nxt += len;
1114 toep->tid, len);
1115 while (toep->ddp.active_count > 0) {
1116 MPASS(toep->ddp.active_id != -1);
1117 db_idx = toep->ddp.active_id;
1121 MPASS((toep->ddp.flags & db_flag) != 0);
1122 db = &toep->ddp.db[db_idx];
1125 if (placed > db->drb->len - db->placed)
1126 placed = db->drb->len - db->placed;
1134 len -= placed;
1137 job = db->job;
1138 copied = job->aio_received;
1140 if (placed > job->uaiocb.aio_nbytes - copied)
1141 placed = job->uaiocb.aio_nbytes - copied;
1143 job->msgrcv = 1;
1144 toep->ofld_rxq->rx_aio_ddp_jobs++;
1146 toep->ofld_rxq->rx_aio_ddp_octets += placed;
1153 job->aio_received += placed;
1156 __func__, toep->tid, db_idx, placed);
1159 len -= placed;
1164 if ((toep->ddp.flags & DDP_AIO) != 0)
1178 struct adapter *sc = iq->adapter;
1185 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
1186 KASSERT(!(toep->flags & TPF_SYNQE),
1189 vld = be32toh(cpl->ddpvld);
1200 handle_ddp_data(toep, cpl->u.ddp_report, cpl->seq, be16toh(cpl->len));
1209 struct adapter *sc = iq->adapter;
1215 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
1216 KASSERT(!(toep->flags & TPF_SYNQE),
1219 handle_ddp_data(toep, cpl->ddp_report, cpl->rcv_nxt, 0);
1227 struct adapter *sc = toep->vi->adapter;
1233 if (!sc->tt.ddp)
1253 wr = alloc_wrqe(len, toep->ctrlq);
1257 CTR(KTR_CXGBE, "%s: tid %u", __func__, toep->tid);
1267 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, 26,
1271 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, 28,
1275 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, 30,
1279 toep->params.ulp_mode = ULP_MODE_TCPDDP;
1280 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_ULP_TYPE,
1285 ulpmc = mk_set_tcb_field_ulp(sc, ulpmc, toep->tid, W_TCB_T_FLAGS,
1300 KASSERT((toep->ddp.flags & (DDP_ON | DDP_OK | DDP_SC_REQ)) == DDP_OK,
1302 __func__, toep, toep->ddp.flags));
1305 __func__, toep->tid, time_uptime);
1308 if ((toep->ddp.flags & DDP_AIO) != 0)
1312 toep->ddp.flags |= DDP_SC_REQ;
1313 t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_RX_DDP_FLAGS,
1317 t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_T_FLAGS,
1349 return (howmany(npages >> (ddp_page_shift - PAGE_SHIFT), PPOD_PAGES));
1358 if (vmem_alloc(pr->pr_arena, PPOD_SZ(nppods), M_NOWAIT | M_FIRSTFIT,
1363 CTR5(KTR_CXGBE, "%-17s arena %p, addr 0x%08x, nppods %d, pgsz %d",
1364 __func__, pr->pr_arena, (uint32_t)addr & pr->pr_tag_mask,
1365 nppods, 1 << pr->pr_page_shift[pgsz_idx]);
1373 MPASS((addr & pr->pr_tag_mask) == addr);
1374 MPASS((addr & pr->pr_invalid_bit) == 0);
1376 prsv->prsv_pr = pr;
1377 prsv->prsv_tag = V_PPOD_PGSZ(pgsz_idx) | addr;
1378 prsv->prsv_nppods = nppods;
1399 while (i < npages - 1 &&
1407 if (hcf < (1 << pr->pr_page_shift[1])) {
1413 #define PR_PAGE_MASK(x) ((1 << pr->pr_page_shift[(x)]) - 1)
1415 for (idx = nitems(pr->pr_page_shift) - 1; idx > 0; idx--) {
1424 nppods = pages_to_nppods(npages, pr->pr_page_shift[idx]);
1427 MPASS(prsv->prsv_nppods > 0);
1433 t4_alloc_page_pods_for_ps(struct ppod_region *pr, struct pageset *ps)
1435 struct ppod_reservation *prsv = &ps->prsv;
1437 KASSERT(prsv->prsv_nppods == 0,
1440 return (t4_alloc_page_pods_for_vmpages(pr, ps->pages, ps->npages,
1449 MPASS(bp->bio_flags & BIO_UNMAPPED);
1451 return (t4_alloc_page_pods_for_vmpages(pr, bp->bio_ma, bp->bio_ma_n,
1474 end_pva = trunc_page(buf + len - 1);
1486 if (hcf < (1 << pr->pr_page_shift[1])) {
1492 #define PR_PAGE_MASK(x) ((1 << pr->pr_page_shift[(x)]) - 1)
1494 for (idx = nitems(pr->pr_page_shift) - 1; idx > 0; idx--) {
1504 npages += (end_pva - start_pva) >> pr->pr_page_shift[idx];
1508 MPASS(prsv->prsv_nppods > 0);
1517 struct ppod_reservation *prsv = &drb->prsv;
1519 KASSERT(prsv->prsv_nppods == 0,
1522 return (t4_alloc_page_pods_for_buf(pr, (vm_offset_t)drb->buf, drb->len,
1546 for (i = entries - 1; i >= 0; i--) {
1548 buf = (vm_offset_t)sge->addr;
1549 len = sge->len;
1551 end_pva = trunc_page(buf + len - 1);
1564 if (hcf < (1 << pr->pr_page_shift[1])) {
1570 #define PR_PAGE_MASK(x) ((1 << pr->pr_page_shift[(x)]) - 1)
1572 for (idx = nitems(pr->pr_page_shift) - 1; idx > 0; idx--) {
1582 while (entries--) {
1584 start_pva = trunc_page((vm_offset_t)sgl->addr);
1585 end_pva = trunc_page((vm_offset_t)sgl->addr + sgl->len - 1);
1586 npages += (end_pva - start_pva) >> pr->pr_page_shift[idx];
1592 MPASS(prsv->prsv_nppods > 0);
1599 struct ppod_region *pr = prsv->prsv_pr;
1603 MPASS(prsv->prsv_nppods != 0);
1605 addr = prsv->prsv_tag & pr->pr_tag_mask;
1606 MPASS((addr & pr->pr_invalid_bit) == 0);
1609 CTR4(KTR_CXGBE, "%-17s arena %p, addr 0x%08x, nppods %d", __func__,
1610 pr->pr_arena, addr, prsv->prsv_nppods);
1613 vmem_free(pr->pr_arena, addr, PPOD_SZ(prsv->prsv_nppods));
1614 prsv->prsv_nppods = 0;
1621 struct pageset *ps)
1630 struct ppod_reservation *prsv = &ps->prsv;
1631 struct ppod_region *pr = prsv->prsv_pr;
1634 KASSERT(!(ps->flags & PS_PPODS_WRITTEN),
1636 MPASS(prsv->prsv_nppods > 0);
1643 ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)];
1644 ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask);
1645 for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) {
1647 n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS);
1657 ulpmc->cmd = cmd;
1659 ulpmc->dlen = htobe32(V_T7_ULP_MEMIO_DATA_LEN(chunk >> 5));
1661 ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk >> 5));
1662 ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16));
1663 ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5));
1666 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1667 ulpsc->len = htobe32(chunk);
1671 ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID |
1672 V_PPOD_TID(tid) | prsv->prsv_tag);
1673 ppod->len_offset = htobe64(V_PPOD_LEN(ps->len) |
1674 V_PPOD_OFST(ps->offset));
1675 ppod->rsvd = 0;
1677 for (k = 0; k < nitems(ppod->addr); k++) {
1678 if (idx < ps->npages) {
1679 pa = VM_PAGE_TO_PHYS(ps->pages[idx]);
1680 ppod->addr[k] = htobe64(pa);
1683 ppod->addr[k] = 0;
1686 "%s: tid %d ppod[%d]->addr[%d] = %p",
1688 be64toh(ppod->addr[k]));
1696 ps->flags |= PS_PPODS_WRITTEN;
1712 struct ppod_reservation *prsv = &drb->prsv;
1713 struct ppod_region *pr = prsv->prsv_pr;
1717 MPASS(prsv->prsv_nppods > 0);
1724 ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)];
1725 offset = (uintptr_t)drb->buf & PAGE_MASK;
1726 ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask);
1727 pva = trunc_page((uintptr_t)drb->buf);
1728 end_pva = trunc_page((uintptr_t)drb->buf + drb->len - 1);
1729 for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) {
1731 n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS);
1742 ulpmc->cmd = cmd;
1743 ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk / 32));
1744 ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16));
1745 ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5));
1748 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1749 ulpsc->len = htobe32(chunk);
1753 ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID |
1754 V_PPOD_TID(tid) | prsv->prsv_tag);
1755 ppod->len_offset = htobe64(V_PPOD_LEN(drb->len) |
1757 ppod->rsvd = 0;
1759 for (k = 0; k < nitems(ppod->addr); k++) {
1761 ppod->addr[k] = 0;
1764 ppod->addr[k] = htobe64(pa);
1769 "%s: tid %d ppod[%d]->addr[%d] = %p",
1771 be64toh(ppod->addr[k]));
1780 pva -= ddp_pgsz;
1804 m->m_pkthdr.len = len;
1805 m->m_len = len;
1820 struct ppod_region *pr = prsv->prsv_pr;
1824 MPASS(bp->bio_flags & BIO_UNMAPPED);
1831 ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)];
1832 ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask);
1833 for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) {
1836 n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS);
1846 INIT_ULPTX_WR(ulpmc, len, 0, toep->tid);
1847 ulpmc->cmd = cmd;
1849 ulpmc->dlen = htobe32(V_T7_ULP_MEMIO_DATA_LEN(chunk >> 5));
1851 ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk >> 5));
1852 ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16));
1853 ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5));
1856 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1857 ulpsc->len = htobe32(chunk);
1861 ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID |
1862 V_PPOD_TID(toep->tid) |
1863 (prsv->prsv_tag & ~V_PPOD_PGSZ(M_PPOD_PGSZ)));
1864 ppod->len_offset = htobe64(V_PPOD_LEN(bp->bio_bcount) |
1865 V_PPOD_OFST(bp->bio_ma_offset));
1866 ppod->rsvd = 0;
1868 for (k = 0; k < nitems(ppod->addr); k++) {
1869 if (idx < bp->bio_ma_n) {
1870 pa = VM_PAGE_TO_PHYS(bp->bio_ma[idx]);
1871 ppod->addr[k] = htobe64(pa);
1874 ppod->addr[k] = 0;
1877 "%s: tid %d ppod[%d]->addr[%d] = %p",
1878 __func__, toep->tid, i, k,
1879 be64toh(ppod->addr[k]));
1901 struct ppod_region *pr = prsv->prsv_pr;
1911 ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)];
1913 ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask);
1915 end_pva = trunc_page(buf + buflen - 1);
1916 for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) {
1919 n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS);
1929 INIT_ULPTX_WR(ulpmc, len, 0, toep->tid);
1930 ulpmc->cmd = cmd;
1932 ulpmc->dlen = htobe32(V_T7_ULP_MEMIO_DATA_LEN(chunk >> 5));
1934 ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk >> 5));
1935 ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16));
1936 ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5));
1939 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
1940 ulpsc->len = htobe32(chunk);
1944 ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID |
1945 V_PPOD_TID(toep->tid) |
1946 (prsv->prsv_tag & ~V_PPOD_PGSZ(M_PPOD_PGSZ)));
1947 ppod->len_offset = htobe64(V_PPOD_LEN(buflen) |
1949 ppod->rsvd = 0;
1951 for (k = 0; k < nitems(ppod->addr); k++) {
1953 ppod->addr[k] = 0;
1956 ppod->addr[k] = htobe64(pa);
1961 "%s: tid %d ppod[%d]->addr[%d] = %p",
1962 __func__, toep->tid, i, k,
1963 be64toh(ppod->addr[k]));
1972 pva -= ddp_pgsz;
1994 struct ppod_region *pr = prsv->prsv_pr;
2006 ddp_pgsz = 1 << pr->pr_page_shift[G_PPOD_PGSZ(prsv->prsv_tag)];
2007 offset = (vm_offset_t)sgl->addr & PAGE_MASK;
2008 ppod_addr = pr->pr_start + (prsv->prsv_tag & pr->pr_tag_mask);
2009 pva = trunc_page((vm_offset_t)sgl->addr);
2010 for (i = 0; i < prsv->prsv_nppods; ppod_addr += chunk) {
2013 n = min(prsv->prsv_nppods - i, NUM_ULP_TX_SC_IMM_PPODS);
2023 INIT_ULPTX_WR(ulpmc, len, 0, toep->tid);
2024 ulpmc->cmd = cmd;
2026 ulpmc->dlen = htobe32(V_T7_ULP_MEMIO_DATA_LEN(chunk >> 5));
2028 ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(chunk >> 5));
2029 ulpmc->len16 = htobe32(howmany(len - sizeof(ulpmc->wr), 16));
2030 ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(ppod_addr >> 5));
2033 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
2034 ulpsc->len = htobe32(chunk);
2038 ppod->vld_tid_pgsz_tag_color = htobe64(F_PPOD_VALID |
2039 V_PPOD_TID(toep->tid) |
2040 (prsv->prsv_tag & ~V_PPOD_PGSZ(M_PPOD_PGSZ)));
2041 ppod->len_offset = htobe64(V_PPOD_LEN(xferlen) |
2043 ppod->rsvd = 0;
2045 for (k = 0; k < nitems(ppod->addr); k++) {
2048 ppod->addr[k] = htobe64(pa);
2050 ppod->addr[k] = 0;
2054 "%s: tid %d ppod[%d]->addr[%d] = %p",
2055 __func__, toep->tid, i, k,
2056 be64toh(ppod->addr[k]));
2064 if (k + 1 == nitems(ppod->addr))
2075 if (sg_offset == sgl->len) {
2080 entries--;
2085 (vm_offset_t)sgl->addr);
2100 prep_pageset(struct adapter *sc, struct toepcb *toep, struct pageset *ps)
2102 struct tom_data *td = sc->tom_softc;
2104 if (ps->prsv.prsv_nppods == 0 &&
2105 t4_alloc_page_pods_for_ps(&td->pr, ps) != 0) {
2108 if (!(ps->flags & PS_PPODS_WRITTEN) &&
2109 t4_write_page_pods_for_ps(sc, toep->ctrlq, toep->tid, ps) != 0) {
2123 MPASS(r->size > 0);
2125 pr->pr_start = r->start;
2126 pr->pr_len = r->size;
2127 pr->pr_page_shift[0] = 12 + G_HPZ0(psz);
2128 pr->pr_page_shift[1] = 12 + G_HPZ1(psz);
2129 pr->pr_page_shift[2] = 12 + G_HPZ2(psz);
2130 pr->pr_page_shift[3] = 12 + G_HPZ3(psz);
2132 /* The SGL -> page pod algorithm requires the sizes to be in order. */
2133 for (i = 1; i < nitems(pr->pr_page_shift); i++) {
2134 if (pr->pr_page_shift[i] <= pr->pr_page_shift[i - 1])
2138 pr->pr_tag_mask = ((1 << fls(r->size)) - 1) & V_PPOD_TAG(M_PPOD_TAG);
2139 pr->pr_alias_mask = V_PPOD_TAG(M_PPOD_TAG) & ~pr->pr_tag_mask;
2140 if (pr->pr_tag_mask == 0 || pr->pr_alias_mask == 0)
2142 pr->pr_alias_shift = fls(pr->pr_tag_mask);
2143 pr->pr_invalid_bit = 1 << (pr->pr_alias_shift - 1);
2145 pr->pr_arena = vmem_create(name, 0, pr->pr_len, PPOD_SIZE, 0,
2147 if (pr->pr_arena == NULL)
2159 if (pr->pr_arena)
2160 vmem_destroy(pr->pr_arena);
2165 pscmp(struct pageset *ps, struct vmspace *vm, vm_offset_t start, int npages,
2169 if (ps->start != start || ps->npages != npages ||
2170 ps->offset != pgoff || ps->len != len)
2173 return (ps->vm != vm || ps->vm_timestamp != vm->vm_map.timestamp);
2182 struct pageset *ps;
2192 vm = job->userproc->p_vmspace;
2193 map = &vm->vm_map;
2194 start = (uintptr_t)job->uaiocb.aio_buf;
2196 end = round_page(start + job->uaiocb.aio_nbytes);
2199 if (end - start > MAX_DDP_BUFFER_SIZE) {
2211 __func__, toep->tid, (unsigned long)job->uaiocb.aio_nbytes,
2212 (unsigned long)(end - (start + pgoff)));
2213 job->uaiocb.aio_nbytes = end - (start + pgoff);
2218 n = atop(end - start);
2223 TAILQ_FOREACH(ps, &toep->ddp.cached_pagesets, link) {
2224 if (pscmp(ps, vm, start, n, pgoff,
2225 job->uaiocb.aio_nbytes) == 0) {
2226 TAILQ_REMOVE(&toep->ddp.cached_pagesets, ps, link);
2227 toep->ddp.cached_count--;
2228 *pps = ps;
2237 KASSERT(toep->ddp.active_count + toep->ddp.cached_count <=
2238 nitems(toep->ddp.db), ("%s: too many wired pagesets", __func__));
2239 if (toep->ddp.active_count + toep->ddp.cached_count ==
2240 nitems(toep->ddp.db)) {
2241 KASSERT(toep->ddp.cached_count > 0,
2243 ps = TAILQ_LAST(&toep->ddp.cached_pagesets, pagesetq);
2244 TAILQ_REMOVE(&toep->ddp.cached_pagesets, ps, link);
2245 toep->ddp.cached_count--;
2246 free_pageset(toep->td, ps);
2251 ps = malloc(sizeof(*ps) + n * sizeof(vm_page_t), M_CXGBE, M_WAITOK |
2253 ps->pages = (vm_page_t *)(ps + 1);
2254 ps->vm_timestamp = map->timestamp;
2255 ps->npages = vm_fault_quick_hold_pages(map, start, end - start,
2256 VM_PROT_WRITE, ps->pages, n);
2259 if (ps->npages < 0) {
2260 free(ps, M_CXGBE);
2264 KASSERT(ps->npages == n, ("hold_aio: page count mismatch: %d vs %d",
2265 ps->npages, n));
2267 ps->offset = pgoff;
2268 ps->len = job->uaiocb.aio_nbytes;
2269 refcount_acquire(&vm->vm_refcnt);
2270 ps->vm = vm;
2271 ps->start = start;
2274 __func__, toep->tid, ps, job, ps->npages);
2275 *pps = ps;
2285 KASSERT((toep->ddp.flags & DDP_AIO) != 0, ("%s: DDP_RCVBUF", __func__));
2286 while (!TAILQ_EMPTY(&toep->ddp.aiojobq)) {
2287 job = TAILQ_FIRST(&toep->ddp.aiojobq);
2288 TAILQ_REMOVE(&toep->ddp.aiojobq, job, list);
2289 toep->ddp.waiting_count--;
2305 copied = job->aio_received;
2313 * Called when the main loop wants to requeue a job to retry it later.
2322 if (!(toep->ddp.flags & DDP_DEAD) &&
2324 TAILQ_INSERT_HEAD(&toep->ddp.aiojobq, job, list);
2325 toep->ddp.waiting_count++;
2333 struct adapter *sc = td_adapter(toep->td);
2340 struct pageset *ps;
2349 if (toep->ddp.flags & DDP_DEAD) {
2350 MPASS(toep->ddp.waiting_count == 0);
2351 MPASS(toep->ddp.active_count == 0);
2355 if (toep->ddp.waiting_count == 0 ||
2356 toep->ddp.active_count == nitems(toep->ddp.db)) {
2360 job = TAILQ_FIRST(&toep->ddp.aiojobq);
2361 so = job->fd_file->f_data;
2362 sb = &so->so_rcv;
2366 if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) {
2372 KASSERT(toep->ddp.active_count == 0 || sbavail(sb) == 0,
2377 if (so->so_error && sbavail(sb) == 0) {
2378 toep->ddp.waiting_count--;
2379 TAILQ_REMOVE(&toep->ddp.aiojobq, job, list);
2390 copied = job->aio_received;
2396 error = so->so_error;
2397 so->so_error = 0;
2399 aio_complete(job, -1, error);
2408 if (sb->sb_state & SBS_CANTRCVMORE && sbavail(sb) == 0) {
2410 if (toep->ddp.active_count != 0)
2420 if (sbavail(sb) == 0 && (toep->ddp.flags & DDP_ON) == 0) {
2434 if ((toep->ddp.flags & DDP_SC_REQ) == 0)
2444 if (toep->ddp.queueing != NULL)
2448 toep->ddp.waiting_count--;
2449 TAILQ_REMOVE(&toep->ddp.aiojobq, job, list);
2452 toep->ddp.queueing = job;
2455 error = hold_aio(toep, job, &ps);
2458 toep->ddp.queueing = NULL;
2463 if (so->so_error && sbavail(sb) == 0) {
2464 copied = job->aio_received;
2467 recycle_pageset(toep, ps);
2469 toep->ddp.queueing = NULL;
2473 error = so->so_error;
2474 so->so_error = 0;
2476 recycle_pageset(toep, ps);
2477 aio_complete(job, -1, error);
2478 toep->ddp.queueing = NULL;
2482 if (sb->sb_state & SBS_CANTRCVMORE && sbavail(sb) == 0) {
2484 recycle_pageset(toep, ps);
2485 if (toep->ddp.active_count != 0) {
2492 toep->ddp.queueing = NULL;
2497 toep->ddp.queueing = NULL;
2506 MPASS(!(toep->ddp.flags & DDP_DEAD));
2514 offset = ps->offset + job->aio_received;
2515 MPASS(job->aio_received <= job->uaiocb.aio_nbytes);
2516 resid = job->uaiocb.aio_nbytes - job->aio_received;
2517 m = sb->sb_mb;
2518 KASSERT(m == NULL || toep->ddp.active_count == 0,
2528 iov[0].iov_len = m->m_len;
2538 error = uiomove_fromphys(ps->pages, offset + copied,
2541 uiomove_fromphys(ps->pages, offset + copied, uio.uio_resid, &uio);
2545 resid -= uio.uio_offset;
2546 m = m->m_next;
2550 job->aio_received += copied;
2551 job->msgrcv = 1;
2552 copied = job->aio_received;
2570 * the next trip around the loop.
2573 t4_rcvd_locked(&toep->td->tod, intotcpcb(inp));
2575 if (resid == 0 || toep->ddp.flags & DDP_DEAD) {
2583 recycle_pageset(toep, ps);
2585 toep->ddp.queueing = NULL;
2594 if ((toep->ddp.flags & (DDP_ON | DDP_SC_REQ)) != DDP_ON) {
2596 recycle_pageset(toep, ps);
2598 toep->ddp.queueing = NULL;
2612 if (prep_pageset(sc, toep, ps) == 0) {
2613 recycle_pageset(toep, ps);
2615 toep->ddp.queueing = NULL;
2626 if (toep->ddp.db[0].job == NULL) {
2629 MPASS(toep->ddp.db[1].job == NULL);
2637 if (so->so_state & SS_NBIO)
2645 if (so->so_state & SS_NBIO)
2652 MPASS((toep->ddp.flags & buf_flag) == 0);
2653 if ((toep->ddp.flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE)) == 0) {
2655 MPASS(toep->ddp.active_id == -1);
2656 MPASS(toep->ddp.active_count == 0);
2668 wr = mk_update_tcb_for_ddp(sc, toep, db_idx, &ps->prsv,
2669 job->aio_received, ps->len, ddp_flags, ddp_flags_mask);
2671 recycle_pageset(toep, ps);
2673 toep->ddp.queueing = NULL;
2689 recycle_pageset(toep, ps);
2691 toep->ddp.queueing = NULL;
2698 toep->tid, job, db_idx, ddp_flags, ddp_flags_mask);
2700 /* Give the chip the go-ahead. */
2702 db = &toep->ddp.db[db_idx];
2703 db->cancel_pending = 0;
2704 db->job = job;
2705 db->ps = ps;
2706 toep->ddp.queueing = NULL;
2707 toep->ddp.flags |= buf_flag;
2708 toep->ddp.active_count++;
2709 if (toep->ddp.active_count == 1) {
2710 MPASS(toep->ddp.active_id == -1);
2711 toep->ddp.active_id = db_idx;
2713 toep->ddp.active_id);
2723 if (toep->ddp.flags & DDP_TASK_ACTIVE)
2725 toep->ddp.flags |= DDP_TASK_ACTIVE;
2727 soaio_enqueue(&toep->ddp.requeue_task);
2737 toep->ddp.flags &= ~DDP_TASK_ACTIVE;
2746 struct socket *so = job->fd_file->f_data;
2748 struct toepcb *toep = tp->t_toe;
2749 struct adapter *sc = td_adapter(toep->td);
2760 for (i = 0; i < nitems(toep->ddp.db); i++) {
2761 if (toep->ddp.db[i].job == job) {
2763 MPASS(toep->ddp.db[i].cancel_pending == 0);
2772 t4_set_tcb_field(sc, toep->ctrlq, toep,
2775 toep->ddp.db[i].cancel_pending = 1;
2787 struct socket *so = job->fd_file->f_data;
2789 struct toepcb *toep = tp->t_toe;
2793 TAILQ_REMOVE(&toep->ddp.aiojobq, job, list);
2794 toep->ddp.waiting_count--;
2795 if (toep->ddp.waiting_count == 0)
2809 struct toepcb *toep = tp->t_toe;
2812 if (job->uaiocb.aio_lio_opcode != LIO_READ)
2830 if ((toep->ddp.flags & DDP_RCVBUF) != 0) {
2835 if ((toep->ddp.flags & DDP_AIO) == 0) {
2836 toep->ddp.flags |= DDP_AIO;
2837 TAILQ_INIT(&toep->ddp.cached_pagesets);
2838 TAILQ_INIT(&toep->ddp.aiojobq);
2839 TASK_INIT(&toep->ddp.requeue_task, 0, aio_ddp_requeue_task,
2850 CTR3(KTR_CXGBE, "%s: queueing %p for tid %u", __func__, job, toep->tid);
2854 TAILQ_INSERT_TAIL(&toep->ddp.aiojobq, job, list);
2855 toep->ddp.waiting_count++;
2877 if ((toep->ddp.flags & DDP_DEAD) != 0) {
2878 MPASS(toep->ddp.active_count == 0);
2883 if (toep->ddp.active_count == nitems(toep->ddp.db)) {
2887 inp = toep->inp;
2888 so = inp->inp_socket;
2889 sb = &so->so_rcv;
2904 if ((toep->ddp.flags & DDP_DEAD) != 0 ||
2905 toep->ddp.active_count == nitems(toep->ddp.db)) {
2912 if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) {
2919 if (so->so_error != 0 || (sb->sb_state & SBS_CANTRCVMORE) != 0) {
2947 toep->ddp.flags &= ~DDP_TASK_ACTIVE;
2957 struct adapter *sc = td_adapter(toep->td);
2979 if ((toep->ddp.flags & DDP_AIO) != 0) {
2984 if ((toep->ddp.flags & DDP_RCVBUF) != 0) {
2989 toep->ddp.flags |= DDP_RCVBUF;
2990 TAILQ_INIT(&toep->ddp.cached_buffers);
2992 TASK_INIT(&toep->ddp.requeue_task, 0, ddp_rcvbuf_requeue_task, toep);