Lines Matching full:iq
395 set_tcb_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, in set_tcb_rpl_handler() argument
405 if (is_hpftid(iq->adapter, tid) || is_ftid(iq->adapter, tid)) { in set_tcb_rpl_handler()
418 return (set_tcb_rpl_handlers[cookie](iq, rss, m)); in set_tcb_rpl_handler()
422 l2t_write_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, in l2t_write_rpl_handler() argument
431 return (l2t_write_rpl_handlers[cookie](iq, rss, m)); in l2t_write_rpl_handler()
435 act_open_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, in act_open_rpl_handler() argument
444 return (act_open_rpl_handlers[cookie](iq, rss, m)); in act_open_rpl_handler()
448 abort_rpl_rss_handler(struct sge_iq *iq, const struct rss_header *rss, in abort_rpl_rss_handler() argument
451 struct adapter *sc = iq->adapter; in abort_rpl_rss_handler()
460 return (abort_rpl_rss_handlers[cookie](iq, rss, m)); in abort_rpl_rss_handler()
464 fw4_ack_handler(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) in fw4_ack_handler() argument
466 struct adapter *sc = iq->adapter; in fw4_ack_handler()
477 return (fw4_ack_handlers[cookie](iq, rss, m)); in fw4_ack_handler()
1090 /* Maximum payload that could arrive with a single iq descriptor. */
1320 * Interrupt handler for iq-only queues. The firmware event queue is the only
1326 struct sge_iq *iq = arg; in t4_intr_evt() local
1328 if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) { in t4_intr_evt()
1329 service_iq(iq, 0); in t4_intr_evt()
1330 (void) atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE); in t4_intr_evt()
1335 * Interrupt handler for iq+fl queues.
1340 struct sge_iq *iq = arg; in t4_intr() local
1342 if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) { in t4_intr()
1343 service_iq_fl(iq, 0); in t4_intr()
1344 (void) atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE); in t4_intr()
1380 * Deals with interrupts on an iq-only (no freelist) queue.
1383 service_iq(struct sge_iq *iq, int budget) in service_iq() argument
1386 struct adapter *sc = iq->adapter; in service_iq()
1387 struct iq_desc *d = &iq->desc[iq->cidx]; in service_iq()
1393 KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq)); in service_iq()
1394 KASSERT((iq->flags & IQ_HAS_FL) == 0, in service_iq()
1395 ("%s: called for iq %p with fl (iq->flags 0x%x)", __func__, iq, in service_iq()
1396 iq->flags)); in service_iq()
1397 MPASS((iq->flags & IQ_ADJ_CREDIT) == 0); in service_iq()
1398 MPASS((iq->flags & IQ_LRO_ENABLED) == 0); in service_iq()
1400 limit = budget ? budget : iq->qsize / 16; in service_iq()
1407 while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) { in service_iq()
1416 panic("%s: data for an iq (%p) with no freelist", in service_iq()
1417 __func__, iq); in service_iq()
1425 t4_cpl_handler[d->rss.opcode](iq, &d->rss, NULL); in service_iq()
1436 t4_an_handler(iq, &d->rsp); in service_iq()
1456 ("%s: illegal response type %d on iq %p", in service_iq()
1457 __func__, rsp_type, iq)); in service_iq()
1459 "%s: illegal response type %d on iq %p", in service_iq()
1460 device_get_nameunit(sc->dev), rsp_type, iq); in service_iq()
1465 if (__predict_false(++iq->cidx == iq->sidx)) { in service_iq()
1466 iq->cidx = 0; in service_iq()
1467 iq->gen ^= F_RSPD_GEN; in service_iq()
1468 d = &iq->desc[0]; in service_iq()
1473 V_INGRESSQID(iq->cntxt_id) | in service_iq()
1499 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params)); in service_iq()
1567 * Deals with interrupts on an iq+fl queue.
1570 service_iq_fl(struct sge_iq *iq, int budget) in service_iq_fl() argument
1572 struct sge_rxq *rxq = iq_to_rxq(iq); in service_iq_fl()
1574 struct adapter *sc = iq->adapter; in service_iq_fl()
1575 struct iq_desc *d = &iq->desc[iq->cidx]; in service_iq_fl()
1586 KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq)); in service_iq_fl()
1587 MPASS(iq->flags & IQ_HAS_FL); in service_iq_fl()
1591 if (iq->flags & IQ_ADJ_CREDIT) { in service_iq_fl()
1593 iq->flags &= ~IQ_ADJ_CREDIT; in service_iq_fl()
1594 if ((d->rsp.u.type_gen & F_RSPD_GEN) != iq->gen) { in service_iq_fl()
1597 V_INGRESSQID((u32)iq->cntxt_id) | in service_iq_fl()
1598 V_SEINTARM(iq->intr_params)); in service_iq_fl()
1604 MPASS((iq->flags & IQ_ADJ_CREDIT) == 0); in service_iq_fl()
1607 limit = budget ? budget : iq->qsize / 16; in service_iq_fl()
1610 while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) { in service_iq_fl()
1646 t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0); in service_iq_fl()
1660 "with qid %u", __func__, iq, lq); in service_iq_fl()
1663 t4_an_handler(iq, &d->rsp); in service_iq_fl()
1667 KASSERT(0, ("%s: illegal response type %d on iq %p", in service_iq_fl()
1668 __func__, rsp_type, iq)); in service_iq_fl()
1669 log(LOG_ERR, "%s: illegal response type %d on iq %p", in service_iq_fl()
1670 device_get_nameunit(sc->dev), rsp_type, iq); in service_iq_fl()
1675 if (__predict_false(++iq->cidx == iq->sidx)) { in service_iq_fl()
1676 iq->cidx = 0; in service_iq_fl()
1677 iq->gen ^= F_RSPD_GEN; in service_iq_fl()
1678 d = &iq->desc[0]; in service_iq_fl()
1682 V_INGRESSQID(iq->cntxt_id) | in service_iq_fl()
1686 if (iq->flags & IQ_LRO_ENABLED && in service_iq_fl()
1699 if (iq->flags & IQ_LRO_ENABLED) { in service_iq_fl()
1703 iq->flags |= IQ_ADJ_CREDIT; in service_iq_fl()
1712 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params)); in service_iq_fl()
2102 if (rxq->iq.flags & IQ_RX_TIMESTAMP) { in eth_rx()
2117 if (rxq->iq.flags & IQ_LRO_ENABLED && tnl_type == 0 && in eth_rx()
3344 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx, in init_iq() argument
3357 iq->flags = 0; in init_iq()
3358 iq->state = IQS_DISABLED; in init_iq()
3359 iq->adapter = sc; in init_iq()
3360 iq->qtype = qtype; in init_iq()
3361 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx); in init_iq()
3362 iq->intr_pktc_idx = SGE_NCOUNTERS - 1; in init_iq()
3364 iq->intr_params |= F_QINTR_CNT_EN; in init_iq()
3365 iq->intr_pktc_idx = pktc_idx; in init_iq()
3367 iq->qsize = roundup2(qsize, 16); /* See FW_IQ_CMD/iqsize */ in init_iq()
3368 iq->sidx = iq->qsize - sc->params.sge.spg_len / IQ_ESIZE; in init_iq()
3369 iq->intr_idx = intr_idx; in init_iq()
3370 iq->cong_drop = cong; in init_iq()
3401 uint8_t port_id, struct sge_iq *iq, char *name) in init_eq() argument
3409 eq->iq = iq; in init_eq()
3468 alloc_iq_fl(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl, in alloc_iq_fl() argument
3475 MPASS(!(iq->flags & IQ_SW_ALLOCATED)); in alloc_iq_fl()
3477 len = iq->qsize * IQ_ESIZE; in alloc_iq_fl()
3478 rc = alloc_ring(sc, len, &iq->desc_tag, &iq->desc_map, &iq->ba, in alloc_iq_fl()
3479 (void **)&iq->desc); in alloc_iq_fl()
3488 free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba, in alloc_iq_fl()
3489 iq->desc); in alloc_iq_fl()
3498 iq->flags |= IQ_HAS_FL; in alloc_iq_fl()
3500 add_iq_sysctls(ctx, oid, iq); in alloc_iq_fl()
3501 iq->flags |= IQ_SW_ALLOCATED; in alloc_iq_fl()
3511 free_iq_fl(struct adapter *sc, struct sge_iq *iq, struct sge_fl *fl) in free_iq_fl() argument
3513 MPASS(iq->flags & IQ_SW_ALLOCATED); in free_iq_fl()
3516 MPASS(iq->flags & IQ_HAS_FL); in free_iq_fl()
3523 free_ring(sc, iq->desc_tag, iq->desc_map, iq->ba, iq->desc); in free_iq_fl()
3524 bzero(iq, sizeof(*iq)); in free_iq_fl()
3535 alloc_iq_fl_hwq(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl) in alloc_iq_fl_hwq() argument
3543 MPASS (!(iq->flags & IQ_HW_ALLOCATED)); in alloc_iq_fl_hwq()
3554 if (iq == &sc->sge.fwq) in alloc_iq_fl_hwq()
3557 if (iq->intr_idx < 0) { in alloc_iq_fl_hwq()
3562 KASSERT(iq->intr_idx < sc->intr_count, in alloc_iq_fl_hwq()
3563 ("%s: invalid direct intr_idx %d", __func__, iq->intr_idx)); in alloc_iq_fl_hwq()
3564 v |= V_FW_IQ_CMD_IQANDSTINDEX(iq->intr_idx); in alloc_iq_fl_hwq()
3567 bzero(iq->desc, iq->qsize * IQ_ESIZE); in alloc_iq_fl_hwq()
3574 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) | in alloc_iq_fl_hwq()
3576 c.iqsize = htobe16(iq->qsize); in alloc_iq_fl_hwq()
3577 c.iqaddr = htobe64(iq->ba); in alloc_iq_fl_hwq()
3578 c.iqns_to_fl0congen = htobe32(V_FW_IQ_CMD_IQTYPE(iq->qtype)); in alloc_iq_fl_hwq()
3579 if (iq->cong_drop != -1) { in alloc_iq_fl_hwq()
3580 cong_map = iq->qtype == IQ_ETH ? pi->rx_e_chan_map : 0; in alloc_iq_fl_hwq()
3592 if (iq->cong_drop != -1) { in alloc_iq_fl_hwq()
3613 iq->cidx = 0; in alloc_iq_fl_hwq()
3614 iq->gen = F_RSPD_GEN; in alloc_iq_fl_hwq()
3615 iq->cntxt_id = be16toh(c.iqid); in alloc_iq_fl_hwq()
3616 iq->abs_id = be16toh(c.physiqid); in alloc_iq_fl_hwq()
3618 cntxt_id = iq->cntxt_id - sc->sge.iq_start; in alloc_iq_fl_hwq()
3620 panic ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__, in alloc_iq_fl_hwq()
3623 sc->sge.iqmap[cntxt_id] = iq; in alloc_iq_fl_hwq()
3670 iq->cong_drop != -1) { in alloc_iq_fl_hwq()
3671 t4_sge_set_conm_context(sc, iq->cntxt_id, iq->cong_drop, in alloc_iq_fl_hwq()
3675 /* Enable IQ interrupts */ in alloc_iq_fl_hwq()
3676 atomic_store_rel_int(&iq->state, IQS_IDLE); in alloc_iq_fl_hwq()
3677 t4_write_reg(sc, sc->sge_gts_reg, V_SEINTARM(iq->intr_params) | in alloc_iq_fl_hwq()
3678 V_INGRESSQID(iq->cntxt_id)); in alloc_iq_fl_hwq()
3680 iq->flags |= IQ_HW_ALLOCATED; in alloc_iq_fl_hwq()
3686 free_iq_fl_hwq(struct adapter *sc, struct sge_iq *iq, struct sge_fl *fl) in free_iq_fl_hwq() argument
3690 MPASS(iq->flags & IQ_HW_ALLOCATED); in free_iq_fl_hwq()
3692 iq->cntxt_id, fl ? fl->cntxt_id : 0xffff, 0xffff); in free_iq_fl_hwq()
3694 CH_ERR(sc, "failed to free iq %p: %d\n", iq, rc); in free_iq_fl_hwq()
3697 iq->flags &= ~IQ_HW_ALLOCATED; in free_iq_fl_hwq()
3704 struct sge_iq *iq) in add_iq_sysctls() argument
3712 SYSCTL_ADD_UAUTO(ctx, children, OID_AUTO, "ba", CTLFLAG_RD, &iq->ba, in add_iq_sysctls()
3715 iq->qsize * IQ_ESIZE, "descriptor ring size in bytes"); in add_iq_sysctls()
3717 &iq->abs_id, 0, "absolute id of the queue"); in add_iq_sysctls()
3719 &iq->cntxt_id, 0, "SGE context id of the queue"); in add_iq_sysctls()
3720 SYSCTL_ADD_U16(ctx, children, OID_AUTO, "cidx", CTLFLAG_RD, &iq->cidx, in add_iq_sysctls()
3964 if (!(rxq->iq.flags & IQ_SW_ALLOCATED)) { in alloc_rxq()
3965 MPASS(!(rxq->iq.flags & IQ_HW_ALLOCATED)); in alloc_rxq()
3979 init_iq(&rxq->iq, sc, vi->tmr_idx, vi->pktc_idx, vi->qsize_rxq, in alloc_rxq()
3983 rxq->iq.flags |= IQ_LRO_ENABLED; in alloc_rxq()
3986 rxq->iq.flags |= IQ_RX_TIMESTAMP; in alloc_rxq()
3990 rc = alloc_iq_fl(vi, &rxq->iq, &rxq->fl, &vi->ctx, oid); in alloc_rxq()
4000 MPASS(rxq->iq.flags & IQ_SW_ALLOCATED); in alloc_rxq()
4004 if (!(rxq->iq.flags & IQ_HW_ALLOCATED)) { in alloc_rxq()
4005 MPASS(rxq->iq.flags & IQ_SW_ALLOCATED); in alloc_rxq()
4006 rc = alloc_iq_fl_hwq(vi, &rxq->iq, &rxq->fl); in alloc_rxq()
4011 MPASS(rxq->iq.flags & IQ_HW_ALLOCATED); in alloc_rxq()
4014 sc->sge.iq_base = rxq->iq.abs_id - rxq->iq.cntxt_id; in alloc_rxq()
4016 KASSERT(rxq->iq.cntxt_id + sc->sge.iq_base == rxq->iq.abs_id, in alloc_rxq()
4039 if (rxq->iq.flags & IQ_HW_ALLOCATED) { in free_rxq()
4040 MPASS(rxq->iq.flags & IQ_SW_ALLOCATED); in free_rxq()
4041 free_iq_fl_hwq(vi->adapter, &rxq->iq, &rxq->fl); in free_rxq()
4042 MPASS(!(rxq->iq.flags & IQ_HW_ALLOCATED)); in free_rxq()
4045 if (rxq->iq.flags & IQ_SW_ALLOCATED) { in free_rxq()
4046 MPASS(!(rxq->iq.flags & IQ_HW_ALLOCATED)); in free_rxq()
4050 free_iq_fl(vi->adapter, &rxq->iq, &rxq->fl); in free_rxq()
4051 MPASS(!(rxq->iq.flags & IQ_SW_ALLOCATED)); in free_rxq()
4094 if (!(ofld_rxq->iq.flags & IQ_SW_ALLOCATED)) { in alloc_ofld_rxq()
4095 MPASS(!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED)); in alloc_ofld_rxq()
4102 init_iq(&ofld_rxq->iq, sc, vi->ofld_tmr_idx, vi->ofld_pktc_idx, in alloc_ofld_rxq()
4107 rc = alloc_iq_fl(vi, &ofld_rxq->iq, &ofld_rxq->fl, &vi->ctx, in alloc_ofld_rxq()
4115 MPASS(ofld_rxq->iq.flags & IQ_SW_ALLOCATED); in alloc_ofld_rxq()
4125 if (!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED)) { in alloc_ofld_rxq()
4126 MPASS(ofld_rxq->iq.flags & IQ_SW_ALLOCATED); in alloc_ofld_rxq()
4127 rc = alloc_iq_fl_hwq(vi, &ofld_rxq->iq, &ofld_rxq->fl); in alloc_ofld_rxq()
4133 MPASS(ofld_rxq->iq.flags & IQ_HW_ALLOCATED); in alloc_ofld_rxq()
4144 if (ofld_rxq->iq.flags & IQ_HW_ALLOCATED) { in free_ofld_rxq()
4145 MPASS(ofld_rxq->iq.flags & IQ_SW_ALLOCATED); in free_ofld_rxq()
4146 free_iq_fl_hwq(vi->adapter, &ofld_rxq->iq, &ofld_rxq->fl); in free_ofld_rxq()
4147 MPASS(!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED)); in free_ofld_rxq()
4150 if (ofld_rxq->iq.flags & IQ_SW_ALLOCATED) { in free_ofld_rxq()
4151 MPASS(!(ofld_rxq->iq.flags & IQ_HW_ALLOCATED)); in free_ofld_rxq()
4152 free_iq_fl(vi->adapter, &ofld_rxq->iq, &ofld_rxq->fl); in free_ofld_rxq()
4153 MPASS(!(ofld_rxq->iq.flags & IQ_SW_ALLOCATED)); in free_ofld_rxq()
4450 eq->iqid = eq->iq->cntxt_id; in alloc_eq_hwq()
4616 &sc->sge.rxq[iqidx].iq, name); in alloc_txq()
4833 &sc->sge.ofld_rxq[iqidx].iq, name); in alloc_ofld_txq()
4837 &sc->sge.rxq[iqidx].iq, name); in alloc_ofld_txq()
6372 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss, in handle_sge_egr_update() argument
6377 struct adapter *sc = iq->adapter; in handle_sge_egr_update()
6398 handle_fw_msg(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) in handle_fw_msg() argument
6400 struct adapter *sc = iq->adapter; in handle_fw_msg()
6410 return (t4_cpl_handler[rss2->opcode](iq, rss2, m)); in handle_fw_msg()
6901 ethofld_fw4_ack(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0) in ethofld_fw4_ack() argument
6903 struct adapter *sc = iq->adapter; in ethofld_fw4_ack()