1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * This file is part of the Chelsio T4 support code. 14 * 15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved. 16 * 17 * This program is distributed in the hope that it will be useful, but WITHOUT 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this 20 * release for licensing terms and conditions. 21 */ 22 23 /* 24 * Copyright 2024 Oxide Computer Company 25 */ 26 27 #include <sys/ddi.h> 28 #include <sys/sunddi.h> 29 #include <sys/sunndi.h> 30 #include <sys/atomic.h> 31 #include <sys/dlpi.h> 32 #include <sys/pattr.h> 33 #include <sys/strsubr.h> 34 #include <sys/stream.h> 35 #include <sys/strsun.h> 36 #include <inet/ip.h> 37 #include <inet/tcp.h> 38 39 #include "version.h" 40 #include "common/common.h" 41 #include "common/t4_msg.h" 42 #include "common/t4_regs.h" 43 #include "common/t4_regs_values.h" 44 45 /* TODO: Tune. */ 46 int rx_buf_size = 8192; 47 int tx_copy_threshold = 256; 48 uint16_t rx_copy_threshold = 256; 49 50 /* Used to track coalesced tx work request */ 51 struct txpkts { 52 mblk_t *tail; /* head is in the software descriptor */ 53 uint64_t *flitp; /* ptr to flit where next pkt should start */ 54 uint8_t npkt; /* # of packets in this work request */ 55 uint8_t nflits; /* # of flits used by this work request */ 56 uint16_t plen; /* total payload (sum of all packets) */ 57 }; 58 59 /* All information needed to tx a frame */ 60 struct txinfo { 61 uint32_t len; /* Total length of frame */ 62 uint32_t flags; /* Checksum and LSO flags */ 63 uint32_t mss; /* MSS for LSO */ 64 uint8_t nsegs; /* # of segments in the SGL, 0 means imm. tx */ 65 uint8_t nflits; /* # of flits needed for the SGL */ 66 uint8_t hdls_used; /* # of DMA handles used */ 67 uint32_t txb_used; /* txb_space used */ 68 mac_ether_offload_info_t meoi; /* pkt hdr info for offloads */ 69 struct ulptx_sgl sgl __attribute__((aligned(8))); 70 struct ulptx_sge_pair reserved[TX_SGL_SEGS / 2]; 71 }; 72 73 static int service_iq(struct sge_iq *iq, int budget); 74 static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, 75 int8_t pktc_idx, int qsize, uint8_t esize); 76 static inline void init_fl(struct sge_fl *fl, uint16_t qsize); 77 static inline void init_eq(struct adapter *sc, struct sge_eq *eq, 78 uint16_t eqtype, uint16_t qsize,uint8_t tx_chan, uint16_t iqid); 79 static int alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, 80 struct sge_fl *fl, int intr_idx, int cong); 81 static int free_iq_fl(struct port_info *pi, struct sge_iq *iq, 82 struct sge_fl *fl); 83 static int alloc_fwq(struct adapter *sc); 84 static int free_fwq(struct adapter *sc); 85 static int alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, 86 int i); 87 static int free_rxq(struct port_info *pi, struct sge_rxq *rxq); 88 static int ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq); 89 static int eth_eq_alloc(struct adapter *sc, struct port_info *pi, 90 struct sge_eq *eq); 91 static int alloc_eq(struct adapter *sc, struct port_info *pi, 92 struct sge_eq *eq); 93 static int free_eq(struct adapter *sc, struct sge_eq *eq); 94 static int alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx); 95 static int free_txq(struct port_info *pi, struct sge_txq *txq); 96 static int alloc_dma_memory(struct adapter *sc, size_t len, int flags, 97 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr, 98 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 99 caddr_t *pva); 100 static int free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl); 101 static int alloc_desc_ring(struct adapter *sc, size_t len, int rw, 102 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 103 caddr_t *pva); 104 static int free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl); 105 static int alloc_tx_copybuffer(struct adapter *sc, size_t len, 106 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 107 caddr_t *pva); 108 static inline bool is_new_response(const struct sge_iq *iq, 109 struct rsp_ctrl **ctrl); 110 static inline void iq_next(struct sge_iq *iq); 111 static int refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs); 112 static void refill_sfl(void *arg); 113 static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl); 114 static void free_fl_bufs(struct sge_fl *fl); 115 static mblk_t *get_fl_payload(struct adapter *sc, struct sge_fl *fl, 116 uint32_t len_newbuf, int *fl_bufs_used); 117 static int get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, 118 struct txinfo *txinfo, int sgl_only); 119 static inline int fits_in_txb(struct sge_txq *txq, int len, int *waste); 120 static inline int copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, 121 struct txinfo *txinfo); 122 static inline void add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len); 123 static inline int add_mblk(struct sge_txq *txq, struct txinfo *txinfo, 124 mblk_t *m, int len); 125 static void free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo); 126 static int add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m, 127 struct txinfo *txinfo); 128 static void write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts); 129 static int write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m, 130 struct txinfo *txinfo); 131 static inline void write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq, 132 struct txpkts *txpkts, struct txinfo *txinfo); 133 static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, 134 int len); 135 static inline void ring_tx_db(struct adapter *sc, struct sge_eq *eq); 136 static int reclaim_tx_descs(struct sge_txq *txq, int howmany); 137 static void write_txqflush_wr(struct sge_txq *txq); 138 static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, 139 mblk_t *m); 140 static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl); 141 static kstat_t *setup_port_config_kstats(struct port_info *pi); 142 static kstat_t *setup_port_info_kstats(struct port_info *pi); 143 static kstat_t *setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, 144 int idx); 145 static int update_rxq_kstats(kstat_t *ksp, int rw); 146 static int update_port_info_kstats(kstat_t *ksp, int rw); 147 static kstat_t *setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, 148 int idx); 149 static int update_txq_kstats(kstat_t *ksp, int rw); 150 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *, 151 mblk_t *); 152 static int handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, 153 mblk_t *m); 154 155 static inline int 156 reclaimable(struct sge_eq *eq) 157 { 158 unsigned int cidx; 159 160 cidx = eq->spg->cidx; /* stable snapshot */ 161 cidx = be16_to_cpu(cidx); 162 163 if (cidx >= eq->cidx) 164 return (cidx - eq->cidx); 165 else 166 return (cidx + eq->cap - eq->cidx); 167 } 168 169 void 170 t4_sge_init(struct adapter *sc) 171 { 172 struct driver_properties *p = &sc->props; 173 ddi_dma_attr_t *dma_attr; 174 ddi_device_acc_attr_t *acc_attr; 175 uint32_t sge_control, sge_conm_ctrl; 176 int egress_threshold; 177 178 /* 179 * Device access and DMA attributes for descriptor rings 180 */ 181 acc_attr = &sc->sge.acc_attr_desc; 182 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 183 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 184 acc_attr->devacc_attr_dataorder = DDI_STRICTORDER_ACC; 185 186 dma_attr = &sc->sge.dma_attr_desc; 187 dma_attr->dma_attr_version = DMA_ATTR_V0; 188 dma_attr->dma_attr_addr_lo = 0; 189 dma_attr->dma_attr_addr_hi = UINT64_MAX; 190 dma_attr->dma_attr_count_max = UINT64_MAX; 191 dma_attr->dma_attr_align = 512; 192 dma_attr->dma_attr_burstsizes = 0xfff; 193 dma_attr->dma_attr_minxfer = 1; 194 dma_attr->dma_attr_maxxfer = UINT64_MAX; 195 dma_attr->dma_attr_seg = UINT64_MAX; 196 dma_attr->dma_attr_sgllen = 1; 197 dma_attr->dma_attr_granular = 1; 198 dma_attr->dma_attr_flags = 0; 199 200 /* 201 * Device access and DMA attributes for tx buffers 202 */ 203 acc_attr = &sc->sge.acc_attr_tx; 204 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 205 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 206 207 dma_attr = &sc->sge.dma_attr_tx; 208 dma_attr->dma_attr_version = DMA_ATTR_V0; 209 dma_attr->dma_attr_addr_lo = 0; 210 dma_attr->dma_attr_addr_hi = UINT64_MAX; 211 dma_attr->dma_attr_count_max = UINT64_MAX; 212 dma_attr->dma_attr_align = 1; 213 dma_attr->dma_attr_burstsizes = 0xfff; 214 dma_attr->dma_attr_minxfer = 1; 215 dma_attr->dma_attr_maxxfer = UINT64_MAX; 216 dma_attr->dma_attr_seg = UINT64_MAX; 217 dma_attr->dma_attr_sgllen = TX_SGL_SEGS; 218 dma_attr->dma_attr_granular = 1; 219 dma_attr->dma_attr_flags = 0; 220 221 /* 222 * Ingress Padding Boundary and Egress Status Page Size are set up by 223 * t4_fixup_host_params(). 224 */ 225 sge_control = t4_read_reg(sc, A_SGE_CONTROL); 226 sc->sge.pktshift = G_PKTSHIFT(sge_control); 227 sc->sge.stat_len = (sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64; 228 229 /* t4_nex uses FLM packed mode */ 230 sc->sge.fl_align = t4_fl_pkt_align(sc, true); 231 232 /* 233 * Device access and DMA attributes for rx buffers 234 */ 235 sc->sge.rxb_params.dip = sc->dip; 236 sc->sge.rxb_params.buf_size = rx_buf_size; 237 238 acc_attr = &sc->sge.rxb_params.acc_attr_rx; 239 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 240 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 241 242 dma_attr = &sc->sge.rxb_params.dma_attr_rx; 243 dma_attr->dma_attr_version = DMA_ATTR_V0; 244 dma_attr->dma_attr_addr_lo = 0; 245 dma_attr->dma_attr_addr_hi = UINT64_MAX; 246 dma_attr->dma_attr_count_max = UINT64_MAX; 247 /* 248 * Low 4 bits of an rx buffer address have a special meaning to the SGE 249 * and an rx buf cannot have an address with any of these bits set. 250 * FL_ALIGN is >= 32 so we're sure things are ok. 251 */ 252 dma_attr->dma_attr_align = sc->sge.fl_align; 253 dma_attr->dma_attr_burstsizes = 0xfff; 254 dma_attr->dma_attr_minxfer = 1; 255 dma_attr->dma_attr_maxxfer = UINT64_MAX; 256 dma_attr->dma_attr_seg = UINT64_MAX; 257 dma_attr->dma_attr_sgllen = 1; 258 dma_attr->dma_attr_granular = 1; 259 dma_attr->dma_attr_flags = 0; 260 261 sc->sge.rxbuf_cache = rxbuf_cache_create(&sc->sge.rxb_params); 262 263 /* 264 * A FL with <= fl_starve_thres buffers is starving and a periodic 265 * timer will attempt to refill it. This needs to be larger than the 266 * SGE's Egress Congestion Threshold. If it isn't, then we can get 267 * stuck waiting for new packets while the SGE is waiting for us to 268 * give it more Free List entries. (Note that the SGE's Egress 269 * Congestion Threshold is in units of 2 Free List pointers.) For T4, 270 * there was only a single field to control this. For T5 there's the 271 * original field which now only applies to Unpacked Mode Free List 272 * buffers and a new field which only applies to Packed Mode Free List 273 * buffers. 274 */ 275 276 sge_conm_ctrl = t4_read_reg(sc, A_SGE_CONM_CTRL); 277 switch (CHELSIO_CHIP_VERSION(sc->params.chip)) { 278 case CHELSIO_T4: 279 egress_threshold = G_EGRTHRESHOLD(sge_conm_ctrl); 280 break; 281 case CHELSIO_T5: 282 egress_threshold = G_EGRTHRESHOLDPACKING(sge_conm_ctrl); 283 break; 284 case CHELSIO_T6: 285 default: 286 egress_threshold = G_T6_EGRTHRESHOLDPACKING(sge_conm_ctrl); 287 } 288 sc->sge.fl_starve_threshold = 2*egress_threshold + 1; 289 290 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, rx_buf_size); 291 292 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, 293 V_THRESHOLD_0(p->counter_val[0]) | 294 V_THRESHOLD_1(p->counter_val[1]) | 295 V_THRESHOLD_2(p->counter_val[2]) | 296 V_THRESHOLD_3(p->counter_val[3])); 297 298 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, 299 V_TIMERVALUE0(us_to_core_ticks(sc, p->timer_val[0])) | 300 V_TIMERVALUE1(us_to_core_ticks(sc, p->timer_val[1]))); 301 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, 302 V_TIMERVALUE2(us_to_core_ticks(sc, p->timer_val[2])) | 303 V_TIMERVALUE3(us_to_core_ticks(sc, p->timer_val[3]))); 304 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, 305 V_TIMERVALUE4(us_to_core_ticks(sc, p->timer_val[4])) | 306 V_TIMERVALUE5(us_to_core_ticks(sc, p->timer_val[5]))); 307 308 (void) t4_register_cpl_handler(sc, CPL_FW4_MSG, handle_fw_rpl); 309 (void) t4_register_cpl_handler(sc, CPL_FW6_MSG, handle_fw_rpl); 310 (void) t4_register_cpl_handler(sc, CPL_SGE_EGR_UPDATE, handle_sge_egr_update); 311 (void) t4_register_cpl_handler(sc, CPL_RX_PKT, t4_eth_rx); 312 (void) t4_register_fw_msg_handler(sc, FW6_TYPE_CMD_RPL, 313 t4_handle_fw_rpl); 314 } 315 316 /* 317 * Allocate and initialize the firmware event queue and the forwarded interrupt 318 * queues, if any. The adapter owns all these queues as they are not associated 319 * with any particular port. 320 * 321 * Returns errno on failure. Resources allocated up to that point may still be 322 * allocated. Caller is responsible for cleanup in case this function fails. 323 */ 324 int 325 t4_setup_adapter_queues(struct adapter *sc) 326 { 327 int rc; 328 329 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 330 331 /* 332 * Firmware event queue 333 */ 334 rc = alloc_fwq(sc); 335 if (rc != 0) 336 return (rc); 337 338 return (rc); 339 } 340 341 /* 342 * Idempotent 343 */ 344 int 345 t4_teardown_adapter_queues(struct adapter *sc) 346 { 347 348 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 349 350 (void) free_fwq(sc); 351 352 return (0); 353 } 354 355 static inline int 356 first_vector(struct port_info *pi) 357 { 358 struct adapter *sc = pi->adapter; 359 int rc = T4_EXTRA_INTR, i; 360 361 if (sc->intr_count == 1) 362 return (0); 363 364 for_each_port(sc, i) { 365 struct port_info *p = sc->port[i]; 366 367 if (i == pi->port_id) 368 break; 369 370 /* 371 * Not compiled with offload support and intr_count > 1. Only 372 * NIC queues exist and they'd better be taking direct 373 * interrupts. 374 */ 375 ASSERT(!(sc->flags & INTR_FWD)); 376 rc += p->nrxq; 377 } 378 return (rc); 379 } 380 381 /* 382 * Given an arbitrary "index," come up with an iq that can be used by other 383 * queues (of this port) for interrupt forwarding, SGE egress updates, etc. 384 * The iq returned is guaranteed to be something that takes direct interrupts. 385 */ 386 static struct sge_iq * 387 port_intr_iq(struct port_info *pi, int idx) 388 { 389 struct adapter *sc = pi->adapter; 390 struct sge *s = &sc->sge; 391 struct sge_iq *iq = NULL; 392 393 if (sc->intr_count == 1) 394 return (&sc->sge.fwq); 395 396 /* 397 * Not compiled with offload support and intr_count > 1. Only NIC 398 * queues exist and they'd better be taking direct interrupts. 399 */ 400 ASSERT(!(sc->flags & INTR_FWD)); 401 402 idx %= pi->nrxq; 403 iq = &s->rxq[pi->first_rxq + idx].iq; 404 405 return (iq); 406 } 407 408 int 409 t4_setup_port_queues(struct port_info *pi) 410 { 411 int rc = 0, i, intr_idx, j; 412 struct sge_rxq *rxq; 413 struct sge_txq *txq; 414 struct adapter *sc = pi->adapter; 415 struct driver_properties *p = &sc->props; 416 417 pi->ksp_config = setup_port_config_kstats(pi); 418 pi->ksp_info = setup_port_info_kstats(pi); 419 420 /* Interrupt vector to start from (when using multiple vectors) */ 421 intr_idx = first_vector(pi); 422 423 /* 424 * First pass over all rx queues (NIC and TOE): 425 * a) initialize iq and fl 426 * b) allocate queue iff it will take direct interrupts. 427 */ 428 429 for_each_rxq(pi, i, rxq) { 430 431 init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, p->qsize_rxq, 432 RX_IQ_ESIZE); 433 434 init_fl(&rxq->fl, p->qsize_rxq / 8); /* 8 bufs in each entry */ 435 436 if ((!(sc->flags & INTR_FWD)) || 437 (sc->intr_count > 1 && pi->nrxq)) { 438 rxq->iq.flags |= IQ_INTR; 439 rc = alloc_rxq(pi, rxq, intr_idx, i); 440 if (rc != 0) 441 goto done; 442 intr_idx++; 443 } 444 445 } 446 447 /* 448 * Second pass over all rx queues (NIC and TOE). The queues forwarding 449 * their interrupts are allocated now. 450 */ 451 j = 0; 452 for_each_rxq(pi, i, rxq) { 453 if (rxq->iq.flags & IQ_INTR) 454 continue; 455 456 intr_idx = port_intr_iq(pi, j)->abs_id; 457 458 rc = alloc_rxq(pi, rxq, intr_idx, i); 459 if (rc != 0) 460 goto done; 461 j++; 462 } 463 464 /* 465 * Now the tx queues. Only one pass needed. 466 */ 467 j = 0; 468 for_each_txq(pi, i, txq) { 469 uint16_t iqid; 470 471 iqid = port_intr_iq(pi, j)->cntxt_id; 472 init_eq(sc, &txq->eq, EQ_ETH, p->qsize_txq, pi->tx_chan, iqid); 473 rc = alloc_txq(pi, txq, i); 474 if (rc != 0) 475 goto done; 476 } 477 478 done: 479 if (rc != 0) 480 (void) t4_teardown_port_queues(pi); 481 482 return (rc); 483 } 484 485 /* 486 * Idempotent 487 */ 488 int 489 t4_teardown_port_queues(struct port_info *pi) 490 { 491 int i; 492 struct sge_rxq *rxq; 493 struct sge_txq *txq; 494 495 if (pi->ksp_config != NULL) { 496 kstat_delete(pi->ksp_config); 497 pi->ksp_config = NULL; 498 } 499 if (pi->ksp_info != NULL) { 500 kstat_delete(pi->ksp_info); 501 pi->ksp_info = NULL; 502 } 503 504 for_each_txq(pi, i, txq) { 505 (void) free_txq(pi, txq); 506 } 507 508 for_each_rxq(pi, i, rxq) { 509 if ((rxq->iq.flags & IQ_INTR) == 0) 510 (void) free_rxq(pi, rxq); 511 } 512 513 /* 514 * Then take down the rx queues that take direct interrupts. 515 */ 516 517 for_each_rxq(pi, i, rxq) { 518 if (rxq->iq.flags & IQ_INTR) 519 (void) free_rxq(pi, rxq); 520 } 521 522 return (0); 523 } 524 525 /* Deals with errors and forwarded interrupts */ 526 uint_t 527 t4_intr_all(caddr_t arg1, caddr_t arg2) 528 { 529 530 (void) t4_intr_err(arg1, arg2); 531 (void) t4_intr(arg1, arg2); 532 533 return (DDI_INTR_CLAIMED); 534 } 535 536 static void 537 t4_intr_rx_work(struct sge_iq *iq) 538 { 539 mblk_t *mp = NULL; 540 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */ 541 RXQ_LOCK(rxq); 542 if (!iq->polling) { 543 mp = t4_ring_rx(rxq, iq->qsize/8); 544 t4_write_reg(iq->adapter, MYPF_REG(A_SGE_PF_GTS), 545 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next)); 546 } 547 RXQ_UNLOCK(rxq); 548 if (mp != NULL) 549 mac_rx_ring(rxq->port->mh, rxq->ring_handle, mp, 550 rxq->ring_gen_num); 551 } 552 553 /* Deals with interrupts on the given ingress queue */ 554 /* ARGSUSED */ 555 uint_t 556 t4_intr(caddr_t arg1, caddr_t arg2) 557 { 558 struct sge_iq *iq = (struct sge_iq *)arg2; 559 int state; 560 561 /* Right now receive polling is only enabled for MSI-X and 562 * when we have enough msi-x vectors i.e no interrupt forwarding. 563 */ 564 if (iq->adapter->props.multi_rings) { 565 t4_intr_rx_work(iq); 566 } else { 567 state = atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY); 568 if (state == IQS_IDLE) { 569 (void) service_iq(iq, 0); 570 (void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE); 571 } 572 } 573 return (DDI_INTR_CLAIMED); 574 } 575 576 /* Deals with error interrupts */ 577 /* ARGSUSED */ 578 uint_t 579 t4_intr_err(caddr_t arg1, caddr_t arg2) 580 { 581 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 582 struct adapter *sc = (struct adapter *)arg1; 583 584 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0); 585 (void) t4_slow_intr_handler(sc); 586 587 return (DDI_INTR_CLAIMED); 588 } 589 590 /* 591 * t4_ring_rx - Process responses from an SGE response queue. 592 * 593 * This function processes responses from an SGE response queue up to the supplied budget. 594 * Responses include received packets as well as control messages from FW 595 * or HW. 596 * It returns a chain of mblks containing the received data, to be 597 * passed up to mac_ring_rx(). 598 */ 599 mblk_t * 600 t4_ring_rx(struct sge_rxq *rxq, int budget) 601 { 602 struct sge_iq *iq = &rxq->iq; 603 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */ 604 struct adapter *sc = iq->adapter; 605 struct rsp_ctrl *ctrl; 606 const struct rss_header *rss; 607 int ndescs = 0, fl_bufs_used = 0; 608 int rsp_type; 609 uint32_t lq; 610 mblk_t *mblk_head = NULL, **mblk_tail, *m; 611 struct cpl_rx_pkt *cpl; 612 uint32_t received_bytes = 0, pkt_len = 0; 613 bool csum_ok; 614 uint16_t err_vec; 615 616 mblk_tail = &mblk_head; 617 618 while (is_new_response(iq, &ctrl)) { 619 620 membar_consumer(); 621 622 m = NULL; 623 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen); 624 lq = be32_to_cpu(ctrl->pldbuflen_qid); 625 rss = (const void *)iq->cdesc; 626 627 switch (rsp_type) { 628 case X_RSPD_TYPE_FLBUF: 629 630 ASSERT(iq->flags & IQ_HAS_FL); 631 632 if (CPL_RX_PKT == rss->opcode) { 633 cpl = (void *)(rss + 1); 634 pkt_len = be16_to_cpu(cpl->len); 635 636 if (iq->polling && ((received_bytes + pkt_len) > budget)) 637 goto done; 638 639 m = get_fl_payload(sc, fl, lq, &fl_bufs_used); 640 if (m == NULL) 641 goto done; 642 643 iq->intr_next = iq->intr_params; 644 m->b_rptr += sc->sge.pktshift; 645 if (sc->params.tp.rx_pkt_encap) 646 /* It is enabled only in T6 config file */ 647 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec)); 648 else 649 err_vec = ntohs(cpl->err_vec); 650 651 csum_ok = cpl->csum_calc && !err_vec; 652 653 /* TODO: what about cpl->ip_frag? */ 654 if (csum_ok && !cpl->ip_frag) { 655 mac_hcksum_set(m, 0, 0, 0, 0xffff, 656 HCK_FULLCKSUM_OK | HCK_FULLCKSUM | 657 HCK_IPV4_HDRCKSUM_OK); 658 rxq->rxcsum++; 659 } 660 rxq->rxpkts++; 661 rxq->rxbytes += pkt_len; 662 received_bytes += pkt_len; 663 664 *mblk_tail = m; 665 mblk_tail = &m->b_next; 666 667 break; 668 } 669 670 m = get_fl_payload(sc, fl, lq, &fl_bufs_used); 671 if (m == NULL) 672 goto done; 673 /* FALLTHROUGH */ 674 675 case X_RSPD_TYPE_CPL: 676 ASSERT(rss->opcode < NUM_CPL_CMDS); 677 sc->cpl_handler[rss->opcode](iq, rss, m); 678 break; 679 680 default: 681 break; 682 } 683 iq_next(iq); 684 ++ndescs; 685 if (!iq->polling && (ndescs == budget)) 686 break; 687 } 688 689 done: 690 691 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 692 V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) | 693 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); 694 695 if ((fl_bufs_used > 0) || (iq->flags & IQ_HAS_FL)) { 696 int starved; 697 FL_LOCK(fl); 698 fl->needed += fl_bufs_used; 699 starved = refill_fl(sc, fl, fl->cap / 8); 700 FL_UNLOCK(fl); 701 if (starved) 702 add_fl_to_sfl(sc, fl); 703 } 704 return (mblk_head); 705 } 706 707 /* 708 * Deals with anything and everything on the given ingress queue. 709 */ 710 static int 711 service_iq(struct sge_iq *iq, int budget) 712 { 713 struct sge_iq *q; 714 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */ 715 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */ 716 struct adapter *sc = iq->adapter; 717 struct rsp_ctrl *ctrl; 718 const struct rss_header *rss; 719 int ndescs = 0, limit, fl_bufs_used = 0; 720 int rsp_type; 721 uint32_t lq; 722 int starved; 723 mblk_t *m; 724 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql); 725 726 limit = budget ? budget : iq->qsize / 8; 727 728 /* 729 * We always come back and check the descriptor ring for new indirect 730 * interrupts and other responses after running a single handler. 731 */ 732 for (;;) { 733 while (is_new_response(iq, &ctrl)) { 734 735 membar_consumer(); 736 737 m = NULL; 738 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen); 739 lq = be32_to_cpu(ctrl->pldbuflen_qid); 740 rss = (const void *)iq->cdesc; 741 742 switch (rsp_type) { 743 case X_RSPD_TYPE_FLBUF: 744 745 ASSERT(iq->flags & IQ_HAS_FL); 746 747 m = get_fl_payload(sc, fl, lq, &fl_bufs_used); 748 if (m == NULL) { 749 /* 750 * Rearm the iq with a 751 * longer-than-default timer 752 */ 753 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) | 754 V_INGRESSQID((u32)iq->cntxt_id) | 755 V_SEINTARM(V_QINTR_TIMER_IDX(SGE_NTIMERS-1))); 756 if (fl_bufs_used > 0) { 757 ASSERT(iq->flags & IQ_HAS_FL); 758 FL_LOCK(fl); 759 fl->needed += fl_bufs_used; 760 starved = refill_fl(sc, fl, fl->cap / 8); 761 FL_UNLOCK(fl); 762 if (starved) 763 add_fl_to_sfl(sc, fl); 764 } 765 return (0); 766 } 767 768 /* FALLTHRU */ 769 case X_RSPD_TYPE_CPL: 770 771 ASSERT(rss->opcode < NUM_CPL_CMDS); 772 sc->cpl_handler[rss->opcode](iq, rss, m); 773 break; 774 775 case X_RSPD_TYPE_INTR: 776 777 /* 778 * Interrupts should be forwarded only to queues 779 * that are not forwarding their interrupts. 780 * This means service_iq can recurse but only 1 781 * level deep. 782 */ 783 ASSERT(budget == 0); 784 785 q = sc->sge.iqmap[lq - sc->sge.iq_start]; 786 if (atomic_cas_uint(&q->state, IQS_IDLE, 787 IQS_BUSY) == IQS_IDLE) { 788 if (service_iq(q, q->qsize / 8) == 0) { 789 (void) atomic_cas_uint( 790 &q->state, IQS_BUSY, 791 IQS_IDLE); 792 } else { 793 STAILQ_INSERT_TAIL(&iql, q, 794 link); 795 } 796 } 797 break; 798 799 default: 800 break; 801 } 802 803 iq_next(iq); 804 if (++ndescs == limit) { 805 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 806 V_CIDXINC(ndescs) | 807 V_INGRESSQID(iq->cntxt_id) | 808 V_SEINTARM(V_QINTR_TIMER_IDX( 809 X_TIMERREG_UPDATE_CIDX))); 810 ndescs = 0; 811 812 if (fl_bufs_used > 0) { 813 ASSERT(iq->flags & IQ_HAS_FL); 814 FL_LOCK(fl); 815 fl->needed += fl_bufs_used; 816 (void) refill_fl(sc, fl, fl->cap / 8); 817 FL_UNLOCK(fl); 818 fl_bufs_used = 0; 819 } 820 821 if (budget != 0) 822 return (EINPROGRESS); 823 } 824 } 825 826 if (STAILQ_EMPTY(&iql) != 0) 827 break; 828 829 /* 830 * Process the head only, and send it to the back of the list if 831 * it's still not done. 832 */ 833 q = STAILQ_FIRST(&iql); 834 STAILQ_REMOVE_HEAD(&iql, link); 835 if (service_iq(q, q->qsize / 8) == 0) 836 (void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE); 837 else 838 STAILQ_INSERT_TAIL(&iql, q, link); 839 } 840 841 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) | 842 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next)); 843 844 if (iq->flags & IQ_HAS_FL) { 845 846 FL_LOCK(fl); 847 fl->needed += fl_bufs_used; 848 starved = refill_fl(sc, fl, fl->cap / 4); 849 FL_UNLOCK(fl); 850 if (starved != 0) 851 add_fl_to_sfl(sc, fl); 852 } 853 854 return (0); 855 } 856 857 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */ 858 #define TXPKTS_PKT_HDR ((\ 859 sizeof (struct ulp_txpkt) + \ 860 sizeof (struct ulptx_idata) + \ 861 sizeof (struct cpl_tx_pkt_core)) / 8) 862 863 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */ 864 #define TXPKTS_WR_HDR (\ 865 sizeof (struct fw_eth_tx_pkts_wr) / 8 + \ 866 TXPKTS_PKT_HDR) 867 868 /* Header of a tx WR, before SGL of first packet (in flits) */ 869 #define TXPKT_WR_HDR ((\ 870 sizeof (struct fw_eth_tx_pkt_wr) + \ 871 sizeof (struct cpl_tx_pkt_core)) / 8) 872 873 /* Header of a tx LSO WR, before SGL of first packet (in flits) */ 874 #define TXPKT_LSO_WR_HDR ((\ 875 sizeof (struct fw_eth_tx_pkt_wr) + \ 876 sizeof(struct cpl_tx_pkt_lso_core) + \ 877 sizeof (struct cpl_tx_pkt_core)) / 8) 878 879 mblk_t * 880 t4_eth_tx(void *arg, mblk_t *frame) 881 { 882 struct sge_txq *txq = (struct sge_txq *) arg; 883 struct port_info *pi = txq->port; 884 struct adapter *sc = pi->adapter; 885 struct sge_eq *eq = &txq->eq; 886 mblk_t *next_frame; 887 int rc, coalescing; 888 struct txpkts txpkts; 889 struct txinfo txinfo; 890 891 txpkts.npkt = 0; /* indicates there's nothing in txpkts */ 892 coalescing = 0; 893 894 TXQ_LOCK(txq); 895 if (eq->avail < 8) 896 (void) reclaim_tx_descs(txq, 8); 897 for (; frame; frame = next_frame) { 898 899 if (eq->avail < 8) 900 break; 901 902 next_frame = frame->b_next; 903 frame->b_next = NULL; 904 905 if (next_frame != NULL) 906 coalescing = 1; 907 908 rc = get_frame_txinfo(txq, &frame, &txinfo, coalescing); 909 if (rc != 0) { 910 if (rc == ENOMEM) { 911 912 /* Short of resources, suspend tx */ 913 914 frame->b_next = next_frame; 915 break; 916 } 917 918 /* 919 * Unrecoverable error for this frame, throw it 920 * away and move on to the next. 921 */ 922 923 freemsg(frame); 924 continue; 925 } 926 927 if (coalescing != 0 && 928 add_to_txpkts(txq, &txpkts, frame, &txinfo) == 0) { 929 930 /* Successfully absorbed into txpkts */ 931 932 write_ulp_cpl_sgl(pi, txq, &txpkts, &txinfo); 933 goto doorbell; 934 } 935 936 /* 937 * We weren't coalescing to begin with, or current frame could 938 * not be coalesced (add_to_txpkts flushes txpkts if a frame 939 * given to it can't be coalesced). Either way there should be 940 * nothing in txpkts. 941 */ 942 ASSERT(txpkts.npkt == 0); 943 944 /* We're sending out individual frames now */ 945 coalescing = 0; 946 947 if (eq->avail < 8) 948 (void) reclaim_tx_descs(txq, 8); 949 rc = write_txpkt_wr(pi, txq, frame, &txinfo); 950 if (rc != 0) { 951 952 /* Short of hardware descriptors, suspend tx */ 953 954 /* 955 * This is an unlikely but expensive failure. We've 956 * done all the hard work (DMA bindings etc.) and now we 957 * can't send out the frame. What's worse, we have to 958 * spend even more time freeing up everything in txinfo. 959 */ 960 txq->qfull++; 961 free_txinfo_resources(txq, &txinfo); 962 963 frame->b_next = next_frame; 964 break; 965 } 966 967 doorbell: 968 /* Fewer and fewer doorbells as the queue fills up */ 969 if (eq->pending >= (1 << (fls(eq->qsize - eq->avail) / 2))) { 970 txq->txbytes += txinfo.len; 971 txq->txpkts++; 972 ring_tx_db(sc, eq); 973 } 974 (void) reclaim_tx_descs(txq, 32); 975 } 976 977 if (txpkts.npkt > 0) 978 write_txpkts_wr(txq, &txpkts); 979 980 /* 981 * frame not NULL means there was an error but we haven't thrown it 982 * away. This can happen when we're short of tx descriptors (qfull) or 983 * maybe even DMA handles (dma_hdl_failed). Either way, a credit flush 984 * and reclaim will get things going again. 985 * 986 * If eq->avail is already 0 we know a credit flush was requested in the 987 * WR that reduced it to 0 so we don't need another flush (we don't have 988 * any descriptor for a flush WR anyway, duh). 989 */ 990 if (frame && eq->avail > 0) 991 write_txqflush_wr(txq); 992 993 if (eq->pending != 0) 994 ring_tx_db(sc, eq); 995 996 (void) reclaim_tx_descs(txq, eq->qsize); 997 TXQ_UNLOCK(txq); 998 999 return (frame); 1000 } 1001 1002 static inline void 1003 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int8_t pktc_idx, 1004 int qsize, uint8_t esize) 1005 { 1006 ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS); 1007 ASSERT(pktc_idx < SGE_NCOUNTERS); /* -ve is ok, means don't use */ 1008 1009 iq->flags = 0; 1010 iq->adapter = sc; 1011 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx); 1012 iq->intr_pktc_idx = SGE_NCOUNTERS - 1; 1013 if (pktc_idx >= 0) { 1014 iq->intr_params |= F_QINTR_CNT_EN; 1015 iq->intr_pktc_idx = pktc_idx; 1016 } 1017 iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */ 1018 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */ 1019 } 1020 1021 static inline void 1022 init_fl(struct sge_fl *fl, uint16_t qsize) 1023 { 1024 1025 fl->qsize = qsize; 1026 fl->allocb_fail = 0; 1027 } 1028 1029 static inline void 1030 init_eq(struct adapter *sc, struct sge_eq *eq, uint16_t eqtype, uint16_t qsize, 1031 uint8_t tx_chan, uint16_t iqid) 1032 { 1033 struct sge *s = &sc->sge; 1034 uint32_t r; 1035 1036 ASSERT(tx_chan < NCHAN); 1037 ASSERT(eqtype <= EQ_TYPEMASK); 1038 1039 if (is_t5(sc->params.chip)) { 1040 r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF); 1041 r >>= S_QUEUESPERPAGEPF0 + 1042 (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf; 1043 s->s_qpp = r & M_QUEUESPERPAGEPF0; 1044 } 1045 1046 eq->flags = eqtype & EQ_TYPEMASK; 1047 eq->tx_chan = tx_chan; 1048 eq->iqid = iqid; 1049 eq->qsize = qsize; 1050 } 1051 1052 /* 1053 * Allocates the ring for an ingress queue and an optional freelist. If the 1054 * freelist is specified it will be allocated and then associated with the 1055 * ingress queue. 1056 * 1057 * Returns errno on failure. Resources allocated up to that point may still be 1058 * allocated. Caller is responsible for cleanup in case this function fails. 1059 * 1060 * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then 1061 * the intr_idx specifies the vector, starting from 0. Otherwise it specifies 1062 * the index of the queue to which its interrupts will be forwarded. 1063 */ 1064 static int 1065 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl, 1066 int intr_idx, int cong) 1067 { 1068 int rc, i, cntxt_id; 1069 size_t len; 1070 struct fw_iq_cmd c; 1071 struct adapter *sc = iq->adapter; 1072 uint32_t v = 0; 1073 1074 len = iq->qsize * iq->esize; 1075 rc = alloc_desc_ring(sc, len, DDI_DMA_READ, &iq->dhdl, &iq->ahdl, 1076 &iq->ba, (caddr_t *)&iq->desc); 1077 if (rc != 0) 1078 return (rc); 1079 1080 bzero(&c, sizeof (c)); 1081 c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | 1082 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) | 1083 V_FW_IQ_CMD_VFN(0)); 1084 1085 c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART | 1086 FW_LEN16(c)); 1087 1088 /* Special handling for firmware event queue */ 1089 if (iq == &sc->sge.fwq) 1090 v |= F_FW_IQ_CMD_IQASYNCH; 1091 1092 if (iq->flags & IQ_INTR) 1093 ASSERT(intr_idx < sc->intr_count); 1094 else 1095 v |= F_FW_IQ_CMD_IQANDST; 1096 v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx); 1097 1098 c.type_to_iqandstindex = cpu_to_be32(v | 1099 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | 1100 V_FW_IQ_CMD_VIID(pi->viid) | 1101 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT)); 1102 c.iqdroprss_to_iqesize = cpu_to_be16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | 1103 F_FW_IQ_CMD_IQGTSMODE | 1104 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) | 1105 V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4)); 1106 c.iqsize = cpu_to_be16(iq->qsize); 1107 c.iqaddr = cpu_to_be64(iq->ba); 1108 if (cong >= 0) 1109 c.iqns_to_fl0congen = BE_32(F_FW_IQ_CMD_IQFLINTCONGEN | 1110 V_FW_IQ_CMD_IQTYPE(cong ? 1111 FW_IQ_IQTYPE_NIC : FW_IQ_IQTYPE_OFLD)); 1112 1113 if (fl != NULL) { 1114 unsigned int chip_ver = CHELSIO_CHIP_VERSION(sc->params.chip); 1115 1116 mutex_init(&fl->lock, NULL, MUTEX_DRIVER, 1117 DDI_INTR_PRI(sc->intr_pri)); 1118 fl->flags |= FL_MTX; 1119 1120 len = fl->qsize * RX_FL_ESIZE; 1121 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &fl->dhdl, 1122 &fl->ahdl, &fl->ba, (caddr_t *)&fl->desc); 1123 if (rc != 0) 1124 return (rc); 1125 1126 /* Allocate space for one software descriptor per buffer. */ 1127 fl->cap = (fl->qsize - sc->sge.stat_len / RX_FL_ESIZE) * 8; 1128 fl->sdesc = kmem_zalloc(sizeof (struct fl_sdesc) * fl->cap, 1129 KM_SLEEP); 1130 fl->needed = fl->cap; 1131 fl->lowat = roundup(sc->sge.fl_starve_threshold, 8); 1132 1133 c.iqns_to_fl0congen |= 1134 cpu_to_be32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) | 1135 F_FW_IQ_CMD_FL0PACKEN | F_FW_IQ_CMD_FL0PADEN); 1136 if (cong >= 0) { 1137 c.iqns_to_fl0congen |= 1138 BE_32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) | 1139 F_FW_IQ_CMD_FL0CONGCIF | 1140 F_FW_IQ_CMD_FL0CONGEN); 1141 } 1142 1143 /* In T6, for egress queue type FL there is internal overhead 1144 * of 16B for header going into FLM module. Hence the maximum 1145 * allowed burst size is 448 bytes. For T4/T5, the hardware 1146 * doesn't coalesce fetch requests if more than 64 bytes of 1147 * Free List pointers are provided, so we use a 128-byte Fetch 1148 * Burst Minimum there (T6 implements coalescing so we can use 1149 * the smaller 64-byte value there). 1150 */ 1151 1152 c.fl0dcaen_to_fl0cidxfthresh = 1153 cpu_to_be16(V_FW_IQ_CMD_FL0FBMIN(chip_ver <= CHELSIO_T5 1154 ? X_FETCHBURSTMIN_128B 1155 : X_FETCHBURSTMIN_64B) | 1156 V_FW_IQ_CMD_FL0FBMAX(chip_ver <= CHELSIO_T5 1157 ? X_FETCHBURSTMAX_512B 1158 : X_FETCHBURSTMAX_256B)); 1159 c.fl0size = cpu_to_be16(fl->qsize); 1160 c.fl0addr = cpu_to_be64(fl->ba); 1161 } 1162 1163 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1164 if (rc != 0) { 1165 cxgb_printf(sc->dip, CE_WARN, 1166 "failed to create ingress queue: %d", rc); 1167 return (rc); 1168 } 1169 1170 iq->cdesc = iq->desc; 1171 iq->cidx = 0; 1172 iq->gen = 1; 1173 iq->intr_next = iq->intr_params; 1174 iq->adapter = sc; 1175 iq->cntxt_id = be16_to_cpu(c.iqid); 1176 iq->abs_id = be16_to_cpu(c.physiqid); 1177 iq->flags |= IQ_ALLOCATED; 1178 mutex_init(&iq->lock, NULL, 1179 MUTEX_DRIVER, DDI_INTR_PRI(DDI_INTR_PRI(sc->intr_pri))); 1180 iq->polling = 0; 1181 1182 cntxt_id = iq->cntxt_id - sc->sge.iq_start; 1183 if (cntxt_id >= sc->sge.iqmap_sz) { 1184 panic("%s: iq->cntxt_id (%d) more than the max (%d)", __func__, 1185 cntxt_id, sc->sge.iqmap_sz - 1); 1186 } 1187 sc->sge.iqmap[cntxt_id] = iq; 1188 1189 if (fl != NULL) { 1190 fl->cntxt_id = be16_to_cpu(c.fl0id); 1191 fl->pidx = fl->cidx = 0; 1192 fl->copy_threshold = rx_copy_threshold; 1193 1194 cntxt_id = fl->cntxt_id - sc->sge.eq_start; 1195 if (cntxt_id >= sc->sge.eqmap_sz) { 1196 panic("%s: fl->cntxt_id (%d) more than the max (%d)", 1197 __func__, cntxt_id, sc->sge.eqmap_sz - 1); 1198 } 1199 sc->sge.eqmap[cntxt_id] = (void *)fl; 1200 1201 FL_LOCK(fl); 1202 (void) refill_fl(sc, fl, fl->lowat); 1203 FL_UNLOCK(fl); 1204 1205 iq->flags |= IQ_HAS_FL; 1206 } 1207 1208 if (is_t5(sc->params.chip) && cong >= 0) { 1209 uint32_t param, val; 1210 1211 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 1212 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) | 1213 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id); 1214 if (cong == 0) 1215 val = 1 << 19; 1216 else { 1217 val = 2 << 19; 1218 for (i = 0; i < 4; i++) { 1219 if (cong & (1 << i)) 1220 val |= 1 << (i << 2); 1221 } 1222 } 1223 1224 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1225 if (rc != 0) { 1226 /* report error but carry on */ 1227 cxgb_printf(sc->dip, CE_WARN, 1228 "failed to set congestion manager context for " 1229 "ingress queue %d: %d", iq->cntxt_id, rc); 1230 } 1231 } 1232 1233 /* Enable IQ interrupts */ 1234 iq->state = IQS_IDLE; 1235 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) | 1236 V_INGRESSQID(iq->cntxt_id)); 1237 1238 return (0); 1239 } 1240 1241 static int 1242 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl) 1243 { 1244 int rc; 1245 1246 if (iq != NULL) { 1247 struct adapter *sc = iq->adapter; 1248 dev_info_t *dip; 1249 1250 dip = pi ? pi->dip : sc->dip; 1251 if (iq->flags & IQ_ALLOCATED) { 1252 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, 1253 FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id, 1254 fl ? fl->cntxt_id : 0xffff, 0xffff); 1255 if (rc != 0) { 1256 cxgb_printf(dip, CE_WARN, 1257 "failed to free queue %p: %d", iq, rc); 1258 return (rc); 1259 } 1260 mutex_destroy(&iq->lock); 1261 iq->flags &= ~IQ_ALLOCATED; 1262 } 1263 1264 if (iq->desc != NULL) { 1265 (void) free_desc_ring(&iq->dhdl, &iq->ahdl); 1266 iq->desc = NULL; 1267 } 1268 1269 bzero(iq, sizeof (*iq)); 1270 } 1271 1272 if (fl != NULL) { 1273 if (fl->sdesc != NULL) { 1274 FL_LOCK(fl); 1275 free_fl_bufs(fl); 1276 FL_UNLOCK(fl); 1277 1278 kmem_free(fl->sdesc, sizeof (struct fl_sdesc) * 1279 fl->cap); 1280 fl->sdesc = NULL; 1281 } 1282 1283 if (fl->desc != NULL) { 1284 (void) free_desc_ring(&fl->dhdl, &fl->ahdl); 1285 fl->desc = NULL; 1286 } 1287 1288 if (fl->flags & FL_MTX) { 1289 mutex_destroy(&fl->lock); 1290 fl->flags &= ~FL_MTX; 1291 } 1292 1293 bzero(fl, sizeof (struct sge_fl)); 1294 } 1295 1296 return (0); 1297 } 1298 1299 static int 1300 alloc_fwq(struct adapter *sc) 1301 { 1302 int rc, intr_idx; 1303 struct sge_iq *fwq = &sc->sge.fwq; 1304 1305 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE); 1306 fwq->flags |= IQ_INTR; /* always */ 1307 intr_idx = sc->intr_count > 1 ? 1 : 0; 1308 rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1); 1309 if (rc != 0) { 1310 cxgb_printf(sc->dip, CE_WARN, 1311 "failed to create firmware event queue: %d.", rc); 1312 return (rc); 1313 } 1314 1315 return (0); 1316 } 1317 1318 static int 1319 free_fwq(struct adapter *sc) 1320 { 1321 1322 return (free_iq_fl(NULL, &sc->sge.fwq, NULL)); 1323 } 1324 1325 static int 1326 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int i) 1327 { 1328 int rc; 1329 1330 rxq->port = pi; 1331 rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx, 1332 t4_get_tp_ch_map(pi->adapter, pi->tx_chan)); 1333 if (rc != 0) 1334 return (rc); 1335 1336 rxq->ksp = setup_rxq_kstats(pi, rxq, i); 1337 1338 return (rc); 1339 } 1340 1341 static int 1342 free_rxq(struct port_info *pi, struct sge_rxq *rxq) 1343 { 1344 int rc; 1345 1346 if (rxq->ksp != NULL) { 1347 kstat_delete(rxq->ksp); 1348 rxq->ksp = NULL; 1349 } 1350 1351 rc = free_iq_fl(pi, &rxq->iq, &rxq->fl); 1352 if (rc == 0) 1353 bzero(&rxq->fl, sizeof (*rxq) - offsetof(struct sge_rxq, fl)); 1354 1355 return (rc); 1356 } 1357 1358 static int 1359 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) 1360 { 1361 int rc, cntxt_id; 1362 struct fw_eq_ctrl_cmd c; 1363 1364 bzero(&c, sizeof (c)); 1365 1366 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST | 1367 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) | 1368 V_FW_EQ_CTRL_CMD_VFN(0)); 1369 c.alloc_to_len16 = BE_32(F_FW_EQ_CTRL_CMD_ALLOC | 1370 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c)); 1371 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* TODO */ 1372 c.physeqid_pkd = BE_32(0); 1373 c.fetchszm_to_iqid = 1374 BE_32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1375 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) | 1376 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid)); 1377 c.dcaen_to_eqsize = 1378 BE_32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1379 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1380 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1381 V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize)); 1382 c.eqaddr = BE_64(eq->ba); 1383 1384 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1385 if (rc != 0) { 1386 cxgb_printf(sc->dip, CE_WARN, 1387 "failed to create control queue %d: %d", eq->tx_chan, rc); 1388 return (rc); 1389 } 1390 eq->flags |= EQ_ALLOCATED; 1391 1392 eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(BE_32(c.cmpliqid_eqid)); 1393 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1394 if (cntxt_id >= sc->sge.eqmap_sz) 1395 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1396 cntxt_id, sc->sge.eqmap_sz - 1); 1397 sc->sge.eqmap[cntxt_id] = eq; 1398 1399 return (rc); 1400 } 1401 1402 static int 1403 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1404 { 1405 int rc, cntxt_id; 1406 struct fw_eq_eth_cmd c; 1407 1408 bzero(&c, sizeof (c)); 1409 1410 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST | 1411 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) | 1412 V_FW_EQ_ETH_CMD_VFN(0)); 1413 c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC | 1414 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c)); 1415 c.autoequiqe_to_viid = BE_32(F_FW_EQ_ETH_CMD_AUTOEQUIQE | 1416 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(pi->viid)); 1417 c.fetchszm_to_iqid = 1418 BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1419 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | 1420 V_FW_EQ_ETH_CMD_IQID(eq->iqid)); 1421 c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1422 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1423 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1424 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize)); 1425 c.eqaddr = BE_64(eq->ba); 1426 1427 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1428 if (rc != 0) { 1429 cxgb_printf(pi->dip, CE_WARN, 1430 "failed to create Ethernet egress queue: %d", rc); 1431 return (rc); 1432 } 1433 eq->flags |= EQ_ALLOCATED; 1434 1435 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd)); 1436 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1437 if (cntxt_id >= sc->sge.eqmap_sz) 1438 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1439 cntxt_id, sc->sge.eqmap_sz - 1); 1440 sc->sge.eqmap[cntxt_id] = eq; 1441 1442 return (rc); 1443 } 1444 1445 static int 1446 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1447 { 1448 int rc; 1449 size_t len; 1450 1451 mutex_init(&eq->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(sc->intr_pri)); 1452 eq->flags |= EQ_MTX; 1453 1454 len = eq->qsize * EQ_ESIZE; 1455 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &eq->desc_dhdl, 1456 &eq->desc_ahdl, &eq->ba, (caddr_t *)&eq->desc); 1457 if (rc != 0) 1458 return (rc); 1459 1460 eq->cap = eq->qsize - sc->sge.stat_len / EQ_ESIZE; 1461 eq->spg = (void *)&eq->desc[eq->cap]; 1462 eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */ 1463 eq->pidx = eq->cidx = 0; 1464 eq->doorbells = sc->doorbells; 1465 1466 switch (eq->flags & EQ_TYPEMASK) { 1467 case EQ_CTRL: 1468 rc = ctrl_eq_alloc(sc, eq); 1469 break; 1470 1471 case EQ_ETH: 1472 rc = eth_eq_alloc(sc, pi, eq); 1473 break; 1474 1475 default: 1476 panic("%s: invalid eq type %d.", __func__, 1477 eq->flags & EQ_TYPEMASK); 1478 } 1479 1480 if (eq->doorbells & 1481 (DOORBELL_UDB | DOORBELL_UDBWC | DOORBELL_WCWR)) { 1482 uint32_t s_qpp = sc->sge.s_qpp; 1483 uint32_t mask = (1 << s_qpp) - 1; 1484 volatile uint8_t *udb; 1485 1486 udb = (volatile uint8_t *)sc->reg1p + UDBS_DB_OFFSET; 1487 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */ 1488 eq->udb_qid = eq->cntxt_id & mask; /* id in page */ 1489 if (eq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE) 1490 eq->doorbells &= ~DOORBELL_WCWR; 1491 else { 1492 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */ 1493 eq->udb_qid = 0; 1494 } 1495 eq->udb = (volatile void *)udb; 1496 } 1497 1498 if (rc != 0) { 1499 cxgb_printf(sc->dip, CE_WARN, 1500 "failed to allocate egress queue(%d): %d", 1501 eq->flags & EQ_TYPEMASK, rc); 1502 } 1503 1504 return (rc); 1505 } 1506 1507 static int 1508 free_eq(struct adapter *sc, struct sge_eq *eq) 1509 { 1510 int rc; 1511 1512 if (eq->flags & EQ_ALLOCATED) { 1513 switch (eq->flags & EQ_TYPEMASK) { 1514 case EQ_CTRL: 1515 rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0, 1516 eq->cntxt_id); 1517 break; 1518 1519 case EQ_ETH: 1520 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, 1521 eq->cntxt_id); 1522 break; 1523 default: 1524 panic("%s: invalid eq type %d.", __func__, 1525 eq->flags & EQ_TYPEMASK); 1526 } 1527 if (rc != 0) { 1528 cxgb_printf(sc->dip, CE_WARN, 1529 "failed to free egress queue (%d): %d", 1530 eq->flags & EQ_TYPEMASK, rc); 1531 return (rc); 1532 } 1533 eq->flags &= ~EQ_ALLOCATED; 1534 } 1535 1536 if (eq->desc != NULL) { 1537 (void) free_desc_ring(&eq->desc_dhdl, &eq->desc_ahdl); 1538 eq->desc = NULL; 1539 } 1540 1541 if (eq->flags & EQ_MTX) 1542 mutex_destroy(&eq->lock); 1543 1544 bzero(eq, sizeof (*eq)); 1545 return (0); 1546 } 1547 1548 static int 1549 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx) 1550 { 1551 int rc, i; 1552 struct adapter *sc = pi->adapter; 1553 struct sge_eq *eq = &txq->eq; 1554 1555 rc = alloc_eq(sc, pi, eq); 1556 if (rc != 0) 1557 return (rc); 1558 1559 txq->port = pi; 1560 txq->sdesc = kmem_zalloc(sizeof (struct tx_sdesc) * eq->cap, KM_SLEEP); 1561 txq->txb_size = eq->qsize * tx_copy_threshold; 1562 rc = alloc_tx_copybuffer(sc, txq->txb_size, &txq->txb_dhdl, 1563 &txq->txb_ahdl, &txq->txb_ba, &txq->txb_va); 1564 if (rc == 0) 1565 txq->txb_avail = txq->txb_size; 1566 else 1567 txq->txb_avail = txq->txb_size = 0; 1568 1569 /* 1570 * TODO: is this too low? Worst case would need around 4 times qsize 1571 * (all tx descriptors filled to the brim with SGLs, with each entry in 1572 * the SGL coming from a distinct DMA handle). Increase tx_dhdl_total 1573 * if you see too many dma_hdl_failed. 1574 */ 1575 txq->tx_dhdl_total = eq->qsize * 2; 1576 txq->tx_dhdl = kmem_zalloc(sizeof (ddi_dma_handle_t) * 1577 txq->tx_dhdl_total, KM_SLEEP); 1578 for (i = 0; i < txq->tx_dhdl_total; i++) { 1579 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx, 1580 DDI_DMA_SLEEP, 0, &txq->tx_dhdl[i]); 1581 if (rc != DDI_SUCCESS) { 1582 cxgb_printf(sc->dip, CE_WARN, 1583 "%s: failed to allocate DMA handle (%d)", 1584 __func__, rc); 1585 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 1586 } 1587 txq->tx_dhdl_avail++; 1588 } 1589 1590 txq->ksp = setup_txq_kstats(pi, txq, idx); 1591 1592 return (rc); 1593 } 1594 1595 static int 1596 free_txq(struct port_info *pi, struct sge_txq *txq) 1597 { 1598 int i; 1599 struct adapter *sc = pi->adapter; 1600 struct sge_eq *eq = &txq->eq; 1601 1602 if (txq->ksp != NULL) { 1603 kstat_delete(txq->ksp); 1604 txq->ksp = NULL; 1605 } 1606 1607 if (txq->txb_va != NULL) { 1608 (void) free_desc_ring(&txq->txb_dhdl, &txq->txb_ahdl); 1609 txq->txb_va = NULL; 1610 } 1611 1612 if (txq->sdesc != NULL) { 1613 struct tx_sdesc *sd; 1614 ddi_dma_handle_t hdl; 1615 1616 TXQ_LOCK(txq); 1617 while (eq->cidx != eq->pidx) { 1618 sd = &txq->sdesc[eq->cidx]; 1619 1620 for (i = sd->hdls_used; i; i--) { 1621 hdl = txq->tx_dhdl[txq->tx_dhdl_cidx]; 1622 (void) ddi_dma_unbind_handle(hdl); 1623 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total) 1624 txq->tx_dhdl_cidx = 0; 1625 } 1626 1627 ASSERT(sd->m); 1628 freemsgchain(sd->m); 1629 1630 eq->cidx += sd->desc_used; 1631 if (eq->cidx >= eq->cap) 1632 eq->cidx -= eq->cap; 1633 1634 txq->txb_avail += txq->txb_used; 1635 } 1636 ASSERT(txq->tx_dhdl_cidx == txq->tx_dhdl_pidx); 1637 ASSERT(txq->txb_avail == txq->txb_size); 1638 TXQ_UNLOCK(txq); 1639 1640 kmem_free(txq->sdesc, sizeof (struct tx_sdesc) * eq->cap); 1641 txq->sdesc = NULL; 1642 } 1643 1644 if (txq->tx_dhdl != NULL) { 1645 for (i = 0; i < txq->tx_dhdl_total; i++) { 1646 if (txq->tx_dhdl[i] != NULL) 1647 ddi_dma_free_handle(&txq->tx_dhdl[i]); 1648 } 1649 } 1650 1651 (void) free_eq(sc, &txq->eq); 1652 1653 bzero(txq, sizeof (*txq)); 1654 return (0); 1655 } 1656 1657 /* 1658 * Allocates a block of contiguous memory for DMA. Can be used to allocate 1659 * memory for descriptor rings or for tx/rx copy buffers. 1660 * 1661 * Caller does not have to clean up anything if this function fails, it cleans 1662 * up after itself. 1663 * 1664 * Caller provides the following: 1665 * len length of the block of memory to allocate. 1666 * flags DDI_DMA_* flags to use (CONSISTENT/STREAMING, READ/WRITE/RDWR) 1667 * acc_attr device access attributes for the allocation. 1668 * dma_attr DMA attributes for the allocation 1669 * 1670 * If the function is successful it fills up this information: 1671 * dma_hdl DMA handle for the allocated memory 1672 * acc_hdl access handle for the allocated memory 1673 * ba bus address of the allocated memory 1674 * va KVA of the allocated memory. 1675 */ 1676 static int 1677 alloc_dma_memory(struct adapter *sc, size_t len, int flags, 1678 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr, 1679 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 1680 uint64_t *pba, caddr_t *pva) 1681 { 1682 int rc; 1683 ddi_dma_handle_t dhdl; 1684 ddi_acc_handle_t ahdl; 1685 ddi_dma_cookie_t cookie; 1686 uint_t ccount; 1687 caddr_t va; 1688 size_t real_len; 1689 1690 *pva = NULL; 1691 1692 /* 1693 * DMA handle. 1694 */ 1695 rc = ddi_dma_alloc_handle(sc->dip, dma_attr, DDI_DMA_SLEEP, 0, &dhdl); 1696 if (rc != DDI_SUCCESS) { 1697 cxgb_printf(sc->dip, CE_WARN, 1698 "failed to allocate DMA handle: %d", rc); 1699 1700 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 1701 } 1702 1703 /* 1704 * Memory suitable for DMA. 1705 */ 1706 rc = ddi_dma_mem_alloc(dhdl, len, acc_attr, 1707 flags & DDI_DMA_CONSISTENT ? DDI_DMA_CONSISTENT : DDI_DMA_STREAMING, 1708 DDI_DMA_SLEEP, 0, &va, &real_len, &ahdl); 1709 if (rc != DDI_SUCCESS) { 1710 cxgb_printf(sc->dip, CE_WARN, 1711 "failed to allocate DMA memory: %d", rc); 1712 1713 ddi_dma_free_handle(&dhdl); 1714 return (ENOMEM); 1715 } 1716 1717 if (len != real_len) { 1718 cxgb_printf(sc->dip, CE_WARN, 1719 "%s: len (%u) != real_len (%u)\n", len, real_len); 1720 } 1721 1722 /* 1723 * DMA bindings. 1724 */ 1725 rc = ddi_dma_addr_bind_handle(dhdl, NULL, va, real_len, flags, NULL, 1726 NULL, &cookie, &ccount); 1727 if (rc != DDI_DMA_MAPPED) { 1728 cxgb_printf(sc->dip, CE_WARN, 1729 "failed to map DMA memory: %d", rc); 1730 1731 ddi_dma_mem_free(&ahdl); 1732 ddi_dma_free_handle(&dhdl); 1733 return (ENOMEM); 1734 } 1735 if (ccount != 1) { 1736 cxgb_printf(sc->dip, CE_WARN, 1737 "unusable DMA mapping (%d segments)", ccount); 1738 (void) free_desc_ring(&dhdl, &ahdl); 1739 } 1740 1741 bzero(va, real_len); 1742 *dma_hdl = dhdl; 1743 *acc_hdl = ahdl; 1744 *pba = cookie.dmac_laddress; 1745 *pva = va; 1746 1747 return (0); 1748 } 1749 1750 static int 1751 free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl) 1752 { 1753 (void) ddi_dma_unbind_handle(*dhdl); 1754 ddi_dma_mem_free(ahdl); 1755 ddi_dma_free_handle(dhdl); 1756 1757 return (0); 1758 } 1759 1760 static int 1761 alloc_desc_ring(struct adapter *sc, size_t len, int rw, 1762 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 1763 uint64_t *pba, caddr_t *pva) 1764 { 1765 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_desc; 1766 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; 1767 1768 return (alloc_dma_memory(sc, len, DDI_DMA_CONSISTENT | rw, acc_attr, 1769 dma_attr, dma_hdl, acc_hdl, pba, pva)); 1770 } 1771 1772 static int 1773 free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl) 1774 { 1775 return (free_dma_memory(dhdl, ahdl)); 1776 } 1777 1778 static int 1779 alloc_tx_copybuffer(struct adapter *sc, size_t len, 1780 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 1781 uint64_t *pba, caddr_t *pva) 1782 { 1783 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_tx; 1784 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; /* NOT dma_attr_tx */ 1785 1786 return (alloc_dma_memory(sc, len, DDI_DMA_STREAMING | DDI_DMA_WRITE, 1787 acc_attr, dma_attr, dma_hdl, acc_hdl, pba, pva)); 1788 } 1789 1790 static inline bool 1791 is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl) 1792 { 1793 (void) ddi_dma_sync(iq->dhdl, (uintptr_t)iq->cdesc - 1794 (uintptr_t)iq->desc, iq->esize, DDI_DMA_SYNC_FORKERNEL); 1795 1796 *ctrl = (void *)((uintptr_t)iq->cdesc + 1797 (iq->esize - sizeof (struct rsp_ctrl))); 1798 1799 return ((((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen)); 1800 } 1801 1802 static inline void 1803 iq_next(struct sge_iq *iq) 1804 { 1805 iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize); 1806 if (++iq->cidx == iq->qsize - 1) { 1807 iq->cidx = 0; 1808 iq->gen ^= 1; 1809 iq->cdesc = iq->desc; 1810 } 1811 } 1812 1813 /* 1814 * Fill up the freelist by upto nbufs and maybe ring its doorbell. 1815 * 1816 * Returns non-zero to indicate that it should be added to the list of starving 1817 * freelists. 1818 */ 1819 static int 1820 refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs) 1821 { 1822 uint64_t *d = &fl->desc[fl->pidx]; 1823 struct fl_sdesc *sd = &fl->sdesc[fl->pidx]; 1824 1825 FL_LOCK_ASSERT_OWNED(fl); 1826 ASSERT(nbufs >= 0); 1827 1828 if (nbufs > fl->needed) 1829 nbufs = fl->needed; 1830 1831 while (nbufs--) { 1832 if (sd->rxb != NULL) { 1833 if (sd->rxb->ref_cnt == 1) { 1834 /* 1835 * Buffer is available for recycling. Two ways 1836 * this can happen: 1837 * 1838 * a) All the packets DMA'd into it last time 1839 * around were within the rx_copy_threshold 1840 * and no part of the buffer was ever passed 1841 * up (ref_cnt never went over 1). 1842 * 1843 * b) Packets DMA'd into the buffer were passed 1844 * up but have all been freed by the upper 1845 * layers by now (ref_cnt went over 1 but is 1846 * now back to 1). 1847 * 1848 * Either way the bus address in the descriptor 1849 * ring is already valid. 1850 */ 1851 ASSERT(*d == cpu_to_be64(sd->rxb->ba)); 1852 d++; 1853 goto recycled; 1854 } else { 1855 /* 1856 * Buffer still in use and we need a 1857 * replacement. But first release our reference 1858 * on the existing buffer. 1859 */ 1860 rxbuf_free(sd->rxb); 1861 } 1862 } 1863 1864 sd->rxb = rxbuf_alloc(sc->sge.rxbuf_cache, KM_NOSLEEP, 1); 1865 if (sd->rxb == NULL) 1866 break; 1867 *d++ = cpu_to_be64(sd->rxb->ba); 1868 1869 recycled: fl->pending++; 1870 sd++; 1871 fl->needed--; 1872 if (++fl->pidx == fl->cap) { 1873 fl->pidx = 0; 1874 sd = fl->sdesc; 1875 d = fl->desc; 1876 } 1877 } 1878 1879 if (fl->pending >= 8) 1880 ring_fl_db(sc, fl); 1881 1882 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING)); 1883 } 1884 1885 #ifndef TAILQ_FOREACH_SAFE 1886 #define TAILQ_FOREACH_SAFE(var, head, field, tvar) \ 1887 for ((var) = TAILQ_FIRST((head)); \ 1888 (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \ 1889 (var) = (tvar)) 1890 #endif 1891 1892 /* 1893 * Attempt to refill all starving freelists. 1894 */ 1895 static void 1896 refill_sfl(void *arg) 1897 { 1898 struct adapter *sc = arg; 1899 struct sge_fl *fl, *fl_temp; 1900 1901 mutex_enter(&sc->sfl_lock); 1902 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) { 1903 FL_LOCK(fl); 1904 (void) refill_fl(sc, fl, 64); 1905 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) { 1906 TAILQ_REMOVE(&sc->sfl, fl, link); 1907 fl->flags &= ~FL_STARVING; 1908 } 1909 FL_UNLOCK(fl); 1910 } 1911 1912 if (!TAILQ_EMPTY(&sc->sfl) != 0) 1913 sc->sfl_timer = timeout(refill_sfl, sc, drv_usectohz(100000)); 1914 mutex_exit(&sc->sfl_lock); 1915 } 1916 1917 static void 1918 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl) 1919 { 1920 mutex_enter(&sc->sfl_lock); 1921 FL_LOCK(fl); 1922 if ((fl->flags & FL_DOOMED) == 0) { 1923 if (TAILQ_EMPTY(&sc->sfl) != 0) { 1924 sc->sfl_timer = timeout(refill_sfl, sc, 1925 drv_usectohz(100000)); 1926 } 1927 fl->flags |= FL_STARVING; 1928 TAILQ_INSERT_TAIL(&sc->sfl, fl, link); 1929 } 1930 FL_UNLOCK(fl); 1931 mutex_exit(&sc->sfl_lock); 1932 } 1933 1934 static void 1935 free_fl_bufs(struct sge_fl *fl) 1936 { 1937 struct fl_sdesc *sd; 1938 unsigned int i; 1939 1940 FL_LOCK_ASSERT_OWNED(fl); 1941 1942 for (i = 0; i < fl->cap; i++) { 1943 sd = &fl->sdesc[i]; 1944 1945 if (sd->rxb != NULL) { 1946 rxbuf_free(sd->rxb); 1947 sd->rxb = NULL; 1948 } 1949 } 1950 } 1951 1952 /* 1953 * Note that fl->cidx and fl->offset are left unchanged in case of failure. 1954 */ 1955 static mblk_t * 1956 get_fl_payload(struct adapter *sc, struct sge_fl *fl, 1957 uint32_t len_newbuf, int *fl_bufs_used) 1958 { 1959 struct mblk_pair frame = {0}; 1960 struct rxbuf *rxb; 1961 mblk_t *m = NULL; 1962 uint_t nbuf = 0, len, copy, n; 1963 uint32_t cidx, offset, rcidx, roffset; 1964 1965 /* 1966 * The SGE won't pack a new frame into the current buffer if the entire 1967 * payload doesn't fit in the remaining space. Move on to the next buf 1968 * in that case. 1969 */ 1970 rcidx = fl->cidx; 1971 roffset = fl->offset; 1972 if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) { 1973 fl->offset = 0; 1974 if (++fl->cidx == fl->cap) 1975 fl->cidx = 0; 1976 nbuf++; 1977 } 1978 cidx = fl->cidx; 1979 offset = fl->offset; 1980 1981 len = G_RSPD_LEN(len_newbuf); /* pktshift + payload length */ 1982 copy = (len <= fl->copy_threshold); 1983 if (copy != 0) { 1984 frame.head = m = allocb(len, BPRI_HI); 1985 if (m == NULL) { 1986 fl->allocb_fail++; 1987 cmn_err(CE_WARN,"%s: mbuf allocation failure " 1988 "count = %llu", __func__, 1989 (unsigned long long)fl->allocb_fail); 1990 fl->cidx = rcidx; 1991 fl->offset = roffset; 1992 return (NULL); 1993 } 1994 } 1995 1996 while (len) { 1997 rxb = fl->sdesc[cidx].rxb; 1998 n = min(len, rxb->buf_size - offset); 1999 2000 (void) ddi_dma_sync(rxb->dhdl, offset, n, 2001 DDI_DMA_SYNC_FORKERNEL); 2002 2003 if (copy != 0) 2004 bcopy(rxb->va + offset, m->b_wptr, n); 2005 else { 2006 m = desballoc((unsigned char *)rxb->va + offset, n, 2007 BPRI_HI, &rxb->freefunc); 2008 if (m == NULL) { 2009 fl->allocb_fail++; 2010 cmn_err(CE_WARN, 2011 "%s: mbuf allocation failure " 2012 "count = %llu", __func__, 2013 (unsigned long long)fl->allocb_fail); 2014 if (frame.head) 2015 freemsgchain(frame.head); 2016 fl->cidx = rcidx; 2017 fl->offset = roffset; 2018 return (NULL); 2019 } 2020 atomic_inc_uint(&rxb->ref_cnt); 2021 if (frame.head != NULL) 2022 frame.tail->b_cont = m; 2023 else 2024 frame.head = m; 2025 frame.tail = m; 2026 } 2027 m->b_wptr += n; 2028 len -= n; 2029 offset += roundup(n, sc->sge.fl_align); 2030 ASSERT(offset <= rxb->buf_size); 2031 if (offset == rxb->buf_size) { 2032 offset = 0; 2033 if (++cidx == fl->cap) 2034 cidx = 0; 2035 nbuf++; 2036 } 2037 } 2038 2039 fl->cidx = cidx; 2040 fl->offset = offset; 2041 (*fl_bufs_used) += nbuf; 2042 2043 ASSERT(frame.head != NULL); 2044 return (frame.head); 2045 } 2046 2047 /* 2048 * We'll do immediate data tx for non-LSO, but only when not coalescing. We're 2049 * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes 2050 * of immediate data. 2051 */ 2052 #define IMM_LEN ( \ 2053 2 * EQ_ESIZE \ 2054 - sizeof (struct fw_eth_tx_pkt_wr) \ 2055 - sizeof (struct cpl_tx_pkt_core)) 2056 2057 /* 2058 * Returns non-zero on failure, no need to cleanup anything in that case. 2059 * 2060 * Note 1: We always try to pull up the mblk if required and return E2BIG only 2061 * if this fails. 2062 * 2063 * Note 2: We'll also pullup incoming mblk if HW_LSO is set and the first mblk 2064 * does not have the TCP header in it. 2065 */ 2066 static int 2067 get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, struct txinfo *txinfo, 2068 int sgl_only) 2069 { 2070 uint32_t flags = 0, len, n; 2071 mblk_t *m = *fp; 2072 int rc; 2073 2074 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb and dma_hdls */ 2075 2076 mac_hcksum_get(m, NULL, NULL, NULL, NULL, &flags); 2077 txinfo->flags = (flags & HCK_TX_FLAGS); 2078 2079 mac_lso_get(m, &txinfo->mss, &flags); 2080 txinfo->flags |= (flags & HW_LSO_FLAGS); 2081 2082 if (flags & HW_LSO) 2083 sgl_only = 1; /* Do not allow immediate data with LSO */ 2084 2085 /* 2086 * If checksum or segmentation offloads are requested, gather 2087 * information about the sizes and types of headers in the packet. 2088 */ 2089 if (txinfo->flags != 0) { 2090 /* 2091 * Even if this fails, the meoi_flags field will be capable of 2092 * communicating the lack of useful packet information. 2093 */ 2094 (void) mac_ether_offload_info(m, &txinfo->meoi); 2095 } else { 2096 bzero(&txinfo->meoi, sizeof (txinfo->meoi)); 2097 } 2098 2099 start: txinfo->nsegs = 0; 2100 txinfo->hdls_used = 0; 2101 txinfo->txb_used = 0; 2102 txinfo->len = 0; 2103 2104 /* total length and a rough estimate of # of segments */ 2105 n = 0; 2106 for (; m; m = m->b_cont) { 2107 len = MBLKL(m); 2108 n += (len / PAGE_SIZE) + 1; 2109 txinfo->len += len; 2110 } 2111 m = *fp; 2112 2113 if (n >= TX_SGL_SEGS || (flags & HW_LSO && MBLKL(m) < 50)) { 2114 txq->pullup_early++; 2115 m = msgpullup(*fp, -1); 2116 if (m == NULL) { 2117 txq->pullup_failed++; 2118 return (E2BIG); /* (*fp) left as it was */ 2119 } 2120 freemsg(*fp); 2121 *fp = m; 2122 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags); 2123 } 2124 2125 if (txinfo->len <= IMM_LEN && !sgl_only) 2126 return (0); /* nsegs = 0 tells caller to use imm. tx */ 2127 2128 if (txinfo->len <= txq->copy_threshold && 2129 copy_into_txb(txq, m, txinfo->len, txinfo) == 0) 2130 goto done; 2131 2132 for (; m; m = m->b_cont) { 2133 2134 len = MBLKL(m); 2135 2136 /* Use tx copy buffer if this mblk is small enough */ 2137 if (len <= txq->copy_threshold && 2138 copy_into_txb(txq, m, len, txinfo) == 0) 2139 continue; 2140 2141 /* Add DMA bindings for this mblk to the SGL */ 2142 rc = add_mblk(txq, txinfo, m, len); 2143 2144 if (rc == E2BIG || 2145 (txinfo->nsegs == TX_SGL_SEGS && m->b_cont)) { 2146 2147 txq->pullup_late++; 2148 m = msgpullup(*fp, -1); 2149 if (m != NULL) { 2150 free_txinfo_resources(txq, txinfo); 2151 freemsg(*fp); 2152 *fp = m; 2153 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags); 2154 goto start; 2155 } 2156 2157 txq->pullup_failed++; 2158 rc = E2BIG; 2159 } 2160 2161 if (rc != 0) { 2162 free_txinfo_resources(txq, txinfo); 2163 return (rc); 2164 } 2165 } 2166 2167 ASSERT(txinfo->nsegs > 0 && txinfo->nsegs <= TX_SGL_SEGS); 2168 2169 done: 2170 2171 /* 2172 * Store the # of flits required to hold this frame's SGL in nflits. An 2173 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by 2174 * multiple (len0 + len1, addr0, addr1) tuples. If addr1 is not used 2175 * then len1 must be set to 0. 2176 */ 2177 n = txinfo->nsegs - 1; 2178 txinfo->nflits = (3 * n) / 2 + (n & 1) + 2; 2179 if (n & 1) 2180 txinfo->sgl.sge[n / 2].len[1] = cpu_to_be32(0); 2181 2182 txinfo->sgl.cmd_nsge = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_DSGL) | 2183 V_ULPTX_NSGE(txinfo->nsegs)); 2184 2185 return (0); 2186 } 2187 2188 static inline int 2189 fits_in_txb(struct sge_txq *txq, int len, int *waste) 2190 { 2191 if (txq->txb_avail < len) 2192 return (0); 2193 2194 if (txq->txb_next + len <= txq->txb_size) { 2195 *waste = 0; 2196 return (1); 2197 } 2198 2199 *waste = txq->txb_size - txq->txb_next; 2200 2201 return (txq->txb_avail - *waste < len ? 0 : 1); 2202 } 2203 2204 #define TXB_CHUNK 64 2205 2206 /* 2207 * Copies the specified # of bytes into txq's tx copy buffer and updates txinfo 2208 * and txq to indicate resources used. Caller has to make sure that those many 2209 * bytes are available in the mblk chain (b_cont linked). 2210 */ 2211 static inline int 2212 copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, struct txinfo *txinfo) 2213 { 2214 int waste, n; 2215 2216 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb */ 2217 2218 if (!fits_in_txb(txq, len, &waste)) { 2219 txq->txb_full++; 2220 return (ENOMEM); 2221 } 2222 2223 if (waste != 0) { 2224 ASSERT((waste & (TXB_CHUNK - 1)) == 0); 2225 txinfo->txb_used += waste; 2226 txq->txb_avail -= waste; 2227 txq->txb_next = 0; 2228 } 2229 2230 for (n = 0; n < len; m = m->b_cont) { 2231 bcopy(m->b_rptr, txq->txb_va + txq->txb_next + n, MBLKL(m)); 2232 n += MBLKL(m); 2233 } 2234 2235 add_seg(txinfo, txq->txb_ba + txq->txb_next, len); 2236 2237 n = roundup(len, TXB_CHUNK); 2238 txinfo->txb_used += n; 2239 txq->txb_avail -= n; 2240 txq->txb_next += n; 2241 ASSERT(txq->txb_next <= txq->txb_size); 2242 if (txq->txb_next == txq->txb_size) 2243 txq->txb_next = 0; 2244 2245 return (0); 2246 } 2247 2248 static inline void 2249 add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len) 2250 { 2251 ASSERT(txinfo->nsegs < TX_SGL_SEGS); /* must have room */ 2252 2253 if (txinfo->nsegs != 0) { 2254 int idx = txinfo->nsegs - 1; 2255 txinfo->sgl.sge[idx / 2].len[idx & 1] = cpu_to_be32(len); 2256 txinfo->sgl.sge[idx / 2].addr[idx & 1] = cpu_to_be64(ba); 2257 } else { 2258 txinfo->sgl.len0 = cpu_to_be32(len); 2259 txinfo->sgl.addr0 = cpu_to_be64(ba); 2260 } 2261 txinfo->nsegs++; 2262 } 2263 2264 /* 2265 * This function cleans up any partially allocated resources when it fails so 2266 * there's nothing for the caller to clean up in that case. 2267 * 2268 * EIO indicates permanent failure. Caller should drop the frame containing 2269 * this mblk and continue. 2270 * 2271 * E2BIG indicates that the SGL length for this mblk exceeds the hardware 2272 * limit. Caller should pull up the frame before trying to send it out. 2273 * (This error means our pullup_early heuristic did not work for this frame) 2274 * 2275 * ENOMEM indicates a temporary shortage of resources (DMA handles, other DMA 2276 * resources, etc.). Caller should suspend the tx queue and wait for reclaim to 2277 * free up resources. 2278 */ 2279 static inline int 2280 add_mblk(struct sge_txq *txq, struct txinfo *txinfo, mblk_t *m, int len) 2281 { 2282 ddi_dma_handle_t dhdl; 2283 ddi_dma_cookie_t cookie; 2284 uint_t ccount = 0; 2285 int rc; 2286 2287 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate dhdls */ 2288 2289 if (txq->tx_dhdl_avail == 0) { 2290 txq->dma_hdl_failed++; 2291 return (ENOMEM); 2292 } 2293 2294 dhdl = txq->tx_dhdl[txq->tx_dhdl_pidx]; 2295 rc = ddi_dma_addr_bind_handle(dhdl, NULL, (caddr_t)m->b_rptr, len, 2296 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie, 2297 &ccount); 2298 if (rc != DDI_DMA_MAPPED) { 2299 txq->dma_map_failed++; 2300 2301 ASSERT(rc != DDI_DMA_INUSE && rc != DDI_DMA_PARTIAL_MAP); 2302 2303 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EIO); 2304 } 2305 2306 if (ccount + txinfo->nsegs > TX_SGL_SEGS) { 2307 (void) ddi_dma_unbind_handle(dhdl); 2308 return (E2BIG); 2309 } 2310 2311 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size); 2312 while (--ccount) { 2313 ddi_dma_nextcookie(dhdl, &cookie); 2314 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size); 2315 } 2316 2317 if (++txq->tx_dhdl_pidx == txq->tx_dhdl_total) 2318 txq->tx_dhdl_pidx = 0; 2319 txq->tx_dhdl_avail--; 2320 txinfo->hdls_used++; 2321 2322 return (0); 2323 } 2324 2325 /* 2326 * Releases all the txq resources used up in the specified txinfo. 2327 */ 2328 static void 2329 free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo) 2330 { 2331 int n; 2332 2333 TXQ_LOCK_ASSERT_OWNED(txq); /* dhdls, txb */ 2334 2335 n = txinfo->txb_used; 2336 if (n > 0) { 2337 txq->txb_avail += n; 2338 if (n <= txq->txb_next) 2339 txq->txb_next -= n; 2340 else { 2341 n -= txq->txb_next; 2342 txq->txb_next = txq->txb_size - n; 2343 } 2344 } 2345 2346 for (n = txinfo->hdls_used; n > 0; n--) { 2347 if (txq->tx_dhdl_pidx > 0) 2348 txq->tx_dhdl_pidx--; 2349 else 2350 txq->tx_dhdl_pidx = txq->tx_dhdl_total - 1; 2351 txq->tx_dhdl_avail++; 2352 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_pidx]); 2353 } 2354 } 2355 2356 /* 2357 * Returns 0 to indicate that m has been accepted into a coalesced tx work 2358 * request. It has either been folded into txpkts or txpkts was flushed and m 2359 * has started a new coalesced work request (as the first frame in a fresh 2360 * txpkts). 2361 * 2362 * Returns non-zero to indicate a failure - caller is responsible for 2363 * transmitting m, if there was anything in txpkts it has been flushed. 2364 */ 2365 static int 2366 add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m, 2367 struct txinfo *txinfo) 2368 { 2369 struct sge_eq *eq = &txq->eq; 2370 int can_coalesce; 2371 struct tx_sdesc *txsd; 2372 uint8_t flits; 2373 2374 TXQ_LOCK_ASSERT_OWNED(txq); 2375 2376 if (txpkts->npkt > 0) { 2377 flits = TXPKTS_PKT_HDR + txinfo->nflits; 2378 can_coalesce = (txinfo->flags & HW_LSO) == 0 && 2379 txpkts->nflits + flits <= TX_WR_FLITS && 2380 txpkts->nflits + flits <= eq->avail * 8 && 2381 txpkts->plen + txinfo->len < 65536; 2382 2383 if (can_coalesce != 0) { 2384 txpkts->tail->b_next = m; 2385 txpkts->tail = m; 2386 txpkts->npkt++; 2387 txpkts->nflits += flits; 2388 txpkts->plen += txinfo->len; 2389 2390 txsd = &txq->sdesc[eq->pidx]; 2391 txsd->txb_used += txinfo->txb_used; 2392 txsd->hdls_used += txinfo->hdls_used; 2393 2394 return (0); 2395 } 2396 2397 /* 2398 * Couldn't coalesce m into txpkts. The first order of business 2399 * is to send txpkts on its way. Then we'll revisit m. 2400 */ 2401 write_txpkts_wr(txq, txpkts); 2402 } 2403 2404 /* 2405 * Check if we can start a new coalesced tx work request with m as 2406 * the first packet in it. 2407 */ 2408 2409 ASSERT(txpkts->npkt == 0); 2410 ASSERT(txinfo->len < 65536); 2411 2412 flits = TXPKTS_WR_HDR + txinfo->nflits; 2413 can_coalesce = (txinfo->flags & HW_LSO) == 0 && 2414 flits <= eq->avail * 8 && flits <= TX_WR_FLITS; 2415 2416 if (can_coalesce == 0) 2417 return (EINVAL); 2418 2419 /* 2420 * Start a fresh coalesced tx WR with m as the first frame in it. 2421 */ 2422 txpkts->tail = m; 2423 txpkts->npkt = 1; 2424 txpkts->nflits = flits; 2425 txpkts->flitp = &eq->desc[eq->pidx].flit[2]; 2426 txpkts->plen = txinfo->len; 2427 2428 txsd = &txq->sdesc[eq->pidx]; 2429 txsd->m = m; 2430 txsd->txb_used = txinfo->txb_used; 2431 txsd->hdls_used = txinfo->hdls_used; 2432 2433 return (0); 2434 } 2435 2436 /* 2437 * Note that write_txpkts_wr can never run out of hardware descriptors (but 2438 * write_txpkt_wr can). add_to_txpkts ensures that a frame is accepted for 2439 * coalescing only if sufficient hardware descriptors are available. 2440 */ 2441 static void 2442 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts) 2443 { 2444 struct sge_eq *eq = &txq->eq; 2445 struct fw_eth_tx_pkts_wr *wr; 2446 struct tx_sdesc *txsd; 2447 uint32_t ctrl; 2448 uint16_t ndesc; 2449 2450 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */ 2451 2452 ndesc = howmany(txpkts->nflits, 8); 2453 2454 wr = (void *)&eq->desc[eq->pidx]; 2455 wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR) | 2456 V_FW_WR_IMMDLEN(0)); /* immdlen does not matter in this WR */ 2457 ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2)); 2458 if (eq->avail == ndesc) 2459 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ; 2460 wr->equiq_to_len16 = cpu_to_be32(ctrl); 2461 wr->plen = cpu_to_be16(txpkts->plen); 2462 wr->npkt = txpkts->npkt; 2463 wr->r3 = wr->type = 0; 2464 2465 /* Everything else already written */ 2466 2467 txsd = &txq->sdesc[eq->pidx]; 2468 txsd->desc_used = ndesc; 2469 2470 txq->txb_used += txsd->txb_used / TXB_CHUNK; 2471 txq->hdl_used += txsd->hdls_used; 2472 2473 ASSERT(eq->avail >= ndesc); 2474 2475 eq->pending += ndesc; 2476 eq->avail -= ndesc; 2477 eq->pidx += ndesc; 2478 if (eq->pidx >= eq->cap) 2479 eq->pidx -= eq->cap; 2480 2481 txq->txpkts_pkts += txpkts->npkt; 2482 txq->txpkts_wrs++; 2483 txpkts->npkt = 0; /* emptied */ 2484 } 2485 2486 typedef enum { 2487 COS_SUCCESS, /* ctrl flit contains proper bits for csum offload */ 2488 COS_IGNORE, /* no csum offload requested */ 2489 COS_FAIL, /* csum offload requested, but pkt data missing */ 2490 } csum_offload_status_t; 2491 /* 2492 * Build a ctrl1 flit for checksum offload in CPL_TX_PKT_XT command 2493 */ 2494 static csum_offload_status_t 2495 csum_to_ctrl(const struct txinfo *txinfo, uint32_t chip_version, 2496 uint64_t *ctrlp) 2497 { 2498 const mac_ether_offload_info_t *meoi = &txinfo->meoi; 2499 const uint32_t tx_flags = txinfo->flags; 2500 const boolean_t needs_l3_csum = (tx_flags & HW_LSO) != 0 || 2501 (tx_flags & HCK_IPV4_HDRCKSUM) != 0; 2502 const boolean_t needs_l4_csum = (tx_flags & HW_LSO) != 0 || 2503 (tx_flags & (HCK_FULLCKSUM | HCK_PARTIALCKSUM)) != 0; 2504 2505 /* 2506 * Default to disabling any checksumming both for cases where it is not 2507 * requested, but also if we cannot appropriately interrogate the 2508 * required information from the packet. 2509 */ 2510 uint64_t ctrl = F_TXPKT_L4CSUM_DIS | F_TXPKT_IPCSUM_DIS; 2511 if (!needs_l3_csum && !needs_l4_csum) { 2512 *ctrlp = ctrl; 2513 return (COS_IGNORE); 2514 } 2515 2516 if (needs_l3_csum) { 2517 /* Only IPv4 checksums are supported (for L3) */ 2518 if ((meoi->meoi_flags & MEOI_L3INFO_SET) == 0 || 2519 meoi->meoi_l3proto != ETHERTYPE_IP) { 2520 *ctrlp = ctrl; 2521 return (COS_FAIL); 2522 } 2523 ctrl &= ~F_TXPKT_IPCSUM_DIS; 2524 } 2525 2526 if (needs_l4_csum) { 2527 /* 2528 * We need at least all of the L3 header to make decisions about 2529 * the contained L4 protocol. If not all of the L4 information 2530 * is present, we will leave it to the NIC to checksum all it is 2531 * able to. 2532 */ 2533 if ((meoi->meoi_flags & MEOI_L3INFO_SET) == 0) { 2534 *ctrlp = ctrl; 2535 return (COS_FAIL); 2536 } 2537 2538 /* 2539 * Since we are parsing the packet anyways, make the checksum 2540 * decision based on the L4 protocol, rather than using the 2541 * Generic TCP/UDP checksum using start & end offsets in the 2542 * packet (like requested with PARTIALCKSUM). 2543 */ 2544 int csum_type = -1; 2545 if (meoi->meoi_l3proto == ETHERTYPE_IP && 2546 meoi->meoi_l4proto == IPPROTO_TCP) { 2547 csum_type = TX_CSUM_TCPIP; 2548 } else if (meoi->meoi_l3proto == ETHERTYPE_IPV6 && 2549 meoi->meoi_l4proto == IPPROTO_TCP) { 2550 csum_type = TX_CSUM_TCPIP6; 2551 } else if (meoi->meoi_l3proto == ETHERTYPE_IP && 2552 meoi->meoi_l4proto == IPPROTO_UDP) { 2553 csum_type = TX_CSUM_UDPIP; 2554 } else if (meoi->meoi_l3proto == ETHERTYPE_IPV6 && 2555 meoi->meoi_l4proto == IPPROTO_UDP) { 2556 csum_type = TX_CSUM_UDPIP6; 2557 } else { 2558 *ctrlp = ctrl; 2559 return (COS_FAIL); 2560 } 2561 2562 ASSERT(csum_type != -1); 2563 ctrl &= ~F_TXPKT_L4CSUM_DIS; 2564 ctrl |= V_TXPKT_CSUM_TYPE(csum_type); 2565 } 2566 2567 if ((ctrl & F_TXPKT_IPCSUM_DIS) == 0 && 2568 (ctrl & F_TXPKT_L4CSUM_DIS) != 0) { 2569 /* 2570 * If only the IPv4 checksum is requested, we need to set an 2571 * appropriate type in the command for it. 2572 */ 2573 ctrl |= V_TXPKT_CSUM_TYPE(TX_CSUM_IP); 2574 } 2575 2576 ASSERT(ctrl != (F_TXPKT_L4CSUM_DIS | F_TXPKT_IPCSUM_DIS)); 2577 2578 /* 2579 * Fill in the requisite L2/L3 header length data. 2580 * 2581 * The Ethernet header length is recorded as 'size - 14 bytes' 2582 */ 2583 const uint8_t eth_len = meoi->meoi_l2hlen - 14; 2584 if (chip_version >= CHELSIO_T6) { 2585 ctrl |= V_T6_TXPKT_ETHHDR_LEN(eth_len); 2586 } else { 2587 ctrl |= V_TXPKT_ETHHDR_LEN(eth_len); 2588 } 2589 ctrl |= V_TXPKT_IPHDR_LEN(meoi->meoi_l3hlen); 2590 2591 *ctrlp = ctrl; 2592 return (COS_SUCCESS); 2593 } 2594 2595 static int 2596 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m, 2597 struct txinfo *txinfo) 2598 { 2599 struct sge_eq *eq = &txq->eq; 2600 struct fw_eth_tx_pkt_wr *wr; 2601 struct cpl_tx_pkt_core *cpl; 2602 uint32_t ctrl; /* used in many unrelated places */ 2603 uint64_t ctrl1; 2604 int nflits, ndesc; 2605 struct tx_sdesc *txsd; 2606 caddr_t dst; 2607 const mac_ether_offload_info_t *meoi = &txinfo->meoi; 2608 2609 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */ 2610 2611 /* 2612 * Do we have enough flits to send this frame out? 2613 */ 2614 ctrl = sizeof (struct cpl_tx_pkt_core); 2615 if (txinfo->flags & HW_LSO) { 2616 nflits = TXPKT_LSO_WR_HDR; 2617 ctrl += sizeof(struct cpl_tx_pkt_lso_core); 2618 } else 2619 nflits = TXPKT_WR_HDR; 2620 if (txinfo->nsegs > 0) 2621 nflits += txinfo->nflits; 2622 else { 2623 nflits += howmany(txinfo->len, 8); 2624 ctrl += txinfo->len; 2625 } 2626 ndesc = howmany(nflits, 8); 2627 if (ndesc > eq->avail) 2628 return (ENOMEM); 2629 2630 /* Firmware work request header */ 2631 wr = (void *)&eq->desc[eq->pidx]; 2632 wr->op_immdlen = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | 2633 V_FW_WR_IMMDLEN(ctrl)); 2634 ctrl = V_FW_WR_LEN16(howmany(nflits, 2)); 2635 if (eq->avail == ndesc) 2636 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ; 2637 wr->equiq_to_len16 = cpu_to_be32(ctrl); 2638 wr->r3 = 0; 2639 2640 if (txinfo->flags & HW_LSO && 2641 (meoi->meoi_flags & MEOI_L4INFO_SET) != 0 && 2642 meoi->meoi_l4proto == IPPROTO_TCP) { 2643 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); 2644 2645 ctrl = V_LSO_OPCODE((u32)CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | 2646 F_LSO_LAST_SLICE; 2647 2648 if (meoi->meoi_l2hlen > sizeof (struct ether_header)) { 2649 /* 2650 * This presently assumes a standard VLAN header, 2651 * without support for Q-in-Q. 2652 */ 2653 ctrl |= V_LSO_ETHHDR_LEN(1); 2654 } 2655 2656 switch (meoi->meoi_l3proto) { 2657 case ETHERTYPE_IPV6: 2658 ctrl |= F_LSO_IPV6; 2659 /* FALLTHROUGH */ 2660 case ETHERTYPE_IP: 2661 ctrl |= V_LSO_IPHDR_LEN(meoi->meoi_l3hlen / 4); 2662 break; 2663 default: 2664 break; 2665 } 2666 2667 ctrl |= V_LSO_TCPHDR_LEN(meoi->meoi_l4hlen / 4); 2668 2669 lso->lso_ctrl = cpu_to_be32(ctrl); 2670 lso->ipid_ofst = cpu_to_be16(0); 2671 lso->mss = cpu_to_be16(txinfo->mss); 2672 lso->seqno_offset = cpu_to_be32(0); 2673 if (is_t4(pi->adapter->params.chip)) 2674 lso->len = cpu_to_be32(txinfo->len); 2675 else 2676 lso->len = cpu_to_be32(V_LSO_T5_XFER_SIZE(txinfo->len)); 2677 2678 cpl = (void *)(lso + 1); 2679 2680 txq->tso_wrs++; 2681 } else { 2682 cpl = (void *)(wr + 1); 2683 } 2684 2685 /* Checksum offload */ 2686 switch (csum_to_ctrl(txinfo, 2687 CHELSIO_CHIP_VERSION(pi->adapter->params.chip), &ctrl1)) { 2688 case COS_SUCCESS: 2689 txq->txcsum++; 2690 break; 2691 case COS_FAIL: 2692 /* 2693 * Packet will be going out with checksums which are probably 2694 * wrong but there is little we can do now. 2695 */ 2696 txq->csum_failed++; 2697 break; 2698 default: 2699 break; 2700 } 2701 2702 /* CPL header */ 2703 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | 2704 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf)); 2705 cpl->pack = 0; 2706 cpl->len = cpu_to_be16(txinfo->len); 2707 cpl->ctrl1 = cpu_to_be64(ctrl1); 2708 2709 /* Software descriptor */ 2710 txsd = &txq->sdesc[eq->pidx]; 2711 txsd->m = m; 2712 txsd->txb_used = txinfo->txb_used; 2713 txsd->hdls_used = txinfo->hdls_used; 2714 /* LINTED: E_ASSIGN_NARROW_CONV */ 2715 txsd->desc_used = ndesc; 2716 2717 txq->txb_used += txinfo->txb_used / TXB_CHUNK; 2718 txq->hdl_used += txinfo->hdls_used; 2719 2720 eq->pending += ndesc; 2721 eq->avail -= ndesc; 2722 eq->pidx += ndesc; 2723 if (eq->pidx >= eq->cap) 2724 eq->pidx -= eq->cap; 2725 2726 /* SGL */ 2727 dst = (void *)(cpl + 1); 2728 if (txinfo->nsegs > 0) { 2729 txq->sgl_wrs++; 2730 copy_to_txd(eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8); 2731 2732 /* Need to zero-pad to a 16 byte boundary if not on one */ 2733 if ((uintptr_t)dst & 0xf) 2734 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 2735 *(uint64_t *)dst = 0; 2736 2737 } else { 2738 txq->imm_wrs++; 2739 #ifdef DEBUG 2740 ctrl = txinfo->len; 2741 #endif 2742 for (; m; m = m->b_cont) { 2743 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m)); 2744 #ifdef DEBUG 2745 ctrl -= MBLKL(m); 2746 #endif 2747 } 2748 ASSERT(ctrl == 0); 2749 } 2750 2751 txq->txpkt_wrs++; 2752 return (0); 2753 } 2754 2755 static inline void 2756 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq, 2757 struct txpkts *txpkts, struct txinfo *txinfo) 2758 { 2759 struct ulp_txpkt *ulpmc; 2760 struct ulptx_idata *ulpsc; 2761 struct cpl_tx_pkt_core *cpl; 2762 uintptr_t flitp, start, end; 2763 uint64_t ctrl; 2764 caddr_t dst; 2765 2766 ASSERT(txpkts->npkt > 0); 2767 2768 start = (uintptr_t)txq->eq.desc; 2769 end = (uintptr_t)txq->eq.spg; 2770 2771 /* Checksum offload */ 2772 switch (csum_to_ctrl(txinfo, 2773 CHELSIO_CHIP_VERSION(pi->adapter->params.chip), &ctrl)) { 2774 case COS_SUCCESS: 2775 txq->txcsum++; 2776 break; 2777 case COS_FAIL: 2778 /* 2779 * Packet will be going out with checksums which are probably 2780 * wrong but there is little we can do now. 2781 */ 2782 txq->csum_failed++; 2783 break; 2784 default: 2785 break; 2786 } 2787 2788 /* 2789 * The previous packet's SGL must have ended at a 16 byte boundary (this 2790 * is required by the firmware/hardware). It follows that flitp cannot 2791 * wrap around between the ULPTX master command and ULPTX subcommand (8 2792 * bytes each), and that it can not wrap around in the middle of the 2793 * cpl_tx_pkt_core either. 2794 */ 2795 flitp = (uintptr_t)txpkts->flitp; 2796 ASSERT((flitp & 0xf) == 0); 2797 2798 /* ULP master command */ 2799 ulpmc = (void *)flitp; 2800 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); 2801 ulpmc->len = htonl(howmany(sizeof (*ulpmc) + sizeof (*ulpsc) + 2802 sizeof (*cpl) + 8 * txinfo->nflits, 16)); 2803 2804 /* ULP subcommand */ 2805 ulpsc = (void *)(ulpmc + 1); 2806 ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) | 2807 F_ULP_TX_SC_MORE); 2808 ulpsc->len = cpu_to_be32(sizeof (struct cpl_tx_pkt_core)); 2809 2810 flitp += sizeof (*ulpmc) + sizeof (*ulpsc); 2811 if (flitp == end) 2812 flitp = start; 2813 2814 /* CPL_TX_PKT_XT */ 2815 cpl = (void *)flitp; 2816 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | 2817 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf)); 2818 cpl->pack = 0; 2819 cpl->len = cpu_to_be16(txinfo->len); 2820 cpl->ctrl1 = cpu_to_be64(ctrl); 2821 2822 flitp += sizeof (*cpl); 2823 if (flitp == end) 2824 flitp = start; 2825 2826 /* SGL for this frame */ 2827 dst = (caddr_t)flitp; 2828 copy_to_txd(&txq->eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8); 2829 flitp = (uintptr_t)dst; 2830 2831 /* Zero pad and advance to a 16 byte boundary if not already at one. */ 2832 if (flitp & 0xf) { 2833 2834 /* no matter what, flitp should be on an 8 byte boundary */ 2835 ASSERT((flitp & 0x7) == 0); 2836 2837 *(uint64_t *)flitp = 0; 2838 flitp += sizeof (uint64_t); 2839 txpkts->nflits++; 2840 } 2841 2842 if (flitp == end) 2843 flitp = start; 2844 2845 txpkts->flitp = (void *)flitp; 2846 } 2847 2848 static inline void 2849 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) 2850 { 2851 if ((uintptr_t)(*to) + len <= (uintptr_t)eq->spg) { 2852 bcopy(from, *to, len); 2853 (*to) += len; 2854 } else { 2855 int portion = (uintptr_t)eq->spg - (uintptr_t)(*to); 2856 2857 bcopy(from, *to, portion); 2858 from += portion; 2859 portion = len - portion; /* remaining */ 2860 bcopy(from, (void *)eq->desc, portion); 2861 (*to) = (caddr_t)eq->desc + portion; 2862 } 2863 } 2864 2865 static inline void 2866 ring_tx_db(struct adapter *sc, struct sge_eq *eq) 2867 { 2868 int val, db_mode; 2869 u_int db = eq->doorbells; 2870 2871 if (eq->pending > 1) 2872 db &= ~DOORBELL_WCWR; 2873 2874 if (eq->pending > eq->pidx) { 2875 int offset = eq->cap - (eq->pending - eq->pidx); 2876 2877 /* pidx has wrapped around since last doorbell */ 2878 2879 (void) ddi_dma_sync(eq->desc_dhdl, 2880 offset * sizeof (struct tx_desc), 0, 2881 DDI_DMA_SYNC_FORDEV); 2882 (void) ddi_dma_sync(eq->desc_dhdl, 2883 0, eq->pidx * sizeof (struct tx_desc), 2884 DDI_DMA_SYNC_FORDEV); 2885 } else if (eq->pending > 0) { 2886 (void) ddi_dma_sync(eq->desc_dhdl, 2887 (eq->pidx - eq->pending) * sizeof (struct tx_desc), 2888 eq->pending * sizeof (struct tx_desc), 2889 DDI_DMA_SYNC_FORDEV); 2890 } 2891 2892 membar_producer(); 2893 2894 if (is_t4(sc->params.chip)) 2895 val = V_PIDX(eq->pending); 2896 else 2897 val = V_PIDX_T5(eq->pending); 2898 2899 db_mode = (1 << (ffs(db) - 1)); 2900 switch (db_mode) { 2901 case DOORBELL_UDB: 2902 *eq->udb = LE_32(V_QID(eq->udb_qid) | val); 2903 break; 2904 2905 case DOORBELL_WCWR: 2906 { 2907 volatile uint64_t *dst, *src; 2908 int i; 2909 /* 2910 * Queues whose 128B doorbell segment fits in 2911 * the page do not use relative qid 2912 * (udb_qid is always 0). Only queues with 2913 * doorbell segments can do WCWR. 2914 */ 2915 ASSERT(eq->udb_qid == 0 && eq->pending == 1); 2916 2917 dst = (volatile void *)((uintptr_t)eq->udb + 2918 UDBS_WR_OFFSET - UDBS_DB_OFFSET); 2919 i = eq->pidx ? eq->pidx - 1 : eq->cap - 1; 2920 src = (void *)&eq->desc[i]; 2921 while (src != (void *)&eq->desc[i + 1]) 2922 *dst++ = *src++; 2923 membar_producer(); 2924 break; 2925 } 2926 2927 case DOORBELL_UDBWC: 2928 *eq->udb = LE_32(V_QID(eq->udb_qid) | val); 2929 membar_producer(); 2930 break; 2931 2932 case DOORBELL_KDB: 2933 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), 2934 V_QID(eq->cntxt_id) | val); 2935 break; 2936 } 2937 2938 eq->pending = 0; 2939 } 2940 2941 static int 2942 reclaim_tx_descs(struct sge_txq *txq, int howmany) 2943 { 2944 struct tx_sdesc *txsd; 2945 uint_t cidx, can_reclaim, reclaimed, txb_freed, hdls_freed; 2946 struct sge_eq *eq = &txq->eq; 2947 2948 EQ_LOCK_ASSERT_OWNED(eq); 2949 2950 cidx = eq->spg->cidx; /* stable snapshot */ 2951 cidx = be16_to_cpu(cidx); 2952 2953 if (cidx >= eq->cidx) 2954 can_reclaim = cidx - eq->cidx; 2955 else 2956 can_reclaim = cidx + eq->cap - eq->cidx; 2957 2958 if (can_reclaim == 0) 2959 return (0); 2960 2961 txb_freed = hdls_freed = reclaimed = 0; 2962 do { 2963 int ndesc; 2964 2965 txsd = &txq->sdesc[eq->cidx]; 2966 ndesc = txsd->desc_used; 2967 2968 /* Firmware doesn't return "partial" credits. */ 2969 ASSERT(can_reclaim >= ndesc); 2970 2971 /* 2972 * We always keep mblk around, even for immediate data. If mblk 2973 * is NULL, this has to be the software descriptor for a credit 2974 * flush work request. 2975 */ 2976 if (txsd->m != NULL) 2977 freemsgchain(txsd->m); 2978 #ifdef DEBUG 2979 else { 2980 ASSERT(txsd->txb_used == 0); 2981 ASSERT(txsd->hdls_used == 0); 2982 ASSERT(ndesc == 1); 2983 } 2984 #endif 2985 2986 txb_freed += txsd->txb_used; 2987 hdls_freed += txsd->hdls_used; 2988 reclaimed += ndesc; 2989 2990 eq->cidx += ndesc; 2991 if (eq->cidx >= eq->cap) 2992 eq->cidx -= eq->cap; 2993 2994 can_reclaim -= ndesc; 2995 2996 } while (can_reclaim && reclaimed < howmany); 2997 2998 eq->avail += reclaimed; 2999 ASSERT(eq->avail < eq->cap); /* avail tops out at (cap - 1) */ 3000 3001 txq->txb_avail += txb_freed; 3002 3003 txq->tx_dhdl_avail += hdls_freed; 3004 ASSERT(txq->tx_dhdl_avail <= txq->tx_dhdl_total); 3005 for (; hdls_freed; hdls_freed--) { 3006 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_cidx]); 3007 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total) 3008 txq->tx_dhdl_cidx = 0; 3009 } 3010 3011 return (reclaimed); 3012 } 3013 3014 static void 3015 write_txqflush_wr(struct sge_txq *txq) 3016 { 3017 struct sge_eq *eq = &txq->eq; 3018 struct fw_eq_flush_wr *wr; 3019 struct tx_sdesc *txsd; 3020 3021 EQ_LOCK_ASSERT_OWNED(eq); 3022 ASSERT(eq->avail > 0); 3023 3024 wr = (void *)&eq->desc[eq->pidx]; 3025 bzero(wr, sizeof (*wr)); 3026 wr->opcode = FW_EQ_FLUSH_WR; 3027 wr->equiq_to_len16 = cpu_to_be32(V_FW_WR_LEN16(sizeof (*wr) / 16) | 3028 F_FW_WR_EQUEQ | F_FW_WR_EQUIQ); 3029 3030 txsd = &txq->sdesc[eq->pidx]; 3031 txsd->m = NULL; 3032 txsd->txb_used = 0; 3033 txsd->hdls_used = 0; 3034 txsd->desc_used = 1; 3035 3036 eq->pending++; 3037 eq->avail--; 3038 if (++eq->pidx == eq->cap) 3039 eq->pidx = 0; 3040 } 3041 3042 static int 3043 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m) 3044 { 3045 bool csum_ok; 3046 uint16_t err_vec; 3047 struct sge_rxq *rxq = (void *)iq; 3048 struct mblk_pair chain = {0}; 3049 struct adapter *sc = iq->adapter; 3050 const struct cpl_rx_pkt *cpl = (const void *)(rss + 1); 3051 3052 iq->intr_next = iq->intr_params; 3053 3054 m->b_rptr += sc->sge.pktshift; 3055 3056 /* Compressed error vector is enabled for T6 only */ 3057 if (sc->params.tp.rx_pkt_encap) 3058 /* It is enabled only in T6 config file */ 3059 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec)); 3060 else 3061 err_vec = ntohs(cpl->err_vec); 3062 3063 csum_ok = cpl->csum_calc && !err_vec; 3064 /* TODO: what about cpl->ip_frag? */ 3065 if (csum_ok && !cpl->ip_frag) { 3066 mac_hcksum_set(m, 0, 0, 0, 0xffff, 3067 HCK_FULLCKSUM_OK | HCK_FULLCKSUM | 3068 HCK_IPV4_HDRCKSUM_OK); 3069 rxq->rxcsum++; 3070 } 3071 3072 /* Add to the chain that we'll send up */ 3073 if (chain.head != NULL) 3074 chain.tail->b_next = m; 3075 else 3076 chain.head = m; 3077 chain.tail = m; 3078 3079 t4_mac_rx(rxq->port, rxq, chain.head); 3080 3081 rxq->rxpkts++; 3082 rxq->rxbytes += be16_to_cpu(cpl->len); 3083 return (0); 3084 } 3085 3086 #define FL_HW_IDX(idx) ((idx) >> 3) 3087 3088 static inline void 3089 ring_fl_db(struct adapter *sc, struct sge_fl *fl) 3090 { 3091 int desc_start, desc_last, ndesc; 3092 uint32_t v = sc->params.arch.sge_fl_db ; 3093 3094 ndesc = FL_HW_IDX(fl->pending); 3095 3096 /* Hold back one credit if pidx = cidx */ 3097 if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx)) 3098 ndesc--; 3099 3100 /* 3101 * There are chances of ndesc modified above (to avoid pidx = cidx). 3102 * If there is nothing to post, return. 3103 */ 3104 if (ndesc <= 0) 3105 return; 3106 3107 desc_last = FL_HW_IDX(fl->pidx); 3108 3109 if (fl->pidx < fl->pending) { 3110 /* There was a wrap */ 3111 desc_start = FL_HW_IDX(fl->pidx + fl->cap - fl->pending); 3112 3113 /* From desc_start to the end of list */ 3114 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 0, 3115 DDI_DMA_SYNC_FORDEV); 3116 3117 /* From start of list to the desc_last */ 3118 if (desc_last != 0) 3119 (void) ddi_dma_sync(fl->dhdl, 0, desc_last * 3120 RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV); 3121 } else { 3122 /* There was no wrap, sync from start_desc to last_desc */ 3123 desc_start = FL_HW_IDX(fl->pidx - fl->pending); 3124 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 3125 ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV); 3126 } 3127 3128 if (is_t4(sc->params.chip)) 3129 v |= V_PIDX(ndesc); 3130 else 3131 v |= V_PIDX_T5(ndesc); 3132 v |= V_QID(fl->cntxt_id) | V_PIDX(ndesc); 3133 3134 membar_producer(); 3135 3136 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v); 3137 3138 /* 3139 * Update pending count: 3140 * Deduct the number of descriptors posted 3141 */ 3142 fl->pending -= ndesc * 8; 3143 } 3144 3145 static void 3146 tx_reclaim_task(void *arg) 3147 { 3148 struct sge_txq *txq = arg; 3149 3150 TXQ_LOCK(txq); 3151 reclaim_tx_descs(txq, txq->eq.qsize); 3152 TXQ_UNLOCK(txq); 3153 } 3154 3155 /* ARGSUSED */ 3156 static int 3157 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss, 3158 mblk_t *m) 3159 { 3160 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1); 3161 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid)); 3162 struct adapter *sc = iq->adapter; 3163 struct sge *s = &sc->sge; 3164 struct sge_eq *eq; 3165 struct sge_txq *txq; 3166 3167 txq = (void *)s->eqmap[qid - s->eq_start]; 3168 eq = &txq->eq; 3169 txq->qflush++; 3170 t4_mac_tx_update(txq->port, txq); 3171 3172 ddi_taskq_dispatch(sc->tq[eq->tx_chan], tx_reclaim_task, 3173 (void *)txq, DDI_NOSLEEP); 3174 3175 return (0); 3176 } 3177 3178 static int 3179 handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m) 3180 { 3181 struct adapter *sc = iq->adapter; 3182 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1); 3183 3184 ASSERT(m == NULL); 3185 3186 if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) { 3187 const struct rss_header *rss2; 3188 3189 rss2 = (const struct rss_header *)&cpl->data[0]; 3190 return (sc->cpl_handler[rss2->opcode](iq, rss2, m)); 3191 } 3192 return (sc->fw_msg_handler[cpl->type](sc, &cpl->data[0])); 3193 } 3194 3195 int 3196 t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count, 3197 int flags) 3198 { 3199 int i, rc; 3200 3201 txmaps->map_total = count; 3202 txmaps->map_avail = txmaps->map_cidx = txmaps->map_pidx = 0; 3203 3204 txmaps->map = kmem_zalloc(sizeof (ddi_dma_handle_t) * 3205 txmaps->map_total, flags); 3206 3207 for (i = 0; i < count; i++) { 3208 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx, 3209 DDI_DMA_SLEEP, 0, &txmaps->map[i]); 3210 if (rc != DDI_SUCCESS) { 3211 cxgb_printf(sc->dip, CE_WARN, 3212 "%s: failed to allocate DMA handle (%d)", 3213 __func__, rc); 3214 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 3215 } 3216 txmaps->map_avail++; 3217 } 3218 3219 return (0); 3220 } 3221 3222 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG) 3223 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR) 3224 #define KS_U_SET(x, y) kstatp->x.value.ul = (y) 3225 #define KS_U_FROM(x, y) kstatp->x.value.ul = (y)->x 3226 #define KS_C_SET(x, ...) \ 3227 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__) 3228 3229 /* 3230 * cxgbe:X:config 3231 */ 3232 struct cxgbe_port_config_kstats { 3233 kstat_named_t idx; 3234 kstat_named_t nrxq; 3235 kstat_named_t ntxq; 3236 kstat_named_t first_rxq; 3237 kstat_named_t first_txq; 3238 kstat_named_t controller; 3239 kstat_named_t factory_mac_address; 3240 }; 3241 3242 /* 3243 * cxgbe:X:info 3244 */ 3245 struct cxgbe_port_info_kstats { 3246 kstat_named_t transceiver; 3247 kstat_named_t rx_ovflow0; 3248 kstat_named_t rx_ovflow1; 3249 kstat_named_t rx_ovflow2; 3250 kstat_named_t rx_ovflow3; 3251 kstat_named_t rx_trunc0; 3252 kstat_named_t rx_trunc1; 3253 kstat_named_t rx_trunc2; 3254 kstat_named_t rx_trunc3; 3255 kstat_named_t tx_pause; 3256 kstat_named_t rx_pause; 3257 }; 3258 3259 static kstat_t * 3260 setup_port_config_kstats(struct port_info *pi) 3261 { 3262 kstat_t *ksp; 3263 struct cxgbe_port_config_kstats *kstatp; 3264 int ndata; 3265 dev_info_t *pdip = ddi_get_parent(pi->dip); 3266 uint8_t *ma = &pi->hw_addr[0]; 3267 3268 ndata = sizeof (struct cxgbe_port_config_kstats) / 3269 sizeof (kstat_named_t); 3270 3271 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "config", 3272 "net", KSTAT_TYPE_NAMED, ndata, 0); 3273 if (ksp == NULL) { 3274 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats."); 3275 return (NULL); 3276 } 3277 3278 kstatp = (struct cxgbe_port_config_kstats *)ksp->ks_data; 3279 3280 KS_UINIT(idx); 3281 KS_UINIT(nrxq); 3282 KS_UINIT(ntxq); 3283 KS_UINIT(first_rxq); 3284 KS_UINIT(first_txq); 3285 KS_CINIT(controller); 3286 KS_CINIT(factory_mac_address); 3287 3288 KS_U_SET(idx, pi->port_id); 3289 KS_U_SET(nrxq, pi->nrxq); 3290 KS_U_SET(ntxq, pi->ntxq); 3291 KS_U_SET(first_rxq, pi->first_rxq); 3292 KS_U_SET(first_txq, pi->first_txq); 3293 KS_C_SET(controller, "%s%d", ddi_driver_name(pdip), 3294 ddi_get_instance(pdip)); 3295 KS_C_SET(factory_mac_address, "%02X%02X%02X%02X%02X%02X", 3296 ma[0], ma[1], ma[2], ma[3], ma[4], ma[5]); 3297 3298 /* Do NOT set ksp->ks_update. These kstats do not change. */ 3299 3300 /* Install the kstat */ 3301 ksp->ks_private = (void *)pi; 3302 kstat_install(ksp); 3303 3304 return (ksp); 3305 } 3306 3307 static kstat_t * 3308 setup_port_info_kstats(struct port_info *pi) 3309 { 3310 kstat_t *ksp; 3311 struct cxgbe_port_info_kstats *kstatp; 3312 int ndata; 3313 3314 ndata = sizeof (struct cxgbe_port_info_kstats) / sizeof (kstat_named_t); 3315 3316 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "info", 3317 "net", KSTAT_TYPE_NAMED, ndata, 0); 3318 if (ksp == NULL) { 3319 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats."); 3320 return (NULL); 3321 } 3322 3323 kstatp = (struct cxgbe_port_info_kstats *)ksp->ks_data; 3324 3325 KS_CINIT(transceiver); 3326 KS_UINIT(rx_ovflow0); 3327 KS_UINIT(rx_ovflow1); 3328 KS_UINIT(rx_ovflow2); 3329 KS_UINIT(rx_ovflow3); 3330 KS_UINIT(rx_trunc0); 3331 KS_UINIT(rx_trunc1); 3332 KS_UINIT(rx_trunc2); 3333 KS_UINIT(rx_trunc3); 3334 KS_UINIT(tx_pause); 3335 KS_UINIT(rx_pause); 3336 3337 /* Install the kstat */ 3338 ksp->ks_update = update_port_info_kstats; 3339 ksp->ks_private = (void *)pi; 3340 kstat_install(ksp); 3341 3342 return (ksp); 3343 } 3344 3345 static int 3346 update_port_info_kstats(kstat_t *ksp, int rw) 3347 { 3348 struct cxgbe_port_info_kstats *kstatp = 3349 (struct cxgbe_port_info_kstats *)ksp->ks_data; 3350 struct port_info *pi = ksp->ks_private; 3351 static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX", 3352 "active TWINAX", "LRM" }; 3353 uint32_t bgmap; 3354 3355 if (rw == KSTAT_WRITE) 3356 return (0); 3357 3358 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 3359 KS_C_SET(transceiver, "unplugged"); 3360 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 3361 KS_C_SET(transceiver, "unknown"); 3362 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 3363 KS_C_SET(transceiver, "unsupported"); 3364 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str)) 3365 KS_C_SET(transceiver, "%s", mod_str[pi->mod_type]); 3366 else 3367 KS_C_SET(transceiver, "type %d", pi->mod_type); 3368 3369 #define GET_STAT(name) t4_read_reg64(pi->adapter, \ 3370 PORT_REG(pi->port_id, A_MPS_PORT_STAT_##name##_L)) 3371 #define GET_STAT_COM(name) t4_read_reg64(pi->adapter, \ 3372 A_MPS_STAT_##name##_L) 3373 3374 bgmap = G_NUMPORTS(t4_read_reg(pi->adapter, A_MPS_CMN_CTL)); 3375 if (bgmap == 0) 3376 bgmap = (pi->port_id == 0) ? 0xf : 0; 3377 else if (bgmap == 1) 3378 bgmap = (pi->port_id < 2) ? (3 << (2 * pi->port_id)) : 0; 3379 else 3380 bgmap = 1; 3381 3382 KS_U_SET(rx_ovflow0, (bgmap & 1) ? 3383 GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0); 3384 KS_U_SET(rx_ovflow1, (bgmap & 2) ? 3385 GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0); 3386 KS_U_SET(rx_ovflow2, (bgmap & 4) ? 3387 GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0); 3388 KS_U_SET(rx_ovflow3, (bgmap & 8) ? 3389 GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0); 3390 KS_U_SET(rx_trunc0, (bgmap & 1) ? 3391 GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0); 3392 KS_U_SET(rx_trunc1, (bgmap & 2) ? 3393 GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0); 3394 KS_U_SET(rx_trunc2, (bgmap & 4) ? 3395 GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0); 3396 KS_U_SET(rx_trunc3, (bgmap & 8) ? 3397 GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0); 3398 3399 KS_U_SET(tx_pause, GET_STAT(TX_PORT_PAUSE)); 3400 KS_U_SET(rx_pause, GET_STAT(RX_PORT_PAUSE)); 3401 3402 return (0); 3403 3404 } 3405 3406 /* 3407 * cxgbe:X:rxqY 3408 */ 3409 struct rxq_kstats { 3410 kstat_named_t rxcsum; 3411 kstat_named_t rxpkts; 3412 kstat_named_t rxbytes; 3413 kstat_named_t nomem; 3414 }; 3415 3416 static kstat_t * 3417 setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, int idx) 3418 { 3419 struct kstat *ksp; 3420 struct rxq_kstats *kstatp; 3421 int ndata; 3422 char str[16]; 3423 3424 ndata = sizeof (struct rxq_kstats) / sizeof (kstat_named_t); 3425 (void) snprintf(str, sizeof (str), "rxq%u", idx); 3426 3427 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "rxq", 3428 KSTAT_TYPE_NAMED, ndata, 0); 3429 if (ksp == NULL) { 3430 cxgb_printf(pi->dip, CE_WARN, 3431 "%s: failed to initialize rxq kstats for queue %d.", 3432 __func__, idx); 3433 return (NULL); 3434 } 3435 3436 kstatp = (struct rxq_kstats *)ksp->ks_data; 3437 3438 KS_UINIT(rxcsum); 3439 KS_UINIT(rxpkts); 3440 KS_UINIT(rxbytes); 3441 KS_UINIT(nomem); 3442 3443 ksp->ks_update = update_rxq_kstats; 3444 ksp->ks_private = (void *)rxq; 3445 kstat_install(ksp); 3446 3447 return (ksp); 3448 } 3449 3450 static int 3451 update_rxq_kstats(kstat_t *ksp, int rw) 3452 { 3453 struct rxq_kstats *kstatp = (struct rxq_kstats *)ksp->ks_data; 3454 struct sge_rxq *rxq = ksp->ks_private; 3455 3456 if (rw == KSTAT_WRITE) 3457 return (0); 3458 3459 KS_U_FROM(rxcsum, rxq); 3460 KS_U_FROM(rxpkts, rxq); 3461 KS_U_FROM(rxbytes, rxq); 3462 KS_U_FROM(nomem, rxq); 3463 3464 return (0); 3465 } 3466 3467 /* 3468 * cxgbe:X:txqY 3469 */ 3470 struct txq_kstats { 3471 kstat_named_t txcsum; 3472 kstat_named_t tso_wrs; 3473 kstat_named_t imm_wrs; 3474 kstat_named_t sgl_wrs; 3475 kstat_named_t txpkt_wrs; 3476 kstat_named_t txpkts_wrs; 3477 kstat_named_t txpkts_pkts; 3478 kstat_named_t txb_used; 3479 kstat_named_t hdl_used; 3480 kstat_named_t txb_full; 3481 kstat_named_t dma_hdl_failed; 3482 kstat_named_t dma_map_failed; 3483 kstat_named_t qfull; 3484 kstat_named_t qflush; 3485 kstat_named_t pullup_early; 3486 kstat_named_t pullup_late; 3487 kstat_named_t pullup_failed; 3488 kstat_named_t csum_failed; 3489 }; 3490 3491 static kstat_t * 3492 setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, int idx) 3493 { 3494 struct kstat *ksp; 3495 struct txq_kstats *kstatp; 3496 int ndata; 3497 char str[16]; 3498 3499 ndata = sizeof (struct txq_kstats) / sizeof (kstat_named_t); 3500 (void) snprintf(str, sizeof (str), "txq%u", idx); 3501 3502 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "txq", 3503 KSTAT_TYPE_NAMED, ndata, 0); 3504 if (ksp == NULL) { 3505 cxgb_printf(pi->dip, CE_WARN, 3506 "%s: failed to initialize txq kstats for queue %d.", 3507 __func__, idx); 3508 return (NULL); 3509 } 3510 3511 kstatp = (struct txq_kstats *)ksp->ks_data; 3512 3513 KS_UINIT(txcsum); 3514 KS_UINIT(tso_wrs); 3515 KS_UINIT(imm_wrs); 3516 KS_UINIT(sgl_wrs); 3517 KS_UINIT(txpkt_wrs); 3518 KS_UINIT(txpkts_wrs); 3519 KS_UINIT(txpkts_pkts); 3520 KS_UINIT(txb_used); 3521 KS_UINIT(hdl_used); 3522 KS_UINIT(txb_full); 3523 KS_UINIT(dma_hdl_failed); 3524 KS_UINIT(dma_map_failed); 3525 KS_UINIT(qfull); 3526 KS_UINIT(qflush); 3527 KS_UINIT(pullup_early); 3528 KS_UINIT(pullup_late); 3529 KS_UINIT(pullup_failed); 3530 KS_UINIT(csum_failed); 3531 3532 ksp->ks_update = update_txq_kstats; 3533 ksp->ks_private = (void *)txq; 3534 kstat_install(ksp); 3535 3536 return (ksp); 3537 } 3538 3539 static int 3540 update_txq_kstats(kstat_t *ksp, int rw) 3541 { 3542 struct txq_kstats *kstatp = (struct txq_kstats *)ksp->ks_data; 3543 struct sge_txq *txq = ksp->ks_private; 3544 3545 if (rw == KSTAT_WRITE) 3546 return (0); 3547 3548 KS_U_FROM(txcsum, txq); 3549 KS_U_FROM(tso_wrs, txq); 3550 KS_U_FROM(imm_wrs, txq); 3551 KS_U_FROM(sgl_wrs, txq); 3552 KS_U_FROM(txpkt_wrs, txq); 3553 KS_U_FROM(txpkts_wrs, txq); 3554 KS_U_FROM(txpkts_pkts, txq); 3555 KS_U_FROM(txb_used, txq); 3556 KS_U_FROM(hdl_used, txq); 3557 KS_U_FROM(txb_full, txq); 3558 KS_U_FROM(dma_hdl_failed, txq); 3559 KS_U_FROM(dma_map_failed, txq); 3560 KS_U_FROM(qfull, txq); 3561 KS_U_FROM(qflush, txq); 3562 KS_U_FROM(pullup_early, txq); 3563 KS_U_FROM(pullup_late, txq); 3564 KS_U_FROM(pullup_failed, txq); 3565 KS_U_FROM(csum_failed, txq); 3566 3567 return (0); 3568 } 3569