1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * This file is part of the Chelsio T4 support code. 14 * 15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved. 16 * 17 * This program is distributed in the hope that it will be useful, but WITHOUT 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this 20 * release for licensing terms and conditions. 21 */ 22 23 /* 24 * Copyright 2025 Oxide Computer Company 25 */ 26 27 #include <sys/ddi.h> 28 #include <sys/sunddi.h> 29 #include <sys/sunndi.h> 30 #include <sys/atomic.h> 31 #include <sys/dlpi.h> 32 #include <sys/pattr.h> 33 #include <sys/strsubr.h> 34 #include <sys/stream.h> 35 #include <sys/strsun.h> 36 #include <inet/ip.h> 37 #include <inet/tcp.h> 38 39 #include "version.h" 40 #include "common/common.h" 41 #include "common/t4_msg.h" 42 #include "common/t4_regs.h" 43 #include "common/t4_regs_values.h" 44 45 /* TODO: Tune. */ 46 int rx_buf_size = 8192; 47 int tx_copy_threshold = 256; 48 uint16_t rx_copy_threshold = 256; 49 50 /* Used to track coalesced tx work request */ 51 struct txpkts { 52 mblk_t *tail; /* head is in the software descriptor */ 53 uint64_t *flitp; /* ptr to flit where next pkt should start */ 54 uint8_t npkt; /* # of packets in this work request */ 55 uint8_t nflits; /* # of flits used by this work request */ 56 uint16_t plen; /* total payload (sum of all packets) */ 57 }; 58 59 /* All information needed to tx a frame */ 60 struct txinfo { 61 uint32_t len; /* Total length of frame */ 62 uint32_t flags; /* Checksum and LSO flags */ 63 uint32_t mss; /* MSS for LSO */ 64 uint8_t nsegs; /* # of segments in the SGL, 0 means imm. tx */ 65 uint8_t nflits; /* # of flits needed for the SGL */ 66 uint8_t hdls_used; /* # of DMA handles used */ 67 uint32_t txb_used; /* txb_space used */ 68 mac_ether_offload_info_t meoi; /* pkt hdr info for offloads */ 69 struct ulptx_sgl sgl __attribute__((aligned(8))); 70 struct ulptx_sge_pair reserved[TX_SGL_SEGS / 2]; 71 }; 72 73 struct mblk_pair { 74 mblk_t *head, *tail; 75 }; 76 77 struct rxbuf { 78 kmem_cache_t *cache; /* the kmem_cache this rxb came from */ 79 ddi_dma_handle_t dhdl; 80 ddi_acc_handle_t ahdl; 81 caddr_t va; /* KVA of buffer */ 82 uint64_t ba; /* bus address of buffer */ 83 frtn_t freefunc; 84 uint_t buf_size; 85 volatile uint_t ref_cnt; 86 }; 87 88 static int service_iq(struct sge_iq *iq, int budget); 89 static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, 90 int8_t pktc_idx, int qsize, uint8_t esize); 91 static inline void init_fl(struct sge_fl *fl, uint16_t qsize); 92 static inline void init_eq(struct adapter *sc, struct sge_eq *eq, 93 uint16_t eqtype, uint16_t qsize, uint8_t tx_chan, uint16_t iqid); 94 static int alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, 95 struct sge_fl *fl, int intr_idx, int cong); 96 static int free_iq_fl(struct port_info *pi, struct sge_iq *iq, 97 struct sge_fl *fl); 98 static int alloc_fwq(struct adapter *sc); 99 static int free_fwq(struct adapter *sc); 100 static int alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, 101 int i); 102 static int free_rxq(struct port_info *pi, struct sge_rxq *rxq); 103 static int ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq); 104 static int eth_eq_alloc(struct adapter *sc, struct port_info *pi, 105 struct sge_eq *eq); 106 static int alloc_eq(struct adapter *sc, struct port_info *pi, 107 struct sge_eq *eq); 108 static int free_eq(struct adapter *sc, struct sge_eq *eq); 109 static int alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx); 110 static int free_txq(struct port_info *pi, struct sge_txq *txq); 111 static int alloc_dma_memory(struct adapter *sc, size_t len, int flags, 112 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr, 113 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 114 caddr_t *pva); 115 static int free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl); 116 static int alloc_desc_ring(struct adapter *sc, size_t len, int rw, 117 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 118 caddr_t *pva); 119 static int free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl); 120 static int alloc_tx_copybuffer(struct adapter *sc, size_t len, 121 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 122 caddr_t *pva); 123 static inline bool is_new_response(const struct sge_iq *iq, 124 struct rsp_ctrl **ctrl); 125 static inline void iq_next(struct sge_iq *iq); 126 static int refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs); 127 static void refill_sfl(void *arg); 128 static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl); 129 static void free_fl_bufs(struct sge_fl *fl); 130 static mblk_t *get_fl_payload(struct adapter *sc, struct sge_fl *fl, 131 uint32_t len_newbuf, int *fl_bufs_used); 132 static int get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, 133 struct txinfo *txinfo, int sgl_only); 134 static inline int fits_in_txb(struct sge_txq *txq, int len, int *waste); 135 static inline int copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, 136 struct txinfo *txinfo); 137 static inline void add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len); 138 static inline int add_mblk(struct sge_txq *txq, struct txinfo *txinfo, 139 mblk_t *m, int len); 140 static void free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo); 141 static int add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m, 142 struct txinfo *txinfo); 143 static void write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts); 144 static int write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m, 145 struct txinfo *txinfo); 146 static inline void write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq, 147 struct txpkts *txpkts, struct txinfo *txinfo); 148 static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, 149 int len); 150 static inline void ring_tx_db(struct adapter *sc, struct sge_eq *eq); 151 static int reclaim_tx_descs(struct sge_txq *txq, int howmany); 152 static void write_txqflush_wr(struct sge_txq *txq); 153 static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, 154 mblk_t *m); 155 static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl); 156 static kstat_t *setup_port_config_kstats(struct port_info *pi); 157 static kstat_t *setup_port_info_kstats(struct port_info *pi); 158 static kstat_t *setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, 159 int idx); 160 static int update_rxq_kstats(kstat_t *ksp, int rw); 161 static int update_port_info_kstats(kstat_t *ksp, int rw); 162 static kstat_t *setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, 163 int idx); 164 static int update_txq_kstats(kstat_t *ksp, int rw); 165 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *, 166 mblk_t *); 167 static int handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, 168 mblk_t *m); 169 170 static kmem_cache_t *rxbuf_cache_create(struct rxbuf_cache_params *); 171 static struct rxbuf *rxbuf_alloc(kmem_cache_t *, int, uint_t); 172 static void rxbuf_free(struct rxbuf *); 173 static int rxbuf_ctor(void *, void *, int); 174 static void rxbuf_dtor(void *, void *); 175 176 static inline int 177 reclaimable(struct sge_eq *eq) 178 { 179 unsigned int cidx; 180 181 cidx = eq->spg->cidx; /* stable snapshot */ 182 cidx = be16_to_cpu(cidx); 183 184 if (cidx >= eq->cidx) 185 return (cidx - eq->cidx); 186 else 187 return (cidx + eq->cap - eq->cidx); 188 } 189 190 void 191 t4_sge_init(struct adapter *sc) 192 { 193 struct driver_properties *p = &sc->props; 194 ddi_dma_attr_t *dma_attr; 195 ddi_device_acc_attr_t *acc_attr; 196 uint32_t sge_control, sge_conm_ctrl; 197 int egress_threshold; 198 199 /* 200 * Device access and DMA attributes for descriptor rings 201 */ 202 acc_attr = &sc->sge.acc_attr_desc; 203 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 204 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 205 acc_attr->devacc_attr_dataorder = DDI_STRICTORDER_ACC; 206 207 dma_attr = &sc->sge.dma_attr_desc; 208 dma_attr->dma_attr_version = DMA_ATTR_V0; 209 dma_attr->dma_attr_addr_lo = 0; 210 dma_attr->dma_attr_addr_hi = UINT64_MAX; 211 dma_attr->dma_attr_count_max = UINT64_MAX; 212 dma_attr->dma_attr_align = 512; 213 dma_attr->dma_attr_burstsizes = 0xfff; 214 dma_attr->dma_attr_minxfer = 1; 215 dma_attr->dma_attr_maxxfer = UINT64_MAX; 216 dma_attr->dma_attr_seg = UINT64_MAX; 217 dma_attr->dma_attr_sgllen = 1; 218 dma_attr->dma_attr_granular = 1; 219 dma_attr->dma_attr_flags = 0; 220 221 /* 222 * Device access and DMA attributes for tx buffers 223 */ 224 acc_attr = &sc->sge.acc_attr_tx; 225 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 226 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 227 228 dma_attr = &sc->sge.dma_attr_tx; 229 dma_attr->dma_attr_version = DMA_ATTR_V0; 230 dma_attr->dma_attr_addr_lo = 0; 231 dma_attr->dma_attr_addr_hi = UINT64_MAX; 232 dma_attr->dma_attr_count_max = UINT64_MAX; 233 dma_attr->dma_attr_align = 1; 234 dma_attr->dma_attr_burstsizes = 0xfff; 235 dma_attr->dma_attr_minxfer = 1; 236 dma_attr->dma_attr_maxxfer = UINT64_MAX; 237 dma_attr->dma_attr_seg = UINT64_MAX; 238 dma_attr->dma_attr_sgllen = TX_SGL_SEGS; 239 dma_attr->dma_attr_granular = 1; 240 dma_attr->dma_attr_flags = 0; 241 242 /* 243 * Ingress Padding Boundary and Egress Status Page Size are set up by 244 * t4_fixup_host_params(). 245 */ 246 sge_control = t4_read_reg(sc, A_SGE_CONTROL); 247 sc->sge.pktshift = G_PKTSHIFT(sge_control); 248 sc->sge.stat_len = (sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64; 249 250 /* t4_nex uses FLM packed mode */ 251 sc->sge.fl_align = t4_fl_pkt_align(sc, true); 252 253 /* 254 * Device access and DMA attributes for rx buffers 255 */ 256 sc->sge.rxb_params.dip = sc->dip; 257 sc->sge.rxb_params.buf_size = rx_buf_size; 258 259 acc_attr = &sc->sge.rxb_params.acc_attr_rx; 260 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 261 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 262 263 dma_attr = &sc->sge.rxb_params.dma_attr_rx; 264 dma_attr->dma_attr_version = DMA_ATTR_V0; 265 dma_attr->dma_attr_addr_lo = 0; 266 dma_attr->dma_attr_addr_hi = UINT64_MAX; 267 dma_attr->dma_attr_count_max = UINT64_MAX; 268 /* 269 * Low 4 bits of an rx buffer address have a special meaning to the SGE 270 * and an rx buf cannot have an address with any of these bits set. 271 * FL_ALIGN is >= 32 so we're sure things are ok. 272 */ 273 dma_attr->dma_attr_align = sc->sge.fl_align; 274 dma_attr->dma_attr_burstsizes = 0xfff; 275 dma_attr->dma_attr_minxfer = 1; 276 dma_attr->dma_attr_maxxfer = UINT64_MAX; 277 dma_attr->dma_attr_seg = UINT64_MAX; 278 dma_attr->dma_attr_sgllen = 1; 279 dma_attr->dma_attr_granular = 1; 280 dma_attr->dma_attr_flags = 0; 281 282 sc->sge.rxbuf_cache = rxbuf_cache_create(&sc->sge.rxb_params); 283 284 /* 285 * A FL with <= fl_starve_thres buffers is starving and a periodic 286 * timer will attempt to refill it. This needs to be larger than the 287 * SGE's Egress Congestion Threshold. If it isn't, then we can get 288 * stuck waiting for new packets while the SGE is waiting for us to 289 * give it more Free List entries. (Note that the SGE's Egress 290 * Congestion Threshold is in units of 2 Free List pointers.) For T4, 291 * there was only a single field to control this. For T5 there's the 292 * original field which now only applies to Unpacked Mode Free List 293 * buffers and a new field which only applies to Packed Mode Free List 294 * buffers. 295 */ 296 297 sge_conm_ctrl = t4_read_reg(sc, A_SGE_CONM_CTRL); 298 switch (CHELSIO_CHIP_VERSION(sc->params.chip)) { 299 case CHELSIO_T4: 300 egress_threshold = G_EGRTHRESHOLD(sge_conm_ctrl); 301 break; 302 case CHELSIO_T5: 303 egress_threshold = G_EGRTHRESHOLDPACKING(sge_conm_ctrl); 304 break; 305 case CHELSIO_T6: 306 default: 307 egress_threshold = G_T6_EGRTHRESHOLDPACKING(sge_conm_ctrl); 308 } 309 sc->sge.fl_starve_threshold = 2*egress_threshold + 1; 310 311 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, rx_buf_size); 312 313 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, 314 V_THRESHOLD_0(p->counter_val[0]) | 315 V_THRESHOLD_1(p->counter_val[1]) | 316 V_THRESHOLD_2(p->counter_val[2]) | 317 V_THRESHOLD_3(p->counter_val[3])); 318 319 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, 320 V_TIMERVALUE0(us_to_core_ticks(sc, p->timer_val[0])) | 321 V_TIMERVALUE1(us_to_core_ticks(sc, p->timer_val[1]))); 322 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, 323 V_TIMERVALUE2(us_to_core_ticks(sc, p->timer_val[2])) | 324 V_TIMERVALUE3(us_to_core_ticks(sc, p->timer_val[3]))); 325 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, 326 V_TIMERVALUE4(us_to_core_ticks(sc, p->timer_val[4])) | 327 V_TIMERVALUE5(us_to_core_ticks(sc, p->timer_val[5]))); 328 329 (void) t4_register_cpl_handler(sc, CPL_FW4_MSG, handle_fw_rpl); 330 (void) t4_register_cpl_handler(sc, CPL_FW6_MSG, handle_fw_rpl); 331 (void) t4_register_cpl_handler(sc, CPL_SGE_EGR_UPDATE, 332 handle_sge_egr_update); 333 (void) t4_register_cpl_handler(sc, CPL_RX_PKT, t4_eth_rx); 334 (void) t4_register_fw_msg_handler(sc, FW6_TYPE_CMD_RPL, 335 t4_handle_fw_rpl); 336 } 337 338 /* 339 * Allocate and initialize the firmware event queue and the forwarded interrupt 340 * queues, if any. The adapter owns all these queues as they are not associated 341 * with any particular port. 342 * 343 * Returns errno on failure. Resources allocated up to that point may still be 344 * allocated. Caller is responsible for cleanup in case this function fails. 345 */ 346 int 347 t4_setup_adapter_queues(struct adapter *sc) 348 { 349 int rc; 350 351 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 352 353 /* 354 * Firmware event queue 355 */ 356 rc = alloc_fwq(sc); 357 if (rc != 0) 358 return (rc); 359 360 return (rc); 361 } 362 363 /* 364 * Idempotent 365 */ 366 int 367 t4_teardown_adapter_queues(struct adapter *sc) 368 { 369 370 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 371 372 (void) free_fwq(sc); 373 374 return (0); 375 } 376 377 static inline int 378 first_vector(struct port_info *pi) 379 { 380 struct adapter *sc = pi->adapter; 381 int rc = T4_EXTRA_INTR, i; 382 383 if (sc->intr_count == 1) 384 return (0); 385 386 for_each_port(sc, i) { 387 struct port_info *p = sc->port[i]; 388 389 if (i == pi->port_id) 390 break; 391 392 /* 393 * Not compiled with offload support and intr_count > 1. Only 394 * NIC queues exist and they'd better be taking direct 395 * interrupts. 396 */ 397 ASSERT(!(sc->flags & INTR_FWD)); 398 rc += p->nrxq; 399 } 400 return (rc); 401 } 402 403 /* 404 * Given an arbitrary "index," come up with an iq that can be used by other 405 * queues (of this port) for interrupt forwarding, SGE egress updates, etc. 406 * The iq returned is guaranteed to be something that takes direct interrupts. 407 */ 408 static struct sge_iq * 409 port_intr_iq(struct port_info *pi, int idx) 410 { 411 struct adapter *sc = pi->adapter; 412 struct sge *s = &sc->sge; 413 struct sge_iq *iq = NULL; 414 415 if (sc->intr_count == 1) 416 return (&sc->sge.fwq); 417 418 /* 419 * Not compiled with offload support and intr_count > 1. Only NIC 420 * queues exist and they'd better be taking direct interrupts. 421 */ 422 ASSERT(!(sc->flags & INTR_FWD)); 423 424 idx %= pi->nrxq; 425 iq = &s->rxq[pi->first_rxq + idx].iq; 426 427 return (iq); 428 } 429 430 int 431 t4_setup_port_queues(struct port_info *pi) 432 { 433 int rc = 0, i, intr_idx, j; 434 struct sge_rxq *rxq; 435 struct sge_txq *txq; 436 struct adapter *sc = pi->adapter; 437 struct driver_properties *p = &sc->props; 438 439 pi->ksp_config = setup_port_config_kstats(pi); 440 pi->ksp_info = setup_port_info_kstats(pi); 441 442 /* Interrupt vector to start from (when using multiple vectors) */ 443 intr_idx = first_vector(pi); 444 445 /* 446 * First pass over all rx queues (NIC and TOE): 447 * a) initialize iq and fl 448 * b) allocate queue iff it will take direct interrupts. 449 */ 450 451 for_each_rxq(pi, i, rxq) { 452 453 init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, p->qsize_rxq, 454 RX_IQ_ESIZE); 455 456 init_fl(&rxq->fl, p->qsize_rxq / 8); /* 8 bufs in each entry */ 457 458 if ((!(sc->flags & INTR_FWD)) || 459 (sc->intr_count > 1 && pi->nrxq)) { 460 rxq->iq.flags |= IQ_INTR; 461 rc = alloc_rxq(pi, rxq, intr_idx, i); 462 if (rc != 0) 463 goto done; 464 intr_idx++; 465 } 466 467 } 468 469 /* 470 * Second pass over all rx queues (NIC and TOE). The queues forwarding 471 * their interrupts are allocated now. 472 */ 473 j = 0; 474 for_each_rxq(pi, i, rxq) { 475 if (rxq->iq.flags & IQ_INTR) 476 continue; 477 478 intr_idx = port_intr_iq(pi, j)->abs_id; 479 480 rc = alloc_rxq(pi, rxq, intr_idx, i); 481 if (rc != 0) 482 goto done; 483 j++; 484 } 485 486 /* 487 * Now the tx queues. Only one pass needed. 488 */ 489 j = 0; 490 for_each_txq(pi, i, txq) { 491 uint16_t iqid; 492 493 iqid = port_intr_iq(pi, j)->cntxt_id; 494 init_eq(sc, &txq->eq, EQ_ETH, p->qsize_txq, pi->tx_chan, iqid); 495 rc = alloc_txq(pi, txq, i); 496 if (rc != 0) 497 goto done; 498 } 499 500 done: 501 if (rc != 0) 502 (void) t4_teardown_port_queues(pi); 503 504 return (rc); 505 } 506 507 /* 508 * Idempotent 509 */ 510 int 511 t4_teardown_port_queues(struct port_info *pi) 512 { 513 int i; 514 struct sge_rxq *rxq; 515 struct sge_txq *txq; 516 517 if (pi->ksp_config != NULL) { 518 kstat_delete(pi->ksp_config); 519 pi->ksp_config = NULL; 520 } 521 if (pi->ksp_info != NULL) { 522 kstat_delete(pi->ksp_info); 523 pi->ksp_info = NULL; 524 } 525 526 for_each_txq(pi, i, txq) { 527 (void) free_txq(pi, txq); 528 } 529 530 for_each_rxq(pi, i, rxq) { 531 if ((rxq->iq.flags & IQ_INTR) == 0) 532 (void) free_rxq(pi, rxq); 533 } 534 535 /* 536 * Then take down the rx queues that take direct interrupts. 537 */ 538 539 for_each_rxq(pi, i, rxq) { 540 if (rxq->iq.flags & IQ_INTR) 541 (void) free_rxq(pi, rxq); 542 } 543 544 return (0); 545 } 546 547 /* Deals with errors and forwarded interrupts */ 548 uint_t 549 t4_intr_all(caddr_t arg1, caddr_t arg2) 550 { 551 552 (void) t4_intr_err(arg1, arg2); 553 (void) t4_intr(arg1, arg2); 554 555 return (DDI_INTR_CLAIMED); 556 } 557 558 static void 559 t4_intr_rx_work(struct sge_iq *iq) 560 { 561 mblk_t *mp = NULL; 562 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */ 563 RXQ_LOCK(rxq); 564 if (!iq->polling) { 565 mp = t4_ring_rx(rxq, iq->qsize/8); 566 t4_write_reg(iq->adapter, MYPF_REG(A_SGE_PF_GTS), 567 V_INGRESSQID((u32)iq->cntxt_id) | 568 V_SEINTARM(iq->intr_next)); 569 } 570 RXQ_UNLOCK(rxq); 571 if (mp != NULL) { 572 mac_rx_ring(rxq->port->mh, rxq->ring_handle, mp, 573 rxq->ring_gen_num); 574 } 575 } 576 577 /* Deals with interrupts on the given ingress queue */ 578 /* ARGSUSED */ 579 uint_t 580 t4_intr(caddr_t arg1, caddr_t arg2) 581 { 582 struct sge_iq *iq = (struct sge_iq *)arg2; 583 int state; 584 585 /* 586 * Right now receive polling is only enabled for MSI-X and 587 * when we have enough msi-x vectors i.e no interrupt forwarding. 588 */ 589 if (iq->adapter->props.multi_rings) { 590 t4_intr_rx_work(iq); 591 } else { 592 state = atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY); 593 if (state == IQS_IDLE) { 594 (void) service_iq(iq, 0); 595 (void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE); 596 } 597 } 598 return (DDI_INTR_CLAIMED); 599 } 600 601 /* Deals with error interrupts */ 602 /* ARGSUSED */ 603 uint_t 604 t4_intr_err(caddr_t arg1, caddr_t arg2) 605 { 606 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 607 struct adapter *sc = (struct adapter *)arg1; 608 609 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0); 610 (void) t4_slow_intr_handler(sc); 611 612 return (DDI_INTR_CLAIMED); 613 } 614 615 /* 616 * t4_ring_rx - Process responses from an SGE response queue. 617 * 618 * This function processes responses from an SGE response queue up to the 619 * supplied budget. Responses include received packets as well as control 620 * messages from FW or HW. 621 * 622 * It returns a chain of mblks containing the received data, to be 623 * passed up to mac_rx_ring(). 624 */ 625 mblk_t * 626 t4_ring_rx(struct sge_rxq *rxq, int budget) 627 { 628 struct sge_iq *iq = &rxq->iq; 629 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */ 630 struct adapter *sc = iq->adapter; 631 struct rsp_ctrl *ctrl; 632 const struct rss_header *rss; 633 int ndescs = 0, fl_bufs_used = 0; 634 int rsp_type; 635 uint32_t lq; 636 mblk_t *mblk_head = NULL, **mblk_tail, *m; 637 struct cpl_rx_pkt *cpl; 638 uint32_t received_bytes = 0, pkt_len = 0; 639 bool csum_ok; 640 uint16_t err_vec; 641 642 mblk_tail = &mblk_head; 643 644 while (is_new_response(iq, &ctrl)) { 645 646 membar_consumer(); 647 648 m = NULL; 649 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen); 650 lq = be32_to_cpu(ctrl->pldbuflen_qid); 651 rss = (const void *)iq->cdesc; 652 653 switch (rsp_type) { 654 case X_RSPD_TYPE_FLBUF: 655 656 ASSERT(iq->flags & IQ_HAS_FL); 657 658 if (CPL_RX_PKT == rss->opcode) { 659 cpl = (void *)(rss + 1); 660 pkt_len = be16_to_cpu(cpl->len); 661 662 if (iq->polling && 663 ((received_bytes + pkt_len) > budget)) 664 goto done; 665 666 m = get_fl_payload(sc, fl, lq, &fl_bufs_used); 667 if (m == NULL) 668 goto done; 669 670 iq->intr_next = iq->intr_params; 671 m->b_rptr += sc->sge.pktshift; 672 if (sc->params.tp.rx_pkt_encap) { 673 /* Enabled only in T6 config file */ 674 err_vec = G_T6_COMPR_RXERR_VEC( 675 ntohs(cpl->err_vec)); 676 } else { 677 err_vec = ntohs(cpl->err_vec); 678 } 679 680 csum_ok = cpl->csum_calc && !err_vec; 681 682 /* TODO: what about cpl->ip_frag? */ 683 if (csum_ok && !cpl->ip_frag) { 684 mac_hcksum_set(m, 0, 0, 0, 0xffff, 685 HCK_FULLCKSUM_OK | HCK_FULLCKSUM | 686 HCK_IPV4_HDRCKSUM_OK); 687 rxq->rxcsum++; 688 } 689 rxq->rxpkts++; 690 rxq->rxbytes += pkt_len; 691 received_bytes += pkt_len; 692 693 *mblk_tail = m; 694 mblk_tail = &m->b_next; 695 696 break; 697 } 698 699 m = get_fl_payload(sc, fl, lq, &fl_bufs_used); 700 if (m == NULL) 701 goto done; 702 /* FALLTHROUGH */ 703 704 case X_RSPD_TYPE_CPL: 705 ASSERT(rss->opcode < NUM_CPL_CMDS); 706 sc->cpl_handler[rss->opcode](iq, rss, m); 707 break; 708 709 default: 710 break; 711 } 712 iq_next(iq); 713 ++ndescs; 714 if (!iq->polling && (ndescs == budget)) 715 break; 716 } 717 718 done: 719 720 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 721 V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) | 722 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); 723 724 if ((fl_bufs_used > 0) || (iq->flags & IQ_HAS_FL)) { 725 int starved; 726 FL_LOCK(fl); 727 fl->needed += fl_bufs_used; 728 starved = refill_fl(sc, fl, fl->cap / 8); 729 FL_UNLOCK(fl); 730 if (starved) 731 add_fl_to_sfl(sc, fl); 732 } 733 return (mblk_head); 734 } 735 736 /* 737 * Deals with anything and everything on the given ingress queue. 738 */ 739 static int 740 service_iq(struct sge_iq *iq, int budget) 741 { 742 struct sge_iq *q; 743 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */ 744 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */ 745 struct adapter *sc = iq->adapter; 746 struct rsp_ctrl *ctrl; 747 const struct rss_header *rss; 748 int ndescs = 0, limit, fl_bufs_used = 0; 749 int rsp_type; 750 uint32_t lq; 751 int starved; 752 mblk_t *m; 753 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql); 754 755 limit = budget ? budget : iq->qsize / 8; 756 757 /* 758 * We always come back and check the descriptor ring for new indirect 759 * interrupts and other responses after running a single handler. 760 */ 761 for (;;) { 762 while (is_new_response(iq, &ctrl)) { 763 764 membar_consumer(); 765 766 m = NULL; 767 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen); 768 lq = be32_to_cpu(ctrl->pldbuflen_qid); 769 rss = (const void *)iq->cdesc; 770 771 switch (rsp_type) { 772 case X_RSPD_TYPE_FLBUF: 773 774 ASSERT(iq->flags & IQ_HAS_FL); 775 776 m = get_fl_payload(sc, fl, lq, &fl_bufs_used); 777 if (m == NULL) { 778 /* 779 * Rearm the iq with a 780 * longer-than-default timer 781 */ 782 const uint32_t timer_idx = 783 V_QINTR_TIMER_IDX(SGE_NTIMERS-1); 784 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 785 V_CIDXINC(ndescs) | 786 V_INGRESSQID((u32)iq->cntxt_id) | 787 V_SEINTARM(timer_idx)); 788 if (fl_bufs_used > 0) { 789 ASSERT(iq->flags & IQ_HAS_FL); 790 FL_LOCK(fl); 791 fl->needed += fl_bufs_used; 792 starved = refill_fl(sc, fl, 793 fl->cap / 8); 794 FL_UNLOCK(fl); 795 if (starved) 796 add_fl_to_sfl(sc, fl); 797 } 798 return (0); 799 } 800 801 /* FALLTHRU */ 802 case X_RSPD_TYPE_CPL: 803 804 ASSERT(rss->opcode < NUM_CPL_CMDS); 805 sc->cpl_handler[rss->opcode](iq, rss, m); 806 break; 807 808 case X_RSPD_TYPE_INTR: 809 810 /* 811 * Interrupts should be forwarded only to queues 812 * that are not forwarding their interrupts. 813 * This means service_iq can recurse but only 1 814 * level deep. 815 */ 816 ASSERT(budget == 0); 817 818 q = sc->sge.iqmap[lq - sc->sge.iq_start]; 819 if (atomic_cas_uint(&q->state, IQS_IDLE, 820 IQS_BUSY) == IQS_IDLE) { 821 if (service_iq(q, q->qsize / 8) == 0) { 822 (void) atomic_cas_uint( 823 &q->state, IQS_BUSY, 824 IQS_IDLE); 825 } else { 826 STAILQ_INSERT_TAIL(&iql, q, 827 link); 828 } 829 } 830 break; 831 832 default: 833 break; 834 } 835 836 iq_next(iq); 837 if (++ndescs == limit) { 838 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 839 V_CIDXINC(ndescs) | 840 V_INGRESSQID(iq->cntxt_id) | 841 V_SEINTARM(V_QINTR_TIMER_IDX( 842 X_TIMERREG_UPDATE_CIDX))); 843 ndescs = 0; 844 845 if (fl_bufs_used > 0) { 846 ASSERT(iq->flags & IQ_HAS_FL); 847 FL_LOCK(fl); 848 fl->needed += fl_bufs_used; 849 (void) refill_fl(sc, fl, fl->cap / 8); 850 FL_UNLOCK(fl); 851 fl_bufs_used = 0; 852 } 853 854 if (budget != 0) 855 return (EINPROGRESS); 856 } 857 } 858 859 if (STAILQ_EMPTY(&iql) != 0) 860 break; 861 862 /* 863 * Process the head only, and send it to the back of the list if 864 * it's still not done. 865 */ 866 q = STAILQ_FIRST(&iql); 867 STAILQ_REMOVE_HEAD(&iql, link); 868 if (service_iq(q, q->qsize / 8) == 0) 869 (void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE); 870 else 871 STAILQ_INSERT_TAIL(&iql, q, link); 872 } 873 874 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) | 875 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next)); 876 877 if (iq->flags & IQ_HAS_FL) { 878 879 FL_LOCK(fl); 880 fl->needed += fl_bufs_used; 881 starved = refill_fl(sc, fl, fl->cap / 4); 882 FL_UNLOCK(fl); 883 if (starved != 0) 884 add_fl_to_sfl(sc, fl); 885 } 886 887 return (0); 888 } 889 890 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */ 891 #define TXPKTS_PKT_HDR ((\ 892 sizeof (struct ulp_txpkt) + \ 893 sizeof (struct ulptx_idata) + \ 894 sizeof (struct cpl_tx_pkt_core)) / 8) 895 896 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */ 897 #define TXPKTS_WR_HDR (\ 898 sizeof (struct fw_eth_tx_pkts_wr) / 8 + \ 899 TXPKTS_PKT_HDR) 900 901 /* Header of a tx WR, before SGL of first packet (in flits) */ 902 #define TXPKT_WR_HDR ((\ 903 sizeof (struct fw_eth_tx_pkt_wr) + \ 904 sizeof (struct cpl_tx_pkt_core)) / 8) 905 906 /* Header of a tx LSO WR, before SGL of first packet (in flits) */ 907 #define TXPKT_LSO_WR_HDR ((\ 908 sizeof (struct fw_eth_tx_pkt_wr) + \ 909 sizeof (struct cpl_tx_pkt_lso_core) + \ 910 sizeof (struct cpl_tx_pkt_core)) / 8) 911 912 mblk_t * 913 t4_eth_tx(void *arg, mblk_t *frame) 914 { 915 struct sge_txq *txq = (struct sge_txq *)arg; 916 struct port_info *pi = txq->port; 917 struct adapter *sc = pi->adapter; 918 struct sge_eq *eq = &txq->eq; 919 mblk_t *next_frame; 920 int rc, coalescing; 921 struct txpkts txpkts; 922 struct txinfo txinfo; 923 924 txpkts.npkt = 0; /* indicates there's nothing in txpkts */ 925 coalescing = 0; 926 927 TXQ_LOCK(txq); 928 if (eq->avail < 8) 929 (void) reclaim_tx_descs(txq, 8); 930 for (; frame; frame = next_frame) { 931 932 if (eq->avail < 8) 933 break; 934 935 next_frame = frame->b_next; 936 frame->b_next = NULL; 937 938 if (next_frame != NULL) 939 coalescing = 1; 940 941 rc = get_frame_txinfo(txq, &frame, &txinfo, coalescing); 942 if (rc != 0) { 943 if (rc == ENOMEM) { 944 945 /* Short of resources, suspend tx */ 946 947 frame->b_next = next_frame; 948 break; 949 } 950 951 /* 952 * Unrecoverable error for this frame, throw it 953 * away and move on to the next. 954 */ 955 956 freemsg(frame); 957 continue; 958 } 959 960 if (coalescing != 0 && 961 add_to_txpkts(txq, &txpkts, frame, &txinfo) == 0) { 962 963 /* Successfully absorbed into txpkts */ 964 965 write_ulp_cpl_sgl(pi, txq, &txpkts, &txinfo); 966 goto doorbell; 967 } 968 969 /* 970 * We weren't coalescing to begin with, or current frame could 971 * not be coalesced (add_to_txpkts flushes txpkts if a frame 972 * given to it can't be coalesced). Either way there should be 973 * nothing in txpkts. 974 */ 975 ASSERT(txpkts.npkt == 0); 976 977 /* We're sending out individual frames now */ 978 coalescing = 0; 979 980 if (eq->avail < 8) 981 (void) reclaim_tx_descs(txq, 8); 982 rc = write_txpkt_wr(pi, txq, frame, &txinfo); 983 if (rc != 0) { 984 985 /* Short of hardware descriptors, suspend tx */ 986 987 /* 988 * This is an unlikely but expensive failure. We've 989 * done all the hard work (DMA bindings etc.) and now we 990 * can't send out the frame. What's worse, we have to 991 * spend even more time freeing up everything in txinfo. 992 */ 993 txq->qfull++; 994 free_txinfo_resources(txq, &txinfo); 995 996 frame->b_next = next_frame; 997 break; 998 } 999 1000 doorbell: 1001 /* Fewer and fewer doorbells as the queue fills up */ 1002 if (eq->pending >= (1 << (fls(eq->qsize - eq->avail) / 2))) { 1003 txq->txbytes += txinfo.len; 1004 txq->txpkts++; 1005 ring_tx_db(sc, eq); 1006 } 1007 (void) reclaim_tx_descs(txq, 32); 1008 } 1009 1010 if (txpkts.npkt > 0) 1011 write_txpkts_wr(txq, &txpkts); 1012 1013 /* 1014 * frame not NULL means there was an error but we haven't thrown it 1015 * away. This can happen when we're short of tx descriptors (qfull) or 1016 * maybe even DMA handles (dma_hdl_failed). Either way, a credit flush 1017 * and reclaim will get things going again. 1018 * 1019 * If eq->avail is already 0 we know a credit flush was requested in the 1020 * WR that reduced it to 0 so we don't need another flush (we don't have 1021 * any descriptor for a flush WR anyway, duh). 1022 */ 1023 if (frame && eq->avail > 0) 1024 write_txqflush_wr(txq); 1025 1026 if (eq->pending != 0) 1027 ring_tx_db(sc, eq); 1028 1029 (void) reclaim_tx_descs(txq, eq->qsize); 1030 TXQ_UNLOCK(txq); 1031 1032 return (frame); 1033 } 1034 1035 static inline void 1036 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int8_t pktc_idx, 1037 int qsize, uint8_t esize) 1038 { 1039 ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS); 1040 ASSERT(pktc_idx < SGE_NCOUNTERS); /* -ve is ok, means don't use */ 1041 1042 iq->flags = 0; 1043 iq->adapter = sc; 1044 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx); 1045 iq->intr_pktc_idx = SGE_NCOUNTERS - 1; 1046 if (pktc_idx >= 0) { 1047 iq->intr_params |= F_QINTR_CNT_EN; 1048 iq->intr_pktc_idx = pktc_idx; 1049 } 1050 iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */ 1051 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */ 1052 } 1053 1054 static inline void 1055 init_fl(struct sge_fl *fl, uint16_t qsize) 1056 { 1057 1058 fl->qsize = qsize; 1059 fl->allocb_fail = 0; 1060 } 1061 1062 static inline void 1063 init_eq(struct adapter *sc, struct sge_eq *eq, uint16_t eqtype, uint16_t qsize, 1064 uint8_t tx_chan, uint16_t iqid) 1065 { 1066 struct sge *s = &sc->sge; 1067 uint32_t r; 1068 1069 ASSERT(tx_chan < NCHAN); 1070 ASSERT(eqtype <= EQ_TYPEMASK); 1071 1072 if (is_t5(sc->params.chip)) { 1073 r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF); 1074 r >>= S_QUEUESPERPAGEPF0 + 1075 (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf; 1076 s->s_qpp = r & M_QUEUESPERPAGEPF0; 1077 } 1078 1079 eq->flags = eqtype & EQ_TYPEMASK; 1080 eq->tx_chan = tx_chan; 1081 eq->iqid = iqid; 1082 eq->qsize = qsize; 1083 } 1084 1085 /* 1086 * Allocates the ring for an ingress queue and an optional freelist. If the 1087 * freelist is specified it will be allocated and then associated with the 1088 * ingress queue. 1089 * 1090 * Returns errno on failure. Resources allocated up to that point may still be 1091 * allocated. Caller is responsible for cleanup in case this function fails. 1092 * 1093 * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then 1094 * the intr_idx specifies the vector, starting from 0. Otherwise it specifies 1095 * the index of the queue to which its interrupts will be forwarded. 1096 */ 1097 static int 1098 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl, 1099 int intr_idx, int cong) 1100 { 1101 int rc, i, cntxt_id; 1102 size_t len; 1103 struct fw_iq_cmd c; 1104 struct adapter *sc = iq->adapter; 1105 uint32_t v = 0; 1106 1107 len = iq->qsize * iq->esize; 1108 rc = alloc_desc_ring(sc, len, DDI_DMA_READ, &iq->dhdl, &iq->ahdl, 1109 &iq->ba, (caddr_t *)&iq->desc); 1110 if (rc != 0) 1111 return (rc); 1112 1113 bzero(&c, sizeof (c)); 1114 c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | 1115 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) | 1116 V_FW_IQ_CMD_VFN(0)); 1117 1118 c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART | 1119 FW_LEN16(c)); 1120 1121 /* Special handling for firmware event queue */ 1122 if (iq == &sc->sge.fwq) 1123 v |= F_FW_IQ_CMD_IQASYNCH; 1124 1125 if (iq->flags & IQ_INTR) 1126 ASSERT(intr_idx < sc->intr_count); 1127 else 1128 v |= F_FW_IQ_CMD_IQANDST; 1129 v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx); 1130 1131 c.type_to_iqandstindex = cpu_to_be32(v | 1132 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | 1133 V_FW_IQ_CMD_VIID(pi->viid) | 1134 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT)); 1135 c.iqdroprss_to_iqesize = cpu_to_be16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | 1136 F_FW_IQ_CMD_IQGTSMODE | 1137 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) | 1138 V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4)); 1139 c.iqsize = cpu_to_be16(iq->qsize); 1140 c.iqaddr = cpu_to_be64(iq->ba); 1141 if (cong >= 0) { 1142 const uint32_t iq_type = 1143 cong ? FW_IQ_IQTYPE_NIC : FW_IQ_IQTYPE_OFLD; 1144 c.iqns_to_fl0congen = BE_32(F_FW_IQ_CMD_IQFLINTCONGEN | 1145 V_FW_IQ_CMD_IQTYPE(iq_type)); 1146 } 1147 1148 if (fl != NULL) { 1149 unsigned int chip_ver = CHELSIO_CHIP_VERSION(sc->params.chip); 1150 1151 mutex_init(&fl->lock, NULL, MUTEX_DRIVER, 1152 DDI_INTR_PRI(sc->intr_pri)); 1153 fl->flags |= FL_MTX; 1154 1155 len = fl->qsize * RX_FL_ESIZE; 1156 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &fl->dhdl, 1157 &fl->ahdl, &fl->ba, (caddr_t *)&fl->desc); 1158 if (rc != 0) 1159 return (rc); 1160 1161 /* Allocate space for one software descriptor per buffer. */ 1162 fl->cap = (fl->qsize - sc->sge.stat_len / RX_FL_ESIZE) * 8; 1163 fl->sdesc = kmem_zalloc(sizeof (struct fl_sdesc) * fl->cap, 1164 KM_SLEEP); 1165 fl->needed = fl->cap; 1166 fl->lowat = roundup(sc->sge.fl_starve_threshold, 8); 1167 1168 c.iqns_to_fl0congen |= 1169 cpu_to_be32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) | 1170 F_FW_IQ_CMD_FL0PACKEN | F_FW_IQ_CMD_FL0PADEN); 1171 if (cong >= 0) { 1172 c.iqns_to_fl0congen |= 1173 BE_32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) | 1174 F_FW_IQ_CMD_FL0CONGCIF | 1175 F_FW_IQ_CMD_FL0CONGEN); 1176 } 1177 1178 /* 1179 * In T6, for egress queue type FL there is internal overhead 1180 * of 16B for header going into FLM module. Hence the maximum 1181 * allowed burst size is 448 bytes. For T4/T5, the hardware 1182 * doesn't coalesce fetch requests if more than 64 bytes of 1183 * Free List pointers are provided, so we use a 128-byte Fetch 1184 * Burst Minimum there (T6 implements coalescing so we can use 1185 * the smaller 64-byte value there). 1186 */ 1187 1188 c.fl0dcaen_to_fl0cidxfthresh = cpu_to_be16( 1189 V_FW_IQ_CMD_FL0FBMIN(chip_ver <= CHELSIO_T5 ? 1190 X_FETCHBURSTMIN_128B : X_FETCHBURSTMIN_64B) | 1191 V_FW_IQ_CMD_FL0FBMAX(chip_ver <= CHELSIO_T5 ? 1192 X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B)); 1193 c.fl0size = cpu_to_be16(fl->qsize); 1194 c.fl0addr = cpu_to_be64(fl->ba); 1195 } 1196 1197 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1198 if (rc != 0) { 1199 cxgb_printf(sc->dip, CE_WARN, 1200 "failed to create ingress queue: %d", rc); 1201 return (rc); 1202 } 1203 1204 iq->cdesc = iq->desc; 1205 iq->cidx = 0; 1206 iq->gen = 1; 1207 iq->intr_next = iq->intr_params; 1208 iq->adapter = sc; 1209 iq->cntxt_id = be16_to_cpu(c.iqid); 1210 iq->abs_id = be16_to_cpu(c.physiqid); 1211 iq->flags |= IQ_ALLOCATED; 1212 mutex_init(&iq->lock, NULL, MUTEX_DRIVER, 1213 DDI_INTR_PRI(DDI_INTR_PRI(sc->intr_pri))); 1214 iq->polling = 0; 1215 1216 cntxt_id = iq->cntxt_id - sc->sge.iq_start; 1217 if (cntxt_id >= sc->sge.iqmap_sz) { 1218 panic("%s: iq->cntxt_id (%d) more than the max (%d)", __func__, 1219 cntxt_id, sc->sge.iqmap_sz - 1); 1220 } 1221 sc->sge.iqmap[cntxt_id] = iq; 1222 1223 if (fl != NULL) { 1224 fl->cntxt_id = be16_to_cpu(c.fl0id); 1225 fl->pidx = fl->cidx = 0; 1226 fl->copy_threshold = rx_copy_threshold; 1227 1228 cntxt_id = fl->cntxt_id - sc->sge.eq_start; 1229 if (cntxt_id >= sc->sge.eqmap_sz) { 1230 panic("%s: fl->cntxt_id (%d) more than the max (%d)", 1231 __func__, cntxt_id, sc->sge.eqmap_sz - 1); 1232 } 1233 sc->sge.eqmap[cntxt_id] = (void *)fl; 1234 1235 FL_LOCK(fl); 1236 (void) refill_fl(sc, fl, fl->lowat); 1237 FL_UNLOCK(fl); 1238 1239 iq->flags |= IQ_HAS_FL; 1240 } 1241 1242 if (is_t5(sc->params.chip) && cong >= 0) { 1243 uint32_t param, val; 1244 1245 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 1246 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) | 1247 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id); 1248 if (cong == 0) 1249 val = 1 << 19; 1250 else { 1251 val = 2 << 19; 1252 for (i = 0; i < 4; i++) { 1253 if (cong & (1 << i)) 1254 val |= 1 << (i << 2); 1255 } 1256 } 1257 1258 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1259 if (rc != 0) { 1260 /* report error but carry on */ 1261 cxgb_printf(sc->dip, CE_WARN, 1262 "failed to set congestion manager context for " 1263 "ingress queue %d: %d", iq->cntxt_id, rc); 1264 } 1265 } 1266 1267 /* Enable IQ interrupts */ 1268 iq->state = IQS_IDLE; 1269 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) | 1270 V_INGRESSQID(iq->cntxt_id)); 1271 1272 return (0); 1273 } 1274 1275 static int 1276 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl) 1277 { 1278 int rc; 1279 1280 if (iq != NULL) { 1281 struct adapter *sc = iq->adapter; 1282 dev_info_t *dip; 1283 1284 dip = pi ? pi->dip : sc->dip; 1285 if (iq->flags & IQ_ALLOCATED) { 1286 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, 1287 FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id, 1288 fl ? fl->cntxt_id : 0xffff, 0xffff); 1289 if (rc != 0) { 1290 cxgb_printf(dip, CE_WARN, 1291 "failed to free queue %p: %d", iq, rc); 1292 return (rc); 1293 } 1294 mutex_destroy(&iq->lock); 1295 iq->flags &= ~IQ_ALLOCATED; 1296 } 1297 1298 if (iq->desc != NULL) { 1299 (void) free_desc_ring(&iq->dhdl, &iq->ahdl); 1300 iq->desc = NULL; 1301 } 1302 1303 bzero(iq, sizeof (*iq)); 1304 } 1305 1306 if (fl != NULL) { 1307 if (fl->sdesc != NULL) { 1308 FL_LOCK(fl); 1309 free_fl_bufs(fl); 1310 FL_UNLOCK(fl); 1311 1312 kmem_free(fl->sdesc, sizeof (struct fl_sdesc) * 1313 fl->cap); 1314 fl->sdesc = NULL; 1315 } 1316 1317 if (fl->desc != NULL) { 1318 (void) free_desc_ring(&fl->dhdl, &fl->ahdl); 1319 fl->desc = NULL; 1320 } 1321 1322 if (fl->flags & FL_MTX) { 1323 mutex_destroy(&fl->lock); 1324 fl->flags &= ~FL_MTX; 1325 } 1326 1327 bzero(fl, sizeof (struct sge_fl)); 1328 } 1329 1330 return (0); 1331 } 1332 1333 static int 1334 alloc_fwq(struct adapter *sc) 1335 { 1336 int rc, intr_idx; 1337 struct sge_iq *fwq = &sc->sge.fwq; 1338 1339 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE); 1340 fwq->flags |= IQ_INTR; /* always */ 1341 intr_idx = sc->intr_count > 1 ? 1 : 0; 1342 rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1); 1343 if (rc != 0) { 1344 cxgb_printf(sc->dip, CE_WARN, 1345 "failed to create firmware event queue: %d.", rc); 1346 return (rc); 1347 } 1348 1349 return (0); 1350 } 1351 1352 static int 1353 free_fwq(struct adapter *sc) 1354 { 1355 1356 return (free_iq_fl(NULL, &sc->sge.fwq, NULL)); 1357 } 1358 1359 static int 1360 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int i) 1361 { 1362 int rc; 1363 1364 rxq->port = pi; 1365 rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx, 1366 t4_get_tp_ch_map(pi->adapter, pi->tx_chan)); 1367 if (rc != 0) 1368 return (rc); 1369 1370 rxq->ksp = setup_rxq_kstats(pi, rxq, i); 1371 1372 return (rc); 1373 } 1374 1375 static int 1376 free_rxq(struct port_info *pi, struct sge_rxq *rxq) 1377 { 1378 int rc; 1379 1380 if (rxq->ksp != NULL) { 1381 kstat_delete(rxq->ksp); 1382 rxq->ksp = NULL; 1383 } 1384 1385 rc = free_iq_fl(pi, &rxq->iq, &rxq->fl); 1386 if (rc == 0) 1387 bzero(&rxq->fl, sizeof (*rxq) - offsetof(struct sge_rxq, fl)); 1388 1389 return (rc); 1390 } 1391 1392 static int 1393 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) 1394 { 1395 int rc, cntxt_id; 1396 struct fw_eq_ctrl_cmd c; 1397 1398 bzero(&c, sizeof (c)); 1399 1400 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST | 1401 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) | 1402 V_FW_EQ_CTRL_CMD_VFN(0)); 1403 c.alloc_to_len16 = BE_32(F_FW_EQ_CTRL_CMD_ALLOC | 1404 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c)); 1405 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* TODO */ 1406 c.physeqid_pkd = BE_32(0); 1407 c.fetchszm_to_iqid = 1408 BE_32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1409 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) | 1410 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid)); 1411 c.dcaen_to_eqsize = 1412 BE_32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1413 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1414 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1415 V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize)); 1416 c.eqaddr = BE_64(eq->ba); 1417 1418 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1419 if (rc != 0) { 1420 cxgb_printf(sc->dip, CE_WARN, 1421 "failed to create control queue %d: %d", eq->tx_chan, rc); 1422 return (rc); 1423 } 1424 eq->flags |= EQ_ALLOCATED; 1425 1426 eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(BE_32(c.cmpliqid_eqid)); 1427 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1428 if (cntxt_id >= sc->sge.eqmap_sz) 1429 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1430 cntxt_id, sc->sge.eqmap_sz - 1); 1431 sc->sge.eqmap[cntxt_id] = eq; 1432 1433 return (rc); 1434 } 1435 1436 static int 1437 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1438 { 1439 int rc, cntxt_id; 1440 struct fw_eq_eth_cmd c; 1441 1442 bzero(&c, sizeof (c)); 1443 1444 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST | 1445 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) | 1446 V_FW_EQ_ETH_CMD_VFN(0)); 1447 c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC | 1448 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c)); 1449 c.autoequiqe_to_viid = BE_32(F_FW_EQ_ETH_CMD_AUTOEQUIQE | 1450 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(pi->viid)); 1451 c.fetchszm_to_iqid = 1452 BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1453 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | 1454 V_FW_EQ_ETH_CMD_IQID(eq->iqid)); 1455 c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1456 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1457 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1458 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize)); 1459 c.eqaddr = BE_64(eq->ba); 1460 1461 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1462 if (rc != 0) { 1463 cxgb_printf(pi->dip, CE_WARN, 1464 "failed to create Ethernet egress queue: %d", rc); 1465 return (rc); 1466 } 1467 eq->flags |= EQ_ALLOCATED; 1468 1469 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd)); 1470 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1471 if (cntxt_id >= sc->sge.eqmap_sz) 1472 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1473 cntxt_id, sc->sge.eqmap_sz - 1); 1474 sc->sge.eqmap[cntxt_id] = eq; 1475 1476 return (rc); 1477 } 1478 1479 static int 1480 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1481 { 1482 int rc; 1483 size_t len; 1484 1485 mutex_init(&eq->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(sc->intr_pri)); 1486 eq->flags |= EQ_MTX; 1487 1488 len = eq->qsize * EQ_ESIZE; 1489 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &eq->desc_dhdl, 1490 &eq->desc_ahdl, &eq->ba, (caddr_t *)&eq->desc); 1491 if (rc != 0) 1492 return (rc); 1493 1494 eq->cap = eq->qsize - sc->sge.stat_len / EQ_ESIZE; 1495 eq->spg = (void *)&eq->desc[eq->cap]; 1496 eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */ 1497 eq->pidx = eq->cidx = 0; 1498 eq->doorbells = sc->doorbells; 1499 1500 switch (eq->flags & EQ_TYPEMASK) { 1501 case EQ_CTRL: 1502 rc = ctrl_eq_alloc(sc, eq); 1503 break; 1504 1505 case EQ_ETH: 1506 rc = eth_eq_alloc(sc, pi, eq); 1507 break; 1508 1509 default: 1510 panic("%s: invalid eq type %d.", __func__, 1511 eq->flags & EQ_TYPEMASK); 1512 } 1513 1514 if (eq->doorbells & (DOORBELL_UDB | DOORBELL_UDBWC | DOORBELL_WCWR)) { 1515 uint32_t s_qpp = sc->sge.s_qpp; 1516 uint32_t mask = (1 << s_qpp) - 1; 1517 volatile uint8_t *udb; 1518 1519 udb = (volatile uint8_t *)sc->reg1p + UDBS_DB_OFFSET; 1520 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */ 1521 eq->udb_qid = eq->cntxt_id & mask; /* id in page */ 1522 if (eq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE) 1523 eq->doorbells &= ~DOORBELL_WCWR; 1524 else { 1525 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */ 1526 eq->udb_qid = 0; 1527 } 1528 eq->udb = (volatile void *)udb; 1529 } 1530 1531 if (rc != 0) { 1532 cxgb_printf(sc->dip, CE_WARN, 1533 "failed to allocate egress queue(%d): %d", 1534 eq->flags & EQ_TYPEMASK, rc); 1535 } 1536 1537 return (rc); 1538 } 1539 1540 static int 1541 free_eq(struct adapter *sc, struct sge_eq *eq) 1542 { 1543 int rc; 1544 1545 if (eq->flags & EQ_ALLOCATED) { 1546 switch (eq->flags & EQ_TYPEMASK) { 1547 case EQ_CTRL: 1548 rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0, 1549 eq->cntxt_id); 1550 break; 1551 1552 case EQ_ETH: 1553 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, 1554 eq->cntxt_id); 1555 break; 1556 default: 1557 panic("%s: invalid eq type %d.", __func__, 1558 eq->flags & EQ_TYPEMASK); 1559 } 1560 if (rc != 0) { 1561 cxgb_printf(sc->dip, CE_WARN, 1562 "failed to free egress queue (%d): %d", 1563 eq->flags & EQ_TYPEMASK, rc); 1564 return (rc); 1565 } 1566 eq->flags &= ~EQ_ALLOCATED; 1567 } 1568 1569 if (eq->desc != NULL) { 1570 (void) free_desc_ring(&eq->desc_dhdl, &eq->desc_ahdl); 1571 eq->desc = NULL; 1572 } 1573 1574 if (eq->flags & EQ_MTX) 1575 mutex_destroy(&eq->lock); 1576 1577 bzero(eq, sizeof (*eq)); 1578 return (0); 1579 } 1580 1581 static int 1582 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx) 1583 { 1584 int rc, i; 1585 struct adapter *sc = pi->adapter; 1586 struct sge_eq *eq = &txq->eq; 1587 1588 rc = alloc_eq(sc, pi, eq); 1589 if (rc != 0) 1590 return (rc); 1591 1592 txq->port = pi; 1593 txq->sdesc = kmem_zalloc(sizeof (struct tx_sdesc) * eq->cap, KM_SLEEP); 1594 txq->txb_size = eq->qsize * tx_copy_threshold; 1595 rc = alloc_tx_copybuffer(sc, txq->txb_size, &txq->txb_dhdl, 1596 &txq->txb_ahdl, &txq->txb_ba, &txq->txb_va); 1597 if (rc == 0) 1598 txq->txb_avail = txq->txb_size; 1599 else 1600 txq->txb_avail = txq->txb_size = 0; 1601 1602 /* 1603 * TODO: is this too low? Worst case would need around 4 times qsize 1604 * (all tx descriptors filled to the brim with SGLs, with each entry in 1605 * the SGL coming from a distinct DMA handle). Increase tx_dhdl_total 1606 * if you see too many dma_hdl_failed. 1607 */ 1608 txq->tx_dhdl_total = eq->qsize * 2; 1609 txq->tx_dhdl = kmem_zalloc(sizeof (ddi_dma_handle_t) * 1610 txq->tx_dhdl_total, KM_SLEEP); 1611 for (i = 0; i < txq->tx_dhdl_total; i++) { 1612 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx, 1613 DDI_DMA_SLEEP, 0, &txq->tx_dhdl[i]); 1614 if (rc != DDI_SUCCESS) { 1615 cxgb_printf(sc->dip, CE_WARN, 1616 "%s: failed to allocate DMA handle (%d)", 1617 __func__, rc); 1618 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 1619 } 1620 txq->tx_dhdl_avail++; 1621 } 1622 1623 txq->ksp = setup_txq_kstats(pi, txq, idx); 1624 1625 return (rc); 1626 } 1627 1628 static int 1629 free_txq(struct port_info *pi, struct sge_txq *txq) 1630 { 1631 int i; 1632 struct adapter *sc = pi->adapter; 1633 struct sge_eq *eq = &txq->eq; 1634 1635 if (txq->ksp != NULL) { 1636 kstat_delete(txq->ksp); 1637 txq->ksp = NULL; 1638 } 1639 1640 if (txq->txb_va != NULL) { 1641 (void) free_desc_ring(&txq->txb_dhdl, &txq->txb_ahdl); 1642 txq->txb_va = NULL; 1643 } 1644 1645 if (txq->sdesc != NULL) { 1646 struct tx_sdesc *sd; 1647 ddi_dma_handle_t hdl; 1648 1649 TXQ_LOCK(txq); 1650 while (eq->cidx != eq->pidx) { 1651 sd = &txq->sdesc[eq->cidx]; 1652 1653 for (i = sd->hdls_used; i; i--) { 1654 hdl = txq->tx_dhdl[txq->tx_dhdl_cidx]; 1655 (void) ddi_dma_unbind_handle(hdl); 1656 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total) 1657 txq->tx_dhdl_cidx = 0; 1658 } 1659 1660 ASSERT(sd->m); 1661 freemsgchain(sd->m); 1662 1663 eq->cidx += sd->desc_used; 1664 if (eq->cidx >= eq->cap) 1665 eq->cidx -= eq->cap; 1666 1667 txq->txb_avail += txq->txb_used; 1668 } 1669 ASSERT(txq->tx_dhdl_cidx == txq->tx_dhdl_pidx); 1670 ASSERT(txq->txb_avail == txq->txb_size); 1671 TXQ_UNLOCK(txq); 1672 1673 kmem_free(txq->sdesc, sizeof (struct tx_sdesc) * eq->cap); 1674 txq->sdesc = NULL; 1675 } 1676 1677 if (txq->tx_dhdl != NULL) { 1678 for (i = 0; i < txq->tx_dhdl_total; i++) { 1679 if (txq->tx_dhdl[i] != NULL) 1680 ddi_dma_free_handle(&txq->tx_dhdl[i]); 1681 } 1682 kmem_free(txq->tx_dhdl, 1683 sizeof (ddi_dma_handle_t) * txq->tx_dhdl_total); 1684 txq->tx_dhdl = NULL; 1685 } 1686 1687 (void) free_eq(sc, &txq->eq); 1688 1689 bzero(txq, sizeof (*txq)); 1690 return (0); 1691 } 1692 1693 /* 1694 * Allocates a block of contiguous memory for DMA. Can be used to allocate 1695 * memory for descriptor rings or for tx/rx copy buffers. 1696 * 1697 * Caller does not have to clean up anything if this function fails, it cleans 1698 * up after itself. 1699 * 1700 * Caller provides the following: 1701 * len length of the block of memory to allocate. 1702 * flags DDI_DMA_* flags to use (CONSISTENT/STREAMING, READ/WRITE/RDWR) 1703 * acc_attr device access attributes for the allocation. 1704 * dma_attr DMA attributes for the allocation 1705 * 1706 * If the function is successful it fills up this information: 1707 * dma_hdl DMA handle for the allocated memory 1708 * acc_hdl access handle for the allocated memory 1709 * ba bus address of the allocated memory 1710 * va KVA of the allocated memory. 1711 */ 1712 static int 1713 alloc_dma_memory(struct adapter *sc, size_t len, int flags, 1714 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr, 1715 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 1716 uint64_t *pba, caddr_t *pva) 1717 { 1718 int rc; 1719 ddi_dma_handle_t dhdl; 1720 ddi_acc_handle_t ahdl; 1721 ddi_dma_cookie_t cookie; 1722 uint_t ccount; 1723 caddr_t va; 1724 size_t real_len; 1725 1726 *pva = NULL; 1727 1728 /* 1729 * DMA handle. 1730 */ 1731 rc = ddi_dma_alloc_handle(sc->dip, dma_attr, DDI_DMA_SLEEP, 0, &dhdl); 1732 if (rc != DDI_SUCCESS) { 1733 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 1734 } 1735 1736 /* 1737 * Memory suitable for DMA. 1738 */ 1739 rc = ddi_dma_mem_alloc(dhdl, len, acc_attr, 1740 flags & DDI_DMA_CONSISTENT ? DDI_DMA_CONSISTENT : DDI_DMA_STREAMING, 1741 DDI_DMA_SLEEP, 0, &va, &real_len, &ahdl); 1742 if (rc != DDI_SUCCESS) { 1743 ddi_dma_free_handle(&dhdl); 1744 return (ENOMEM); 1745 } 1746 1747 /* 1748 * DMA bindings. 1749 */ 1750 rc = ddi_dma_addr_bind_handle(dhdl, NULL, va, real_len, flags, NULL, 1751 NULL, &cookie, &ccount); 1752 if (rc != DDI_DMA_MAPPED) { 1753 ddi_dma_mem_free(&ahdl); 1754 ddi_dma_free_handle(&dhdl); 1755 return (ENOMEM); 1756 } 1757 if (ccount != 1) { 1758 /* unusable DMA mapping */ 1759 (void) free_desc_ring(&dhdl, &ahdl); 1760 return (ENOMEM); 1761 } 1762 1763 bzero(va, real_len); 1764 *dma_hdl = dhdl; 1765 *acc_hdl = ahdl; 1766 *pba = cookie.dmac_laddress; 1767 *pva = va; 1768 1769 return (0); 1770 } 1771 1772 static int 1773 free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl) 1774 { 1775 (void) ddi_dma_unbind_handle(*dhdl); 1776 ddi_dma_mem_free(ahdl); 1777 ddi_dma_free_handle(dhdl); 1778 1779 return (0); 1780 } 1781 1782 static int 1783 alloc_desc_ring(struct adapter *sc, size_t len, int rw, 1784 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 1785 uint64_t *pba, caddr_t *pva) 1786 { 1787 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_desc; 1788 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; 1789 1790 return (alloc_dma_memory(sc, len, DDI_DMA_CONSISTENT | rw, acc_attr, 1791 dma_attr, dma_hdl, acc_hdl, pba, pva)); 1792 } 1793 1794 static int 1795 free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl) 1796 { 1797 return (free_dma_memory(dhdl, ahdl)); 1798 } 1799 1800 static int 1801 alloc_tx_copybuffer(struct adapter *sc, size_t len, 1802 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 1803 uint64_t *pba, caddr_t *pva) 1804 { 1805 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_tx; 1806 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; /* NOT dma_attr_tx */ 1807 1808 return (alloc_dma_memory(sc, len, DDI_DMA_STREAMING | DDI_DMA_WRITE, 1809 acc_attr, dma_attr, dma_hdl, acc_hdl, pba, pva)); 1810 } 1811 1812 static inline bool 1813 is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl) 1814 { 1815 (void) ddi_dma_sync(iq->dhdl, (uintptr_t)iq->cdesc - 1816 (uintptr_t)iq->desc, iq->esize, DDI_DMA_SYNC_FORKERNEL); 1817 1818 *ctrl = (void *)((uintptr_t)iq->cdesc + 1819 (iq->esize - sizeof (struct rsp_ctrl))); 1820 1821 return ((((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen)); 1822 } 1823 1824 static inline void 1825 iq_next(struct sge_iq *iq) 1826 { 1827 iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize); 1828 if (++iq->cidx == iq->qsize - 1) { 1829 iq->cidx = 0; 1830 iq->gen ^= 1; 1831 iq->cdesc = iq->desc; 1832 } 1833 } 1834 1835 /* 1836 * Fill up the freelist by upto nbufs and maybe ring its doorbell. 1837 * 1838 * Returns non-zero to indicate that it should be added to the list of starving 1839 * freelists. 1840 */ 1841 static int 1842 refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs) 1843 { 1844 uint64_t *d = &fl->desc[fl->pidx]; 1845 struct fl_sdesc *sd = &fl->sdesc[fl->pidx]; 1846 1847 FL_LOCK_ASSERT_OWNED(fl); 1848 ASSERT(nbufs >= 0); 1849 1850 if (nbufs > fl->needed) 1851 nbufs = fl->needed; 1852 1853 while (nbufs--) { 1854 if (sd->rxb != NULL) { 1855 if (sd->rxb->ref_cnt == 1) { 1856 /* 1857 * Buffer is available for recycling. Two ways 1858 * this can happen: 1859 * 1860 * a) All the packets DMA'd into it last time 1861 * around were within the rx_copy_threshold 1862 * and no part of the buffer was ever passed 1863 * up (ref_cnt never went over 1). 1864 * 1865 * b) Packets DMA'd into the buffer were passed 1866 * up but have all been freed by the upper 1867 * layers by now (ref_cnt went over 1 but is 1868 * now back to 1). 1869 * 1870 * Either way the bus address in the descriptor 1871 * ring is already valid. 1872 */ 1873 ASSERT(*d == cpu_to_be64(sd->rxb->ba)); 1874 d++; 1875 goto recycled; 1876 } else { 1877 /* 1878 * Buffer still in use and we need a 1879 * replacement. But first release our reference 1880 * on the existing buffer. 1881 */ 1882 rxbuf_free(sd->rxb); 1883 } 1884 } 1885 1886 sd->rxb = rxbuf_alloc(sc->sge.rxbuf_cache, KM_NOSLEEP, 1); 1887 if (sd->rxb == NULL) 1888 break; 1889 *d++ = cpu_to_be64(sd->rxb->ba); 1890 1891 recycled: fl->pending++; 1892 sd++; 1893 fl->needed--; 1894 if (++fl->pidx == fl->cap) { 1895 fl->pidx = 0; 1896 sd = fl->sdesc; 1897 d = fl->desc; 1898 } 1899 } 1900 1901 if (fl->pending >= 8) 1902 ring_fl_db(sc, fl); 1903 1904 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING)); 1905 } 1906 1907 #ifndef TAILQ_FOREACH_SAFE 1908 #define TAILQ_FOREACH_SAFE(var, head, field, tvar) \ 1909 for ((var) = TAILQ_FIRST((head)); \ 1910 (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \ 1911 (var) = (tvar)) 1912 #endif 1913 1914 /* 1915 * Attempt to refill all starving freelists. 1916 */ 1917 static void 1918 refill_sfl(void *arg) 1919 { 1920 struct adapter *sc = arg; 1921 struct sge_fl *fl, *fl_temp; 1922 1923 mutex_enter(&sc->sfl_lock); 1924 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) { 1925 FL_LOCK(fl); 1926 (void) refill_fl(sc, fl, 64); 1927 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) { 1928 TAILQ_REMOVE(&sc->sfl, fl, link); 1929 fl->flags &= ~FL_STARVING; 1930 } 1931 FL_UNLOCK(fl); 1932 } 1933 1934 if (!TAILQ_EMPTY(&sc->sfl) != 0) 1935 sc->sfl_timer = timeout(refill_sfl, sc, drv_usectohz(100000)); 1936 mutex_exit(&sc->sfl_lock); 1937 } 1938 1939 static void 1940 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl) 1941 { 1942 mutex_enter(&sc->sfl_lock); 1943 FL_LOCK(fl); 1944 if ((fl->flags & FL_DOOMED) == 0) { 1945 if (TAILQ_EMPTY(&sc->sfl) != 0) { 1946 sc->sfl_timer = timeout(refill_sfl, sc, 1947 drv_usectohz(100000)); 1948 } 1949 fl->flags |= FL_STARVING; 1950 TAILQ_INSERT_TAIL(&sc->sfl, fl, link); 1951 } 1952 FL_UNLOCK(fl); 1953 mutex_exit(&sc->sfl_lock); 1954 } 1955 1956 static void 1957 free_fl_bufs(struct sge_fl *fl) 1958 { 1959 struct fl_sdesc *sd; 1960 unsigned int i; 1961 1962 FL_LOCK_ASSERT_OWNED(fl); 1963 1964 for (i = 0; i < fl->cap; i++) { 1965 sd = &fl->sdesc[i]; 1966 1967 if (sd->rxb != NULL) { 1968 rxbuf_free(sd->rxb); 1969 sd->rxb = NULL; 1970 } 1971 } 1972 } 1973 1974 /* 1975 * Note that fl->cidx and fl->offset are left unchanged in case of failure. 1976 */ 1977 static mblk_t * 1978 get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf, 1979 int *fl_bufs_used) 1980 { 1981 struct mblk_pair frame = {0}; 1982 struct rxbuf *rxb; 1983 mblk_t *m = NULL; 1984 uint_t nbuf = 0, len, copy, n; 1985 uint32_t cidx, offset, rcidx, roffset; 1986 1987 /* 1988 * The SGE won't pack a new frame into the current buffer if the entire 1989 * payload doesn't fit in the remaining space. Move on to the next buf 1990 * in that case. 1991 */ 1992 rcidx = fl->cidx; 1993 roffset = fl->offset; 1994 if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) { 1995 fl->offset = 0; 1996 if (++fl->cidx == fl->cap) 1997 fl->cidx = 0; 1998 nbuf++; 1999 } 2000 cidx = fl->cidx; 2001 offset = fl->offset; 2002 2003 len = G_RSPD_LEN(len_newbuf); /* pktshift + payload length */ 2004 copy = (len <= fl->copy_threshold); 2005 if (copy != 0) { 2006 frame.head = m = allocb(len, BPRI_HI); 2007 if (m == NULL) { 2008 fl->allocb_fail++; 2009 DTRACE_PROBE1(t4__fl_alloc_fail, struct sge_fl *, fl); 2010 fl->cidx = rcidx; 2011 fl->offset = roffset; 2012 return (NULL); 2013 } 2014 } 2015 2016 while (len) { 2017 rxb = fl->sdesc[cidx].rxb; 2018 n = min(len, rxb->buf_size - offset); 2019 2020 (void) ddi_dma_sync(rxb->dhdl, offset, n, 2021 DDI_DMA_SYNC_FORKERNEL); 2022 2023 if (copy != 0) 2024 bcopy(rxb->va + offset, m->b_wptr, n); 2025 else { 2026 m = desballoc((unsigned char *)rxb->va + offset, n, 2027 BPRI_HI, &rxb->freefunc); 2028 if (m == NULL) { 2029 fl->allocb_fail++; 2030 DTRACE_PROBE1(t4__fl_alloc_fail, 2031 struct sge_fl *, fl); 2032 if (frame.head) 2033 freemsgchain(frame.head); 2034 fl->cidx = rcidx; 2035 fl->offset = roffset; 2036 return (NULL); 2037 } 2038 atomic_inc_uint(&rxb->ref_cnt); 2039 if (frame.head != NULL) 2040 frame.tail->b_cont = m; 2041 else 2042 frame.head = m; 2043 frame.tail = m; 2044 } 2045 m->b_wptr += n; 2046 len -= n; 2047 offset += roundup(n, sc->sge.fl_align); 2048 ASSERT(offset <= rxb->buf_size); 2049 if (offset == rxb->buf_size) { 2050 offset = 0; 2051 if (++cidx == fl->cap) 2052 cidx = 0; 2053 nbuf++; 2054 } 2055 } 2056 2057 fl->cidx = cidx; 2058 fl->offset = offset; 2059 (*fl_bufs_used) += nbuf; 2060 2061 ASSERT(frame.head != NULL); 2062 return (frame.head); 2063 } 2064 2065 /* 2066 * We'll do immediate data tx for non-LSO, but only when not coalescing. We're 2067 * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes 2068 * of immediate data. 2069 */ 2070 #define IMM_LEN ( \ 2071 2 * EQ_ESIZE \ 2072 - sizeof (struct fw_eth_tx_pkt_wr) \ 2073 - sizeof (struct cpl_tx_pkt_core)) 2074 2075 /* 2076 * Returns non-zero on failure, no need to cleanup anything in that case. 2077 * 2078 * Note 1: We always try to pull up the mblk if required and return E2BIG only 2079 * if this fails. 2080 * 2081 * Note 2: We'll also pullup incoming mblk if HW_LSO is set and the first mblk 2082 * does not have the TCP header in it. 2083 */ 2084 static int 2085 get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, struct txinfo *txinfo, 2086 int sgl_only) 2087 { 2088 uint32_t flags = 0, len, n; 2089 mblk_t *m = *fp; 2090 int rc; 2091 2092 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb and dma_hdls */ 2093 2094 mac_hcksum_get(m, NULL, NULL, NULL, NULL, &flags); 2095 txinfo->flags = (flags & HCK_TX_FLAGS); 2096 2097 mac_lso_get(m, &txinfo->mss, &flags); 2098 txinfo->flags |= (flags & HW_LSO_FLAGS); 2099 2100 if (flags & HW_LSO) 2101 sgl_only = 1; /* Do not allow immediate data with LSO */ 2102 2103 /* 2104 * If checksum or segmentation offloads are requested, gather 2105 * information about the sizes and types of headers in the packet. 2106 */ 2107 if (txinfo->flags != 0) { 2108 mac_ether_offload_info(m, &txinfo->meoi); 2109 } else { 2110 bzero(&txinfo->meoi, sizeof (txinfo->meoi)); 2111 } 2112 2113 start: txinfo->nsegs = 0; 2114 txinfo->hdls_used = 0; 2115 txinfo->txb_used = 0; 2116 txinfo->len = 0; 2117 2118 /* total length and a rough estimate of # of segments */ 2119 n = 0; 2120 for (; m; m = m->b_cont) { 2121 len = MBLKL(m); 2122 n += (len / PAGE_SIZE) + 1; 2123 txinfo->len += len; 2124 } 2125 m = *fp; 2126 2127 if (n >= TX_SGL_SEGS || (flags & HW_LSO && MBLKL(m) < 50)) { 2128 txq->pullup_early++; 2129 m = msgpullup(*fp, -1); 2130 if (m == NULL) { 2131 txq->pullup_failed++; 2132 return (E2BIG); /* (*fp) left as it was */ 2133 } 2134 freemsg(*fp); 2135 *fp = m; 2136 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags); 2137 } 2138 2139 if (txinfo->len <= IMM_LEN && !sgl_only) 2140 return (0); /* nsegs = 0 tells caller to use imm. tx */ 2141 2142 if (txinfo->len <= txq->copy_threshold && 2143 copy_into_txb(txq, m, txinfo->len, txinfo) == 0) 2144 goto done; 2145 2146 for (; m; m = m->b_cont) { 2147 2148 len = MBLKL(m); 2149 2150 /* Use tx copy buffer if this mblk is small enough */ 2151 if (len <= txq->copy_threshold && 2152 copy_into_txb(txq, m, len, txinfo) == 0) 2153 continue; 2154 2155 /* Add DMA bindings for this mblk to the SGL */ 2156 rc = add_mblk(txq, txinfo, m, len); 2157 2158 if (rc == E2BIG || 2159 (txinfo->nsegs == TX_SGL_SEGS && m->b_cont)) { 2160 2161 txq->pullup_late++; 2162 m = msgpullup(*fp, -1); 2163 if (m != NULL) { 2164 free_txinfo_resources(txq, txinfo); 2165 freemsg(*fp); 2166 *fp = m; 2167 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags); 2168 goto start; 2169 } 2170 2171 txq->pullup_failed++; 2172 rc = E2BIG; 2173 } 2174 2175 if (rc != 0) { 2176 free_txinfo_resources(txq, txinfo); 2177 return (rc); 2178 } 2179 } 2180 2181 ASSERT(txinfo->nsegs > 0 && txinfo->nsegs <= TX_SGL_SEGS); 2182 2183 done: 2184 2185 /* 2186 * Store the # of flits required to hold this frame's SGL in nflits. An 2187 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by 2188 * multiple (len0 + len1, addr0, addr1) tuples. If addr1 is not used 2189 * then len1 must be set to 0. 2190 */ 2191 n = txinfo->nsegs - 1; 2192 txinfo->nflits = (3 * n) / 2 + (n & 1) + 2; 2193 if (n & 1) 2194 txinfo->sgl.sge[n / 2].len[1] = cpu_to_be32(0); 2195 2196 txinfo->sgl.cmd_nsge = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_DSGL) | 2197 V_ULPTX_NSGE(txinfo->nsegs)); 2198 2199 return (0); 2200 } 2201 2202 static inline int 2203 fits_in_txb(struct sge_txq *txq, int len, int *waste) 2204 { 2205 if (txq->txb_avail < len) 2206 return (0); 2207 2208 if (txq->txb_next + len <= txq->txb_size) { 2209 *waste = 0; 2210 return (1); 2211 } 2212 2213 *waste = txq->txb_size - txq->txb_next; 2214 2215 return (txq->txb_avail - *waste < len ? 0 : 1); 2216 } 2217 2218 #define TXB_CHUNK 64 2219 2220 /* 2221 * Copies the specified # of bytes into txq's tx copy buffer and updates txinfo 2222 * and txq to indicate resources used. Caller has to make sure that those many 2223 * bytes are available in the mblk chain (b_cont linked). 2224 */ 2225 static inline int 2226 copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, struct txinfo *txinfo) 2227 { 2228 int waste, n; 2229 2230 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb */ 2231 2232 if (!fits_in_txb(txq, len, &waste)) { 2233 txq->txb_full++; 2234 return (ENOMEM); 2235 } 2236 2237 if (waste != 0) { 2238 ASSERT((waste & (TXB_CHUNK - 1)) == 0); 2239 txinfo->txb_used += waste; 2240 txq->txb_avail -= waste; 2241 txq->txb_next = 0; 2242 } 2243 2244 for (n = 0; n < len; m = m->b_cont) { 2245 bcopy(m->b_rptr, txq->txb_va + txq->txb_next + n, MBLKL(m)); 2246 n += MBLKL(m); 2247 } 2248 2249 add_seg(txinfo, txq->txb_ba + txq->txb_next, len); 2250 2251 n = roundup(len, TXB_CHUNK); 2252 txinfo->txb_used += n; 2253 txq->txb_avail -= n; 2254 txq->txb_next += n; 2255 ASSERT(txq->txb_next <= txq->txb_size); 2256 if (txq->txb_next == txq->txb_size) 2257 txq->txb_next = 0; 2258 2259 return (0); 2260 } 2261 2262 static inline void 2263 add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len) 2264 { 2265 ASSERT(txinfo->nsegs < TX_SGL_SEGS); /* must have room */ 2266 2267 if (txinfo->nsegs != 0) { 2268 int idx = txinfo->nsegs - 1; 2269 txinfo->sgl.sge[idx / 2].len[idx & 1] = cpu_to_be32(len); 2270 txinfo->sgl.sge[idx / 2].addr[idx & 1] = cpu_to_be64(ba); 2271 } else { 2272 txinfo->sgl.len0 = cpu_to_be32(len); 2273 txinfo->sgl.addr0 = cpu_to_be64(ba); 2274 } 2275 txinfo->nsegs++; 2276 } 2277 2278 /* 2279 * This function cleans up any partially allocated resources when it fails so 2280 * there's nothing for the caller to clean up in that case. 2281 * 2282 * EIO indicates permanent failure. Caller should drop the frame containing 2283 * this mblk and continue. 2284 * 2285 * E2BIG indicates that the SGL length for this mblk exceeds the hardware 2286 * limit. Caller should pull up the frame before trying to send it out. 2287 * (This error means our pullup_early heuristic did not work for this frame) 2288 * 2289 * ENOMEM indicates a temporary shortage of resources (DMA handles, other DMA 2290 * resources, etc.). Caller should suspend the tx queue and wait for reclaim to 2291 * free up resources. 2292 */ 2293 static inline int 2294 add_mblk(struct sge_txq *txq, struct txinfo *txinfo, mblk_t *m, int len) 2295 { 2296 ddi_dma_handle_t dhdl; 2297 ddi_dma_cookie_t cookie; 2298 uint_t ccount = 0; 2299 int rc; 2300 2301 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate dhdls */ 2302 2303 if (txq->tx_dhdl_avail == 0) { 2304 txq->dma_hdl_failed++; 2305 return (ENOMEM); 2306 } 2307 2308 dhdl = txq->tx_dhdl[txq->tx_dhdl_pidx]; 2309 rc = ddi_dma_addr_bind_handle(dhdl, NULL, (caddr_t)m->b_rptr, len, 2310 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie, 2311 &ccount); 2312 if (rc != DDI_DMA_MAPPED) { 2313 txq->dma_map_failed++; 2314 2315 ASSERT(rc != DDI_DMA_INUSE && rc != DDI_DMA_PARTIAL_MAP); 2316 2317 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EIO); 2318 } 2319 2320 if (ccount + txinfo->nsegs > TX_SGL_SEGS) { 2321 (void) ddi_dma_unbind_handle(dhdl); 2322 return (E2BIG); 2323 } 2324 2325 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size); 2326 while (--ccount) { 2327 ddi_dma_nextcookie(dhdl, &cookie); 2328 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size); 2329 } 2330 2331 if (++txq->tx_dhdl_pidx == txq->tx_dhdl_total) 2332 txq->tx_dhdl_pidx = 0; 2333 txq->tx_dhdl_avail--; 2334 txinfo->hdls_used++; 2335 2336 return (0); 2337 } 2338 2339 /* 2340 * Releases all the txq resources used up in the specified txinfo. 2341 */ 2342 static void 2343 free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo) 2344 { 2345 int n; 2346 2347 TXQ_LOCK_ASSERT_OWNED(txq); /* dhdls, txb */ 2348 2349 n = txinfo->txb_used; 2350 if (n > 0) { 2351 txq->txb_avail += n; 2352 if (n <= txq->txb_next) 2353 txq->txb_next -= n; 2354 else { 2355 n -= txq->txb_next; 2356 txq->txb_next = txq->txb_size - n; 2357 } 2358 } 2359 2360 for (n = txinfo->hdls_used; n > 0; n--) { 2361 if (txq->tx_dhdl_pidx > 0) 2362 txq->tx_dhdl_pidx--; 2363 else 2364 txq->tx_dhdl_pidx = txq->tx_dhdl_total - 1; 2365 txq->tx_dhdl_avail++; 2366 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_pidx]); 2367 } 2368 } 2369 2370 /* 2371 * Returns 0 to indicate that m has been accepted into a coalesced tx work 2372 * request. It has either been folded into txpkts or txpkts was flushed and m 2373 * has started a new coalesced work request (as the first frame in a fresh 2374 * txpkts). 2375 * 2376 * Returns non-zero to indicate a failure - caller is responsible for 2377 * transmitting m, if there was anything in txpkts it has been flushed. 2378 */ 2379 static int 2380 add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m, 2381 struct txinfo *txinfo) 2382 { 2383 struct sge_eq *eq = &txq->eq; 2384 int can_coalesce; 2385 struct tx_sdesc *txsd; 2386 uint8_t flits; 2387 2388 TXQ_LOCK_ASSERT_OWNED(txq); 2389 2390 if (txpkts->npkt > 0) { 2391 flits = TXPKTS_PKT_HDR + txinfo->nflits; 2392 can_coalesce = (txinfo->flags & HW_LSO) == 0 && 2393 txpkts->nflits + flits <= TX_WR_FLITS && 2394 txpkts->nflits + flits <= eq->avail * 8 && 2395 txpkts->plen + txinfo->len < 65536; 2396 2397 if (can_coalesce != 0) { 2398 txpkts->tail->b_next = m; 2399 txpkts->tail = m; 2400 txpkts->npkt++; 2401 txpkts->nflits += flits; 2402 txpkts->plen += txinfo->len; 2403 2404 txsd = &txq->sdesc[eq->pidx]; 2405 txsd->txb_used += txinfo->txb_used; 2406 txsd->hdls_used += txinfo->hdls_used; 2407 2408 return (0); 2409 } 2410 2411 /* 2412 * Couldn't coalesce m into txpkts. The first order of business 2413 * is to send txpkts on its way. Then we'll revisit m. 2414 */ 2415 write_txpkts_wr(txq, txpkts); 2416 } 2417 2418 /* 2419 * Check if we can start a new coalesced tx work request with m as 2420 * the first packet in it. 2421 */ 2422 2423 ASSERT(txpkts->npkt == 0); 2424 ASSERT(txinfo->len < 65536); 2425 2426 flits = TXPKTS_WR_HDR + txinfo->nflits; 2427 can_coalesce = (txinfo->flags & HW_LSO) == 0 && 2428 flits <= eq->avail * 8 && flits <= TX_WR_FLITS; 2429 2430 if (can_coalesce == 0) 2431 return (EINVAL); 2432 2433 /* 2434 * Start a fresh coalesced tx WR with m as the first frame in it. 2435 */ 2436 txpkts->tail = m; 2437 txpkts->npkt = 1; 2438 txpkts->nflits = flits; 2439 txpkts->flitp = &eq->desc[eq->pidx].flit[2]; 2440 txpkts->plen = txinfo->len; 2441 2442 txsd = &txq->sdesc[eq->pidx]; 2443 txsd->m = m; 2444 txsd->txb_used = txinfo->txb_used; 2445 txsd->hdls_used = txinfo->hdls_used; 2446 2447 return (0); 2448 } 2449 2450 /* 2451 * Note that write_txpkts_wr can never run out of hardware descriptors (but 2452 * write_txpkt_wr can). add_to_txpkts ensures that a frame is accepted for 2453 * coalescing only if sufficient hardware descriptors are available. 2454 */ 2455 static void 2456 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts) 2457 { 2458 struct sge_eq *eq = &txq->eq; 2459 struct fw_eth_tx_pkts_wr *wr; 2460 struct tx_sdesc *txsd; 2461 uint32_t ctrl; 2462 uint16_t ndesc; 2463 2464 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */ 2465 2466 ndesc = howmany(txpkts->nflits, 8); 2467 2468 wr = (void *)&eq->desc[eq->pidx]; 2469 wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR) | 2470 V_FW_WR_IMMDLEN(0)); /* immdlen does not matter in this WR */ 2471 ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2)); 2472 if (eq->avail == ndesc) 2473 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ; 2474 wr->equiq_to_len16 = cpu_to_be32(ctrl); 2475 wr->plen = cpu_to_be16(txpkts->plen); 2476 wr->npkt = txpkts->npkt; 2477 wr->r3 = wr->type = 0; 2478 2479 /* Everything else already written */ 2480 2481 txsd = &txq->sdesc[eq->pidx]; 2482 txsd->desc_used = ndesc; 2483 2484 txq->txb_used += txsd->txb_used / TXB_CHUNK; 2485 txq->hdl_used += txsd->hdls_used; 2486 2487 ASSERT(eq->avail >= ndesc); 2488 2489 eq->pending += ndesc; 2490 eq->avail -= ndesc; 2491 eq->pidx += ndesc; 2492 if (eq->pidx >= eq->cap) 2493 eq->pidx -= eq->cap; 2494 2495 txq->txpkts_pkts += txpkts->npkt; 2496 txq->txpkts_wrs++; 2497 txpkts->npkt = 0; /* emptied */ 2498 } 2499 2500 typedef enum { 2501 COS_SUCCESS, /* ctrl flit contains proper bits for csum offload */ 2502 COS_IGNORE, /* no csum offload requested */ 2503 COS_FAIL, /* csum offload requested, but pkt data missing */ 2504 } csum_offload_status_t; 2505 /* 2506 * Build a ctrl1 flit for checksum offload in CPL_TX_PKT_XT command 2507 */ 2508 static csum_offload_status_t 2509 csum_to_ctrl(const struct txinfo *txinfo, uint32_t chip_version, 2510 uint64_t *ctrlp) 2511 { 2512 const mac_ether_offload_info_t *meoi = &txinfo->meoi; 2513 const uint32_t tx_flags = txinfo->flags; 2514 const boolean_t needs_l3_csum = (tx_flags & HW_LSO) != 0 || 2515 (tx_flags & HCK_IPV4_HDRCKSUM) != 0; 2516 const boolean_t needs_l4_csum = (tx_flags & HW_LSO) != 0 || 2517 (tx_flags & (HCK_FULLCKSUM | HCK_PARTIALCKSUM)) != 0; 2518 2519 /* 2520 * Default to disabling any checksumming both for cases where it is not 2521 * requested, but also if we cannot appropriately interrogate the 2522 * required information from the packet. 2523 */ 2524 uint64_t ctrl = F_TXPKT_L4CSUM_DIS | F_TXPKT_IPCSUM_DIS; 2525 if (!needs_l3_csum && !needs_l4_csum) { 2526 *ctrlp = ctrl; 2527 return (COS_IGNORE); 2528 } 2529 2530 if (needs_l3_csum) { 2531 /* Only IPv4 checksums are supported (for L3) */ 2532 if ((meoi->meoi_flags & MEOI_L3INFO_SET) == 0 || 2533 meoi->meoi_l3proto != ETHERTYPE_IP) { 2534 *ctrlp = ctrl; 2535 return (COS_FAIL); 2536 } 2537 ctrl &= ~F_TXPKT_IPCSUM_DIS; 2538 } 2539 2540 if (needs_l4_csum) { 2541 /* 2542 * We need at least all of the L3 header to make decisions about 2543 * the contained L4 protocol. If not all of the L4 information 2544 * is present, we will leave it to the NIC to checksum all it is 2545 * able to. 2546 */ 2547 if ((meoi->meoi_flags & MEOI_L3INFO_SET) == 0) { 2548 *ctrlp = ctrl; 2549 return (COS_FAIL); 2550 } 2551 2552 /* 2553 * Since we are parsing the packet anyways, make the checksum 2554 * decision based on the L4 protocol, rather than using the 2555 * Generic TCP/UDP checksum using start & end offsets in the 2556 * packet (like requested with PARTIALCKSUM). 2557 */ 2558 int csum_type = -1; 2559 if (meoi->meoi_l3proto == ETHERTYPE_IP && 2560 meoi->meoi_l4proto == IPPROTO_TCP) { 2561 csum_type = TX_CSUM_TCPIP; 2562 } else if (meoi->meoi_l3proto == ETHERTYPE_IPV6 && 2563 meoi->meoi_l4proto == IPPROTO_TCP) { 2564 csum_type = TX_CSUM_TCPIP6; 2565 } else if (meoi->meoi_l3proto == ETHERTYPE_IP && 2566 meoi->meoi_l4proto == IPPROTO_UDP) { 2567 csum_type = TX_CSUM_UDPIP; 2568 } else if (meoi->meoi_l3proto == ETHERTYPE_IPV6 && 2569 meoi->meoi_l4proto == IPPROTO_UDP) { 2570 csum_type = TX_CSUM_UDPIP6; 2571 } else { 2572 *ctrlp = ctrl; 2573 return (COS_FAIL); 2574 } 2575 2576 ASSERT(csum_type != -1); 2577 ctrl &= ~F_TXPKT_L4CSUM_DIS; 2578 ctrl |= V_TXPKT_CSUM_TYPE(csum_type); 2579 } 2580 2581 if ((ctrl & F_TXPKT_IPCSUM_DIS) == 0 && 2582 (ctrl & F_TXPKT_L4CSUM_DIS) != 0) { 2583 /* 2584 * If only the IPv4 checksum is requested, we need to set an 2585 * appropriate type in the command for it. 2586 */ 2587 ctrl |= V_TXPKT_CSUM_TYPE(TX_CSUM_IP); 2588 } 2589 2590 ASSERT(ctrl != (F_TXPKT_L4CSUM_DIS | F_TXPKT_IPCSUM_DIS)); 2591 2592 /* 2593 * Fill in the requisite L2/L3 header length data. 2594 * 2595 * The Ethernet header length is recorded as 'size - 14 bytes' 2596 */ 2597 const uint8_t eth_len = meoi->meoi_l2hlen - 14; 2598 if (chip_version >= CHELSIO_T6) { 2599 ctrl |= V_T6_TXPKT_ETHHDR_LEN(eth_len); 2600 } else { 2601 ctrl |= V_TXPKT_ETHHDR_LEN(eth_len); 2602 } 2603 ctrl |= V_TXPKT_IPHDR_LEN(meoi->meoi_l3hlen); 2604 2605 *ctrlp = ctrl; 2606 return (COS_SUCCESS); 2607 } 2608 2609 static int 2610 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m, 2611 struct txinfo *txinfo) 2612 { 2613 struct sge_eq *eq = &txq->eq; 2614 struct fw_eth_tx_pkt_wr *wr; 2615 struct cpl_tx_pkt_core *cpl; 2616 uint32_t ctrl; /* used in many unrelated places */ 2617 uint64_t ctrl1; 2618 int nflits, ndesc; 2619 struct tx_sdesc *txsd; 2620 caddr_t dst; 2621 const mac_ether_offload_info_t *meoi = &txinfo->meoi; 2622 2623 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */ 2624 2625 /* 2626 * Do we have enough flits to send this frame out? 2627 */ 2628 ctrl = sizeof (struct cpl_tx_pkt_core); 2629 if (txinfo->flags & HW_LSO) { 2630 nflits = TXPKT_LSO_WR_HDR; 2631 ctrl += sizeof (struct cpl_tx_pkt_lso_core); 2632 } else { 2633 nflits = TXPKT_WR_HDR; 2634 } 2635 if (txinfo->nsegs > 0) 2636 nflits += txinfo->nflits; 2637 else { 2638 nflits += howmany(txinfo->len, 8); 2639 ctrl += txinfo->len; 2640 } 2641 ndesc = howmany(nflits, 8); 2642 if (ndesc > eq->avail) 2643 return (ENOMEM); 2644 2645 /* Firmware work request header */ 2646 wr = (void *)&eq->desc[eq->pidx]; 2647 wr->op_immdlen = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | 2648 V_FW_WR_IMMDLEN(ctrl)); 2649 ctrl = V_FW_WR_LEN16(howmany(nflits, 2)); 2650 if (eq->avail == ndesc) 2651 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ; 2652 wr->equiq_to_len16 = cpu_to_be32(ctrl); 2653 wr->r3 = 0; 2654 2655 if (txinfo->flags & HW_LSO && 2656 (meoi->meoi_flags & MEOI_L4INFO_SET) != 0 && 2657 meoi->meoi_l4proto == IPPROTO_TCP) { 2658 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); 2659 2660 ctrl = V_LSO_OPCODE((u32)CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | 2661 F_LSO_LAST_SLICE; 2662 2663 if (meoi->meoi_l2hlen > sizeof (struct ether_header)) { 2664 /* 2665 * This presently assumes a standard VLAN header, 2666 * without support for Q-in-Q. 2667 */ 2668 ctrl |= V_LSO_ETHHDR_LEN(1); 2669 } 2670 2671 switch (meoi->meoi_l3proto) { 2672 case ETHERTYPE_IPV6: 2673 ctrl |= F_LSO_IPV6; 2674 /* FALLTHROUGH */ 2675 case ETHERTYPE_IP: 2676 ctrl |= V_LSO_IPHDR_LEN(meoi->meoi_l3hlen / 4); 2677 break; 2678 default: 2679 break; 2680 } 2681 2682 ctrl |= V_LSO_TCPHDR_LEN(meoi->meoi_l4hlen / 4); 2683 2684 lso->lso_ctrl = cpu_to_be32(ctrl); 2685 lso->ipid_ofst = cpu_to_be16(0); 2686 lso->mss = cpu_to_be16(txinfo->mss); 2687 lso->seqno_offset = cpu_to_be32(0); 2688 if (is_t4(pi->adapter->params.chip)) 2689 lso->len = cpu_to_be32(txinfo->len); 2690 else 2691 lso->len = cpu_to_be32(V_LSO_T5_XFER_SIZE(txinfo->len)); 2692 2693 cpl = (void *)(lso + 1); 2694 2695 txq->tso_wrs++; 2696 } else { 2697 cpl = (void *)(wr + 1); 2698 } 2699 2700 /* Checksum offload */ 2701 switch (csum_to_ctrl(txinfo, 2702 CHELSIO_CHIP_VERSION(pi->adapter->params.chip), &ctrl1)) { 2703 case COS_SUCCESS: 2704 txq->txcsum++; 2705 break; 2706 case COS_FAIL: 2707 /* 2708 * Packet will be going out with checksums which are probably 2709 * wrong but there is little we can do now. 2710 */ 2711 txq->csum_failed++; 2712 break; 2713 default: 2714 break; 2715 } 2716 2717 /* CPL header */ 2718 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | 2719 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf)); 2720 cpl->pack = 0; 2721 cpl->len = cpu_to_be16(txinfo->len); 2722 cpl->ctrl1 = cpu_to_be64(ctrl1); 2723 2724 /* Software descriptor */ 2725 txsd = &txq->sdesc[eq->pidx]; 2726 txsd->m = m; 2727 txsd->txb_used = txinfo->txb_used; 2728 txsd->hdls_used = txinfo->hdls_used; 2729 /* LINTED: E_ASSIGN_NARROW_CONV */ 2730 txsd->desc_used = ndesc; 2731 2732 txq->txb_used += txinfo->txb_used / TXB_CHUNK; 2733 txq->hdl_used += txinfo->hdls_used; 2734 2735 eq->pending += ndesc; 2736 eq->avail -= ndesc; 2737 eq->pidx += ndesc; 2738 if (eq->pidx >= eq->cap) 2739 eq->pidx -= eq->cap; 2740 2741 /* SGL */ 2742 dst = (void *)(cpl + 1); 2743 if (txinfo->nsegs > 0) { 2744 txq->sgl_wrs++; 2745 copy_to_txd(eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8); 2746 2747 /* Need to zero-pad to a 16 byte boundary if not on one */ 2748 if ((uintptr_t)dst & 0xf) 2749 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 2750 *(uint64_t *)dst = 0; 2751 2752 } else { 2753 txq->imm_wrs++; 2754 #ifdef DEBUG 2755 ctrl = txinfo->len; 2756 #endif 2757 for (; m; m = m->b_cont) { 2758 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m)); 2759 #ifdef DEBUG 2760 ctrl -= MBLKL(m); 2761 #endif 2762 } 2763 ASSERT(ctrl == 0); 2764 } 2765 2766 txq->txpkt_wrs++; 2767 return (0); 2768 } 2769 2770 static inline void 2771 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq, 2772 struct txpkts *txpkts, struct txinfo *txinfo) 2773 { 2774 struct ulp_txpkt *ulpmc; 2775 struct ulptx_idata *ulpsc; 2776 struct cpl_tx_pkt_core *cpl; 2777 uintptr_t flitp, start, end; 2778 uint64_t ctrl; 2779 caddr_t dst; 2780 2781 ASSERT(txpkts->npkt > 0); 2782 2783 start = (uintptr_t)txq->eq.desc; 2784 end = (uintptr_t)txq->eq.spg; 2785 2786 /* Checksum offload */ 2787 switch (csum_to_ctrl(txinfo, 2788 CHELSIO_CHIP_VERSION(pi->adapter->params.chip), &ctrl)) { 2789 case COS_SUCCESS: 2790 txq->txcsum++; 2791 break; 2792 case COS_FAIL: 2793 /* 2794 * Packet will be going out with checksums which are probably 2795 * wrong but there is little we can do now. 2796 */ 2797 txq->csum_failed++; 2798 break; 2799 default: 2800 break; 2801 } 2802 2803 /* 2804 * The previous packet's SGL must have ended at a 16 byte boundary (this 2805 * is required by the firmware/hardware). It follows that flitp cannot 2806 * wrap around between the ULPTX master command and ULPTX subcommand (8 2807 * bytes each), and that it can not wrap around in the middle of the 2808 * cpl_tx_pkt_core either. 2809 */ 2810 flitp = (uintptr_t)txpkts->flitp; 2811 ASSERT((flitp & 0xf) == 0); 2812 2813 /* ULP master command */ 2814 ulpmc = (void *)flitp; 2815 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); 2816 ulpmc->len = htonl(howmany(sizeof (*ulpmc) + sizeof (*ulpsc) + 2817 sizeof (*cpl) + 8 * txinfo->nflits, 16)); 2818 2819 /* ULP subcommand */ 2820 ulpsc = (void *)(ulpmc + 1); 2821 ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) | 2822 F_ULP_TX_SC_MORE); 2823 ulpsc->len = cpu_to_be32(sizeof (struct cpl_tx_pkt_core)); 2824 2825 flitp += sizeof (*ulpmc) + sizeof (*ulpsc); 2826 if (flitp == end) 2827 flitp = start; 2828 2829 /* CPL_TX_PKT_XT */ 2830 cpl = (void *)flitp; 2831 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | 2832 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf)); 2833 cpl->pack = 0; 2834 cpl->len = cpu_to_be16(txinfo->len); 2835 cpl->ctrl1 = cpu_to_be64(ctrl); 2836 2837 flitp += sizeof (*cpl); 2838 if (flitp == end) 2839 flitp = start; 2840 2841 /* SGL for this frame */ 2842 dst = (caddr_t)flitp; 2843 copy_to_txd(&txq->eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8); 2844 flitp = (uintptr_t)dst; 2845 2846 /* Zero pad and advance to a 16 byte boundary if not already at one. */ 2847 if (flitp & 0xf) { 2848 2849 /* no matter what, flitp should be on an 8 byte boundary */ 2850 ASSERT((flitp & 0x7) == 0); 2851 2852 *(uint64_t *)flitp = 0; 2853 flitp += sizeof (uint64_t); 2854 txpkts->nflits++; 2855 } 2856 2857 if (flitp == end) 2858 flitp = start; 2859 2860 txpkts->flitp = (void *)flitp; 2861 } 2862 2863 static inline void 2864 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) 2865 { 2866 if ((uintptr_t)(*to) + len <= (uintptr_t)eq->spg) { 2867 bcopy(from, *to, len); 2868 (*to) += len; 2869 } else { 2870 int portion = (uintptr_t)eq->spg - (uintptr_t)(*to); 2871 2872 bcopy(from, *to, portion); 2873 from += portion; 2874 portion = len - portion; /* remaining */ 2875 bcopy(from, (void *)eq->desc, portion); 2876 (*to) = (caddr_t)eq->desc + portion; 2877 } 2878 } 2879 2880 static inline void 2881 ring_tx_db(struct adapter *sc, struct sge_eq *eq) 2882 { 2883 int val, db_mode; 2884 uint_t db = eq->doorbells; 2885 2886 if (eq->pending > 1) 2887 db &= ~DOORBELL_WCWR; 2888 2889 if (eq->pending > eq->pidx) { 2890 int offset = eq->cap - (eq->pending - eq->pidx); 2891 2892 /* pidx has wrapped around since last doorbell */ 2893 2894 (void) ddi_dma_sync(eq->desc_dhdl, 2895 offset * sizeof (struct tx_desc), 0, 2896 DDI_DMA_SYNC_FORDEV); 2897 (void) ddi_dma_sync(eq->desc_dhdl, 2898 0, eq->pidx * sizeof (struct tx_desc), 2899 DDI_DMA_SYNC_FORDEV); 2900 } else if (eq->pending > 0) { 2901 (void) ddi_dma_sync(eq->desc_dhdl, 2902 (eq->pidx - eq->pending) * sizeof (struct tx_desc), 2903 eq->pending * sizeof (struct tx_desc), 2904 DDI_DMA_SYNC_FORDEV); 2905 } 2906 2907 membar_producer(); 2908 2909 if (is_t4(sc->params.chip)) 2910 val = V_PIDX(eq->pending); 2911 else 2912 val = V_PIDX_T5(eq->pending); 2913 2914 db_mode = (1 << (ffs(db) - 1)); 2915 switch (db_mode) { 2916 case DOORBELL_UDB: 2917 *eq->udb = LE_32(V_QID(eq->udb_qid) | val); 2918 break; 2919 2920 case DOORBELL_WCWR: 2921 { 2922 volatile uint64_t *dst, *src; 2923 int i; 2924 /* 2925 * Queues whose 128B doorbell segment fits in 2926 * the page do not use relative qid 2927 * (udb_qid is always 0). Only queues with 2928 * doorbell segments can do WCWR. 2929 */ 2930 ASSERT(eq->udb_qid == 0 && eq->pending == 1); 2931 2932 dst = (volatile void *)((uintptr_t)eq->udb + 2933 UDBS_WR_OFFSET - UDBS_DB_OFFSET); 2934 i = eq->pidx ? eq->pidx - 1 : eq->cap - 1; 2935 src = (void *)&eq->desc[i]; 2936 while (src != (void *)&eq->desc[i + 1]) { 2937 *dst++ = *src++; 2938 } 2939 membar_producer(); 2940 break; 2941 } 2942 2943 case DOORBELL_UDBWC: 2944 *eq->udb = LE_32(V_QID(eq->udb_qid) | val); 2945 membar_producer(); 2946 break; 2947 2948 case DOORBELL_KDB: 2949 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), 2950 V_QID(eq->cntxt_id) | val); 2951 break; 2952 } 2953 2954 eq->pending = 0; 2955 } 2956 2957 static int 2958 reclaim_tx_descs(struct sge_txq *txq, int howmany) 2959 { 2960 struct tx_sdesc *txsd; 2961 uint_t cidx, can_reclaim, reclaimed, txb_freed, hdls_freed; 2962 struct sge_eq *eq = &txq->eq; 2963 2964 EQ_LOCK_ASSERT_OWNED(eq); 2965 2966 cidx = eq->spg->cidx; /* stable snapshot */ 2967 cidx = be16_to_cpu(cidx); 2968 2969 if (cidx >= eq->cidx) 2970 can_reclaim = cidx - eq->cidx; 2971 else 2972 can_reclaim = cidx + eq->cap - eq->cidx; 2973 2974 if (can_reclaim == 0) 2975 return (0); 2976 2977 txb_freed = hdls_freed = reclaimed = 0; 2978 do { 2979 int ndesc; 2980 2981 txsd = &txq->sdesc[eq->cidx]; 2982 ndesc = txsd->desc_used; 2983 2984 /* Firmware doesn't return "partial" credits. */ 2985 ASSERT(can_reclaim >= ndesc); 2986 2987 /* 2988 * We always keep mblk around, even for immediate data. If mblk 2989 * is NULL, this has to be the software descriptor for a credit 2990 * flush work request. 2991 */ 2992 if (txsd->m != NULL) 2993 freemsgchain(txsd->m); 2994 #ifdef DEBUG 2995 else { 2996 ASSERT(txsd->txb_used == 0); 2997 ASSERT(txsd->hdls_used == 0); 2998 ASSERT(ndesc == 1); 2999 } 3000 #endif 3001 3002 txb_freed += txsd->txb_used; 3003 hdls_freed += txsd->hdls_used; 3004 reclaimed += ndesc; 3005 3006 eq->cidx += ndesc; 3007 if (eq->cidx >= eq->cap) 3008 eq->cidx -= eq->cap; 3009 3010 can_reclaim -= ndesc; 3011 3012 } while (can_reclaim && reclaimed < howmany); 3013 3014 eq->avail += reclaimed; 3015 ASSERT(eq->avail < eq->cap); /* avail tops out at (cap - 1) */ 3016 3017 txq->txb_avail += txb_freed; 3018 3019 txq->tx_dhdl_avail += hdls_freed; 3020 ASSERT(txq->tx_dhdl_avail <= txq->tx_dhdl_total); 3021 for (; hdls_freed; hdls_freed--) { 3022 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_cidx]); 3023 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total) 3024 txq->tx_dhdl_cidx = 0; 3025 } 3026 3027 return (reclaimed); 3028 } 3029 3030 static void 3031 write_txqflush_wr(struct sge_txq *txq) 3032 { 3033 struct sge_eq *eq = &txq->eq; 3034 struct fw_eq_flush_wr *wr; 3035 struct tx_sdesc *txsd; 3036 3037 EQ_LOCK_ASSERT_OWNED(eq); 3038 ASSERT(eq->avail > 0); 3039 3040 wr = (void *)&eq->desc[eq->pidx]; 3041 bzero(wr, sizeof (*wr)); 3042 wr->opcode = FW_EQ_FLUSH_WR; 3043 wr->equiq_to_len16 = cpu_to_be32(V_FW_WR_LEN16(sizeof (*wr) / 16) | 3044 F_FW_WR_EQUEQ | F_FW_WR_EQUIQ); 3045 3046 txsd = &txq->sdesc[eq->pidx]; 3047 txsd->m = NULL; 3048 txsd->txb_used = 0; 3049 txsd->hdls_used = 0; 3050 txsd->desc_used = 1; 3051 3052 eq->pending++; 3053 eq->avail--; 3054 if (++eq->pidx == eq->cap) 3055 eq->pidx = 0; 3056 } 3057 3058 static int 3059 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m) 3060 { 3061 bool csum_ok; 3062 uint16_t err_vec; 3063 struct sge_rxq *rxq = (void *)iq; 3064 struct mblk_pair chain = {0}; 3065 struct adapter *sc = iq->adapter; 3066 const struct cpl_rx_pkt *cpl = (const void *)(rss + 1); 3067 3068 iq->intr_next = iq->intr_params; 3069 3070 m->b_rptr += sc->sge.pktshift; 3071 3072 /* Compressed error vector is enabled for T6 only */ 3073 if (sc->params.tp.rx_pkt_encap) 3074 /* It is enabled only in T6 config file */ 3075 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec)); 3076 else 3077 err_vec = ntohs(cpl->err_vec); 3078 3079 csum_ok = cpl->csum_calc && !err_vec; 3080 /* TODO: what about cpl->ip_frag? */ 3081 if (csum_ok && !cpl->ip_frag) { 3082 mac_hcksum_set(m, 0, 0, 0, 0xffff, 3083 HCK_FULLCKSUM_OK | HCK_FULLCKSUM | 3084 HCK_IPV4_HDRCKSUM_OK); 3085 rxq->rxcsum++; 3086 } 3087 3088 /* Add to the chain that we'll send up */ 3089 if (chain.head != NULL) 3090 chain.tail->b_next = m; 3091 else 3092 chain.head = m; 3093 chain.tail = m; 3094 3095 t4_mac_rx(rxq->port, rxq, chain.head); 3096 3097 rxq->rxpkts++; 3098 rxq->rxbytes += be16_to_cpu(cpl->len); 3099 return (0); 3100 } 3101 3102 #define FL_HW_IDX(idx) ((idx) >> 3) 3103 3104 static inline void 3105 ring_fl_db(struct adapter *sc, struct sge_fl *fl) 3106 { 3107 int desc_start, desc_last, ndesc; 3108 uint32_t v = sc->params.arch.sge_fl_db; 3109 3110 ndesc = FL_HW_IDX(fl->pending); 3111 3112 /* Hold back one credit if pidx = cidx */ 3113 if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx)) 3114 ndesc--; 3115 3116 /* 3117 * There are chances of ndesc modified above (to avoid pidx = cidx). 3118 * If there is nothing to post, return. 3119 */ 3120 if (ndesc <= 0) 3121 return; 3122 3123 desc_last = FL_HW_IDX(fl->pidx); 3124 3125 if (fl->pidx < fl->pending) { 3126 /* There was a wrap */ 3127 desc_start = FL_HW_IDX(fl->pidx + fl->cap - fl->pending); 3128 3129 /* From desc_start to the end of list */ 3130 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 0, 3131 DDI_DMA_SYNC_FORDEV); 3132 3133 /* From start of list to the desc_last */ 3134 if (desc_last != 0) 3135 (void) ddi_dma_sync(fl->dhdl, 0, desc_last * 3136 RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV); 3137 } else { 3138 /* There was no wrap, sync from start_desc to last_desc */ 3139 desc_start = FL_HW_IDX(fl->pidx - fl->pending); 3140 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 3141 ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV); 3142 } 3143 3144 if (is_t4(sc->params.chip)) 3145 v |= V_PIDX(ndesc); 3146 else 3147 v |= V_PIDX_T5(ndesc); 3148 v |= V_QID(fl->cntxt_id) | V_PIDX(ndesc); 3149 3150 membar_producer(); 3151 3152 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v); 3153 3154 /* 3155 * Update pending count: 3156 * Deduct the number of descriptors posted 3157 */ 3158 fl->pending -= ndesc * 8; 3159 } 3160 3161 static void 3162 tx_reclaim_task(void *arg) 3163 { 3164 struct sge_txq *txq = arg; 3165 3166 TXQ_LOCK(txq); 3167 reclaim_tx_descs(txq, txq->eq.qsize); 3168 TXQ_UNLOCK(txq); 3169 } 3170 3171 /* ARGSUSED */ 3172 static int 3173 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss, 3174 mblk_t *m) 3175 { 3176 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1); 3177 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid)); 3178 struct adapter *sc = iq->adapter; 3179 struct sge *s = &sc->sge; 3180 struct sge_eq *eq; 3181 struct sge_txq *txq; 3182 3183 txq = (void *)s->eqmap[qid - s->eq_start]; 3184 eq = &txq->eq; 3185 txq->qflush++; 3186 t4_mac_tx_update(txq->port, txq); 3187 3188 ddi_taskq_dispatch(sc->tq[eq->tx_chan], tx_reclaim_task, 3189 (void *)txq, DDI_NOSLEEP); 3190 3191 return (0); 3192 } 3193 3194 static int 3195 handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m) 3196 { 3197 struct adapter *sc = iq->adapter; 3198 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1); 3199 3200 ASSERT(m == NULL); 3201 3202 if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) { 3203 const struct rss_header *rss2; 3204 3205 rss2 = (const struct rss_header *)&cpl->data[0]; 3206 return (sc->cpl_handler[rss2->opcode](iq, rss2, m)); 3207 } 3208 return (sc->fw_msg_handler[cpl->type](sc, &cpl->data[0])); 3209 } 3210 3211 int 3212 t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count, 3213 int flags) 3214 { 3215 int i, rc; 3216 3217 txmaps->map_total = count; 3218 txmaps->map_avail = txmaps->map_cidx = txmaps->map_pidx = 0; 3219 3220 txmaps->map = kmem_zalloc(sizeof (ddi_dma_handle_t) * 3221 txmaps->map_total, flags); 3222 3223 for (i = 0; i < count; i++) { 3224 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx, 3225 DDI_DMA_SLEEP, 0, &txmaps->map[i]); 3226 if (rc != DDI_SUCCESS) { 3227 cxgb_printf(sc->dip, CE_WARN, 3228 "%s: failed to allocate DMA handle (%d)", 3229 __func__, rc); 3230 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 3231 } 3232 txmaps->map_avail++; 3233 } 3234 3235 return (0); 3236 } 3237 3238 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG) 3239 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR) 3240 #define KS_U_SET(x, y) kstatp->x.value.ul = (y) 3241 #define KS_U_FROM(x, y) kstatp->x.value.ul = (y)->x 3242 #define KS_C_SET(x, ...) \ 3243 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__) 3244 3245 /* 3246 * cxgbe:X:config 3247 */ 3248 struct cxgbe_port_config_kstats { 3249 kstat_named_t idx; 3250 kstat_named_t nrxq; 3251 kstat_named_t ntxq; 3252 kstat_named_t first_rxq; 3253 kstat_named_t first_txq; 3254 kstat_named_t controller; 3255 kstat_named_t factory_mac_address; 3256 }; 3257 3258 /* 3259 * cxgbe:X:info 3260 */ 3261 struct cxgbe_port_info_kstats { 3262 kstat_named_t transceiver; 3263 kstat_named_t rx_ovflow0; 3264 kstat_named_t rx_ovflow1; 3265 kstat_named_t rx_ovflow2; 3266 kstat_named_t rx_ovflow3; 3267 kstat_named_t rx_trunc0; 3268 kstat_named_t rx_trunc1; 3269 kstat_named_t rx_trunc2; 3270 kstat_named_t rx_trunc3; 3271 kstat_named_t tx_pause; 3272 kstat_named_t rx_pause; 3273 }; 3274 3275 static kstat_t * 3276 setup_port_config_kstats(struct port_info *pi) 3277 { 3278 kstat_t *ksp; 3279 struct cxgbe_port_config_kstats *kstatp; 3280 int ndata; 3281 dev_info_t *pdip = ddi_get_parent(pi->dip); 3282 uint8_t *ma = &pi->hw_addr[0]; 3283 3284 ndata = sizeof (struct cxgbe_port_config_kstats) / 3285 sizeof (kstat_named_t); 3286 3287 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "config", 3288 "net", KSTAT_TYPE_NAMED, ndata, 0); 3289 if (ksp == NULL) { 3290 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats."); 3291 return (NULL); 3292 } 3293 3294 kstatp = (struct cxgbe_port_config_kstats *)ksp->ks_data; 3295 3296 KS_UINIT(idx); 3297 KS_UINIT(nrxq); 3298 KS_UINIT(ntxq); 3299 KS_UINIT(first_rxq); 3300 KS_UINIT(first_txq); 3301 KS_CINIT(controller); 3302 KS_CINIT(factory_mac_address); 3303 3304 KS_U_SET(idx, pi->port_id); 3305 KS_U_SET(nrxq, pi->nrxq); 3306 KS_U_SET(ntxq, pi->ntxq); 3307 KS_U_SET(first_rxq, pi->first_rxq); 3308 KS_U_SET(first_txq, pi->first_txq); 3309 KS_C_SET(controller, "%s%d", ddi_driver_name(pdip), 3310 ddi_get_instance(pdip)); 3311 KS_C_SET(factory_mac_address, "%02X%02X%02X%02X%02X%02X", 3312 ma[0], ma[1], ma[2], ma[3], ma[4], ma[5]); 3313 3314 /* Do NOT set ksp->ks_update. These kstats do not change. */ 3315 3316 /* Install the kstat */ 3317 ksp->ks_private = (void *)pi; 3318 kstat_install(ksp); 3319 3320 return (ksp); 3321 } 3322 3323 static kstat_t * 3324 setup_port_info_kstats(struct port_info *pi) 3325 { 3326 kstat_t *ksp; 3327 struct cxgbe_port_info_kstats *kstatp; 3328 int ndata; 3329 3330 ndata = sizeof (struct cxgbe_port_info_kstats) / sizeof (kstat_named_t); 3331 3332 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "info", 3333 "net", KSTAT_TYPE_NAMED, ndata, 0); 3334 if (ksp == NULL) { 3335 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats."); 3336 return (NULL); 3337 } 3338 3339 kstatp = (struct cxgbe_port_info_kstats *)ksp->ks_data; 3340 3341 KS_CINIT(transceiver); 3342 KS_UINIT(rx_ovflow0); 3343 KS_UINIT(rx_ovflow1); 3344 KS_UINIT(rx_ovflow2); 3345 KS_UINIT(rx_ovflow3); 3346 KS_UINIT(rx_trunc0); 3347 KS_UINIT(rx_trunc1); 3348 KS_UINIT(rx_trunc2); 3349 KS_UINIT(rx_trunc3); 3350 KS_UINIT(tx_pause); 3351 KS_UINIT(rx_pause); 3352 3353 /* Install the kstat */ 3354 ksp->ks_update = update_port_info_kstats; 3355 ksp->ks_private = (void *)pi; 3356 kstat_install(ksp); 3357 3358 return (ksp); 3359 } 3360 3361 static int 3362 update_port_info_kstats(kstat_t *ksp, int rw) 3363 { 3364 struct cxgbe_port_info_kstats *kstatp = 3365 (struct cxgbe_port_info_kstats *)ksp->ks_data; 3366 struct port_info *pi = ksp->ks_private; 3367 static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX", 3368 "active TWINAX", "LRM" }; 3369 uint32_t bgmap; 3370 3371 if (rw == KSTAT_WRITE) 3372 return (0); 3373 3374 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 3375 KS_C_SET(transceiver, "unplugged"); 3376 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 3377 KS_C_SET(transceiver, "unknown"); 3378 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 3379 KS_C_SET(transceiver, "unsupported"); 3380 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str)) 3381 KS_C_SET(transceiver, "%s", mod_str[pi->mod_type]); 3382 else 3383 KS_C_SET(transceiver, "type %d", pi->mod_type); 3384 3385 #define GET_STAT(name) t4_read_reg64(pi->adapter, \ 3386 PORT_REG(pi->port_id, A_MPS_PORT_STAT_##name##_L)) 3387 #define GET_STAT_COM(name) t4_read_reg64(pi->adapter, \ 3388 A_MPS_STAT_##name##_L) 3389 3390 bgmap = G_NUMPORTS(t4_read_reg(pi->adapter, A_MPS_CMN_CTL)); 3391 if (bgmap == 0) 3392 bgmap = (pi->port_id == 0) ? 0xf : 0; 3393 else if (bgmap == 1) 3394 bgmap = (pi->port_id < 2) ? (3 << (2 * pi->port_id)) : 0; 3395 else 3396 bgmap = 1; 3397 3398 KS_U_SET(rx_ovflow0, (bgmap & 1) ? 3399 GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0); 3400 KS_U_SET(rx_ovflow1, (bgmap & 2) ? 3401 GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0); 3402 KS_U_SET(rx_ovflow2, (bgmap & 4) ? 3403 GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0); 3404 KS_U_SET(rx_ovflow3, (bgmap & 8) ? 3405 GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0); 3406 KS_U_SET(rx_trunc0, (bgmap & 1) ? 3407 GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0); 3408 KS_U_SET(rx_trunc1, (bgmap & 2) ? 3409 GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0); 3410 KS_U_SET(rx_trunc2, (bgmap & 4) ? 3411 GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0); 3412 KS_U_SET(rx_trunc3, (bgmap & 8) ? 3413 GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0); 3414 3415 KS_U_SET(tx_pause, GET_STAT(TX_PORT_PAUSE)); 3416 KS_U_SET(rx_pause, GET_STAT(RX_PORT_PAUSE)); 3417 3418 return (0); 3419 3420 } 3421 3422 /* 3423 * cxgbe:X:rxqY 3424 */ 3425 struct rxq_kstats { 3426 kstat_named_t rxcsum; 3427 kstat_named_t rxpkts; 3428 kstat_named_t rxbytes; 3429 kstat_named_t nomem; 3430 }; 3431 3432 static kstat_t * 3433 setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, int idx) 3434 { 3435 struct kstat *ksp; 3436 struct rxq_kstats *kstatp; 3437 int ndata; 3438 char str[16]; 3439 3440 ndata = sizeof (struct rxq_kstats) / sizeof (kstat_named_t); 3441 (void) snprintf(str, sizeof (str), "rxq%u", idx); 3442 3443 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "rxq", 3444 KSTAT_TYPE_NAMED, ndata, 0); 3445 if (ksp == NULL) { 3446 cxgb_printf(pi->dip, CE_WARN, 3447 "%s: failed to initialize rxq kstats for queue %d.", 3448 __func__, idx); 3449 return (NULL); 3450 } 3451 3452 kstatp = (struct rxq_kstats *)ksp->ks_data; 3453 3454 KS_UINIT(rxcsum); 3455 KS_UINIT(rxpkts); 3456 KS_UINIT(rxbytes); 3457 KS_UINIT(nomem); 3458 3459 ksp->ks_update = update_rxq_kstats; 3460 ksp->ks_private = (void *)rxq; 3461 kstat_install(ksp); 3462 3463 return (ksp); 3464 } 3465 3466 static int 3467 update_rxq_kstats(kstat_t *ksp, int rw) 3468 { 3469 struct rxq_kstats *kstatp = (struct rxq_kstats *)ksp->ks_data; 3470 struct sge_rxq *rxq = ksp->ks_private; 3471 3472 if (rw == KSTAT_WRITE) 3473 return (0); 3474 3475 KS_U_FROM(rxcsum, rxq); 3476 KS_U_FROM(rxpkts, rxq); 3477 KS_U_FROM(rxbytes, rxq); 3478 KS_U_FROM(nomem, rxq); 3479 3480 return (0); 3481 } 3482 3483 /* 3484 * cxgbe:X:txqY 3485 */ 3486 struct txq_kstats { 3487 kstat_named_t txcsum; 3488 kstat_named_t tso_wrs; 3489 kstat_named_t imm_wrs; 3490 kstat_named_t sgl_wrs; 3491 kstat_named_t txpkt_wrs; 3492 kstat_named_t txpkts_wrs; 3493 kstat_named_t txpkts_pkts; 3494 kstat_named_t txb_used; 3495 kstat_named_t hdl_used; 3496 kstat_named_t txb_full; 3497 kstat_named_t dma_hdl_failed; 3498 kstat_named_t dma_map_failed; 3499 kstat_named_t qfull; 3500 kstat_named_t qflush; 3501 kstat_named_t pullup_early; 3502 kstat_named_t pullup_late; 3503 kstat_named_t pullup_failed; 3504 kstat_named_t csum_failed; 3505 }; 3506 3507 static kstat_t * 3508 setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, int idx) 3509 { 3510 struct kstat *ksp; 3511 struct txq_kstats *kstatp; 3512 int ndata; 3513 char str[16]; 3514 3515 ndata = sizeof (struct txq_kstats) / sizeof (kstat_named_t); 3516 (void) snprintf(str, sizeof (str), "txq%u", idx); 3517 3518 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "txq", 3519 KSTAT_TYPE_NAMED, ndata, 0); 3520 if (ksp == NULL) { 3521 cxgb_printf(pi->dip, CE_WARN, 3522 "%s: failed to initialize txq kstats for queue %d.", 3523 __func__, idx); 3524 return (NULL); 3525 } 3526 3527 kstatp = (struct txq_kstats *)ksp->ks_data; 3528 3529 KS_UINIT(txcsum); 3530 KS_UINIT(tso_wrs); 3531 KS_UINIT(imm_wrs); 3532 KS_UINIT(sgl_wrs); 3533 KS_UINIT(txpkt_wrs); 3534 KS_UINIT(txpkts_wrs); 3535 KS_UINIT(txpkts_pkts); 3536 KS_UINIT(txb_used); 3537 KS_UINIT(hdl_used); 3538 KS_UINIT(txb_full); 3539 KS_UINIT(dma_hdl_failed); 3540 KS_UINIT(dma_map_failed); 3541 KS_UINIT(qfull); 3542 KS_UINIT(qflush); 3543 KS_UINIT(pullup_early); 3544 KS_UINIT(pullup_late); 3545 KS_UINIT(pullup_failed); 3546 KS_UINIT(csum_failed); 3547 3548 ksp->ks_update = update_txq_kstats; 3549 ksp->ks_private = (void *)txq; 3550 kstat_install(ksp); 3551 3552 return (ksp); 3553 } 3554 3555 static int 3556 update_txq_kstats(kstat_t *ksp, int rw) 3557 { 3558 struct txq_kstats *kstatp = (struct txq_kstats *)ksp->ks_data; 3559 struct sge_txq *txq = ksp->ks_private; 3560 3561 if (rw == KSTAT_WRITE) 3562 return (0); 3563 3564 KS_U_FROM(txcsum, txq); 3565 KS_U_FROM(tso_wrs, txq); 3566 KS_U_FROM(imm_wrs, txq); 3567 KS_U_FROM(sgl_wrs, txq); 3568 KS_U_FROM(txpkt_wrs, txq); 3569 KS_U_FROM(txpkts_wrs, txq); 3570 KS_U_FROM(txpkts_pkts, txq); 3571 KS_U_FROM(txb_used, txq); 3572 KS_U_FROM(hdl_used, txq); 3573 KS_U_FROM(txb_full, txq); 3574 KS_U_FROM(dma_hdl_failed, txq); 3575 KS_U_FROM(dma_map_failed, txq); 3576 KS_U_FROM(qfull, txq); 3577 KS_U_FROM(qflush, txq); 3578 KS_U_FROM(pullup_early, txq); 3579 KS_U_FROM(pullup_late, txq); 3580 KS_U_FROM(pullup_failed, txq); 3581 KS_U_FROM(csum_failed, txq); 3582 3583 return (0); 3584 } 3585 3586 static int rxbuf_ctor(void *, void *, int); 3587 static void rxbuf_dtor(void *, void *); 3588 3589 static kmem_cache_t * 3590 rxbuf_cache_create(struct rxbuf_cache_params *p) 3591 { 3592 char name[32]; 3593 3594 (void) snprintf(name, sizeof (name), "%s%d_rxbuf_cache", 3595 ddi_driver_name(p->dip), ddi_get_instance(p->dip)); 3596 3597 return kmem_cache_create(name, sizeof (struct rxbuf), _CACHE_LINE_SIZE, 3598 rxbuf_ctor, rxbuf_dtor, NULL, p, NULL, 0); 3599 } 3600 3601 /* 3602 * If ref_cnt is more than 1 then those many calls to rxbuf_free will 3603 * have to be made before the rxb is released back to the kmem_cache. 3604 */ 3605 static struct rxbuf * 3606 rxbuf_alloc(kmem_cache_t *cache, int kmflags, uint_t ref_cnt) 3607 { 3608 struct rxbuf *rxb; 3609 3610 ASSERT(ref_cnt > 0); 3611 3612 rxb = kmem_cache_alloc(cache, kmflags); 3613 if (rxb != NULL) { 3614 rxb->ref_cnt = ref_cnt; 3615 rxb->cache = cache; 3616 } 3617 3618 return (rxb); 3619 } 3620 3621 /* 3622 * This is normally called via the rxb's freefunc, when an mblk referencing the 3623 * rxb is freed. 3624 */ 3625 static void 3626 rxbuf_free(struct rxbuf *rxb) 3627 { 3628 if (atomic_dec_uint_nv(&rxb->ref_cnt) == 0) 3629 kmem_cache_free(rxb->cache, rxb); 3630 } 3631 3632 static int 3633 rxbuf_ctor(void *arg1, void *arg2, int kmflag) 3634 { 3635 struct rxbuf *rxb = arg1; 3636 struct rxbuf_cache_params *p = arg2; 3637 size_t real_len; 3638 ddi_dma_cookie_t cookie; 3639 uint_t ccount = 0; 3640 int (*callback)(caddr_t); 3641 int rc = ENOMEM; 3642 3643 if ((kmflag & KM_NOSLEEP) != 0) 3644 callback = DDI_DMA_DONTWAIT; 3645 else 3646 callback = DDI_DMA_SLEEP; 3647 3648 rc = ddi_dma_alloc_handle(p->dip, &p->dma_attr_rx, callback, 0, 3649 &rxb->dhdl); 3650 if (rc != DDI_SUCCESS) 3651 return (rc == DDI_DMA_BADATTR ? EINVAL : ENOMEM); 3652 3653 rc = ddi_dma_mem_alloc(rxb->dhdl, p->buf_size, &p->acc_attr_rx, 3654 DDI_DMA_STREAMING, callback, 0, &rxb->va, &real_len, &rxb->ahdl); 3655 if (rc != DDI_SUCCESS) { 3656 rc = ENOMEM; 3657 goto fail1; 3658 } 3659 3660 rc = ddi_dma_addr_bind_handle(rxb->dhdl, NULL, rxb->va, p->buf_size, 3661 DDI_DMA_READ | DDI_DMA_STREAMING, NULL, NULL, &cookie, &ccount); 3662 if (rc != DDI_DMA_MAPPED) { 3663 if (rc == DDI_DMA_INUSE) 3664 rc = EBUSY; 3665 else if (rc == DDI_DMA_TOOBIG) 3666 rc = E2BIG; 3667 else 3668 rc = ENOMEM; 3669 goto fail2; 3670 } 3671 3672 if (ccount != 1) { 3673 rc = E2BIG; 3674 goto fail3; 3675 } 3676 3677 rxb->ref_cnt = 0; 3678 rxb->buf_size = p->buf_size; 3679 rxb->freefunc.free_arg = (caddr_t)rxb; 3680 rxb->freefunc.free_func = rxbuf_free; 3681 rxb->ba = cookie.dmac_laddress; 3682 3683 return (0); 3684 3685 fail3: (void) ddi_dma_unbind_handle(rxb->dhdl); 3686 fail2: ddi_dma_mem_free(&rxb->ahdl); 3687 fail1: ddi_dma_free_handle(&rxb->dhdl); 3688 return (rc); 3689 } 3690 3691 static void 3692 rxbuf_dtor(void *arg1, void *arg2) 3693 { 3694 struct rxbuf *rxb = arg1; 3695 3696 (void) ddi_dma_unbind_handle(rxb->dhdl); 3697 ddi_dma_mem_free(&rxb->ahdl); 3698 ddi_dma_free_handle(&rxb->dhdl); 3699 } 3700