1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * This file is part of the Chelsio T4 support code. 14 * 15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved. 16 * 17 * This program is distributed in the hope that it will be useful, but WITHOUT 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this 20 * release for licensing terms and conditions. 21 */ 22 23 /* 24 * Copyright 2025 Oxide Computer Company 25 */ 26 27 #include <sys/ddi.h> 28 #include <sys/sunddi.h> 29 #include <sys/sunndi.h> 30 #include <sys/atomic.h> 31 #include <sys/dlpi.h> 32 #include <sys/pattr.h> 33 #include <sys/strsubr.h> 34 #include <sys/stream.h> 35 #include <sys/strsun.h> 36 #include <inet/ip.h> 37 #include <inet/tcp.h> 38 39 #include "common/common.h" 40 #include "common/t4_msg.h" 41 #include "common/t4_regs.h" 42 #include "common/t4_regs_values.h" 43 44 /* TODO: Tune. */ 45 int rx_buf_size = 8192; 46 int tx_copy_threshold = 256; 47 uint16_t rx_copy_threshold = 256; 48 49 /* Used to track coalesced tx work request */ 50 struct txpkts { 51 mblk_t *tail; /* head is in the software descriptor */ 52 uint64_t *flitp; /* ptr to flit where next pkt should start */ 53 uint8_t npkt; /* # of packets in this work request */ 54 uint8_t nflits; /* # of flits used by this work request */ 55 uint16_t plen; /* total payload (sum of all packets) */ 56 }; 57 58 /* All information needed to tx a frame */ 59 struct txinfo { 60 uint32_t len; /* Total length of frame */ 61 uint32_t flags; /* Checksum and LSO flags */ 62 uint32_t mss; /* MSS for LSO */ 63 uint8_t nsegs; /* # of segments in the SGL, 0 means imm. tx */ 64 uint8_t nflits; /* # of flits needed for the SGL */ 65 uint8_t hdls_used; /* # of DMA handles used */ 66 uint32_t txb_used; /* txb_space used */ 67 mac_ether_offload_info_t meoi; /* pkt hdr info for offloads */ 68 struct ulptx_sgl sgl __attribute__((aligned(8))); 69 struct ulptx_sge_pair reserved[TX_SGL_SEGS / 2]; 70 }; 71 72 struct mblk_pair { 73 mblk_t *head, *tail; 74 }; 75 76 struct rxbuf { 77 kmem_cache_t *cache; /* the kmem_cache this rxb came from */ 78 ddi_dma_handle_t dhdl; 79 ddi_acc_handle_t ahdl; 80 caddr_t va; /* KVA of buffer */ 81 uint64_t ba; /* bus address of buffer */ 82 frtn_t freefunc; 83 uint_t buf_size; 84 volatile uint_t ref_cnt; 85 }; 86 87 static int service_iq(struct sge_iq *iq, int budget); 88 static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, 89 int8_t pktc_idx, int qsize, uint8_t esize); 90 static inline void init_fl(struct sge_fl *fl, uint16_t qsize); 91 static inline void init_eq(struct adapter *sc, struct sge_eq *eq, 92 uint16_t qsize, uint8_t tx_chan, uint16_t iqid); 93 static int alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, 94 struct sge_fl *fl, int intr_idx, int cong); 95 static int free_iq_fl(struct port_info *pi, struct sge_iq *iq, 96 struct sge_fl *fl); 97 static int alloc_fwq(struct adapter *sc); 98 static int free_fwq(struct adapter *sc); 99 static int alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, 100 int i); 101 static int free_rxq(struct port_info *pi, struct sge_rxq *rxq); 102 static int eth_eq_alloc(struct adapter *sc, struct port_info *pi, 103 struct sge_eq *eq); 104 static int alloc_eq(struct adapter *sc, struct port_info *pi, 105 struct sge_eq *eq); 106 static int free_eq(struct adapter *sc, struct sge_eq *eq); 107 static int alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx); 108 static int free_txq(struct port_info *pi, struct sge_txq *txq); 109 static int alloc_dma_memory(struct adapter *sc, size_t len, int flags, 110 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr, 111 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 112 caddr_t *pva); 113 static int free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl); 114 static int alloc_desc_ring(struct adapter *sc, size_t len, int rw, 115 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 116 caddr_t *pva); 117 static int free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl); 118 static int alloc_tx_copybuffer(struct adapter *sc, size_t len, 119 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 120 caddr_t *pva); 121 static inline bool is_new_response(const struct sge_iq *iq, 122 struct rsp_ctrl **ctrl); 123 static inline void iq_next(struct sge_iq *iq); 124 static int refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs); 125 static void refill_sfl(void *arg); 126 static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl); 127 static void free_fl_bufs(struct sge_fl *fl); 128 static mblk_t *get_fl_payload(struct adapter *sc, struct sge_fl *fl, 129 uint32_t len_newbuf, int *fl_bufs_used); 130 static int get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, 131 struct txinfo *txinfo, int sgl_only); 132 static inline int fits_in_txb(struct sge_txq *txq, int len, int *waste); 133 static inline int copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, 134 struct txinfo *txinfo); 135 static inline void add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len); 136 static inline int add_mblk(struct sge_txq *txq, struct txinfo *txinfo, 137 mblk_t *m, int len); 138 static void free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo); 139 static int add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m, 140 struct txinfo *txinfo); 141 static void write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts); 142 static int write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m, 143 struct txinfo *txinfo); 144 static inline void write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq, 145 struct txpkts *txpkts, struct txinfo *txinfo); 146 static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, 147 int len); 148 static inline void ring_tx_db(struct adapter *sc, struct sge_eq *eq); 149 static int reclaim_tx_descs(struct sge_txq *txq, int howmany); 150 static void write_txqflush_wr(struct sge_txq *txq); 151 static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, 152 mblk_t *m); 153 static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl); 154 static kstat_t *setup_port_config_kstats(struct port_info *pi); 155 static kstat_t *setup_port_info_kstats(struct port_info *pi); 156 static kstat_t *setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, 157 int idx); 158 static int update_rxq_kstats(kstat_t *ksp, int rw); 159 static int update_port_info_kstats(kstat_t *ksp, int rw); 160 static kstat_t *setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, 161 int idx); 162 static int update_txq_kstats(kstat_t *ksp, int rw); 163 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *, 164 mblk_t *); 165 static int t4_handle_cpl_msg(struct sge_iq *, const struct rss_header *, 166 mblk_t *); 167 static int t4_handle_fw_msg(struct sge_iq *, const struct rss_header *); 168 169 static kmem_cache_t *rxbuf_cache_create(struct rxbuf_cache_params *); 170 static struct rxbuf *rxbuf_alloc(kmem_cache_t *, int, uint_t); 171 static void rxbuf_free(struct rxbuf *); 172 static int rxbuf_ctor(void *, void *, int); 173 static void rxbuf_dtor(void *, void *); 174 175 static inline int 176 reclaimable(struct sge_eq *eq) 177 { 178 unsigned int cidx; 179 180 cidx = eq->spg->cidx; /* stable snapshot */ 181 cidx = be16_to_cpu(cidx); 182 183 if (cidx >= eq->cidx) 184 return (cidx - eq->cidx); 185 else 186 return (cidx + eq->cap - eq->cidx); 187 } 188 189 void 190 t4_sge_init(struct adapter *sc) 191 { 192 struct driver_properties *p = &sc->props; 193 ddi_dma_attr_t *dma_attr; 194 ddi_device_acc_attr_t *acc_attr; 195 uint32_t sge_control, sge_conm_ctrl; 196 int egress_threshold; 197 198 /* 199 * Device access and DMA attributes for descriptor rings 200 */ 201 acc_attr = &sc->sge.acc_attr_desc; 202 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 203 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 204 acc_attr->devacc_attr_dataorder = DDI_STRICTORDER_ACC; 205 206 dma_attr = &sc->sge.dma_attr_desc; 207 dma_attr->dma_attr_version = DMA_ATTR_V0; 208 dma_attr->dma_attr_addr_lo = 0; 209 dma_attr->dma_attr_addr_hi = UINT64_MAX; 210 dma_attr->dma_attr_count_max = UINT64_MAX; 211 dma_attr->dma_attr_align = 512; 212 dma_attr->dma_attr_burstsizes = 0xfff; 213 dma_attr->dma_attr_minxfer = 1; 214 dma_attr->dma_attr_maxxfer = UINT64_MAX; 215 dma_attr->dma_attr_seg = UINT64_MAX; 216 dma_attr->dma_attr_sgllen = 1; 217 dma_attr->dma_attr_granular = 1; 218 dma_attr->dma_attr_flags = 0; 219 220 /* 221 * Device access and DMA attributes for tx buffers 222 */ 223 acc_attr = &sc->sge.acc_attr_tx; 224 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 225 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 226 227 dma_attr = &sc->sge.dma_attr_tx; 228 dma_attr->dma_attr_version = DMA_ATTR_V0; 229 dma_attr->dma_attr_addr_lo = 0; 230 dma_attr->dma_attr_addr_hi = UINT64_MAX; 231 dma_attr->dma_attr_count_max = UINT64_MAX; 232 dma_attr->dma_attr_align = 1; 233 dma_attr->dma_attr_burstsizes = 0xfff; 234 dma_attr->dma_attr_minxfer = 1; 235 dma_attr->dma_attr_maxxfer = UINT64_MAX; 236 dma_attr->dma_attr_seg = UINT64_MAX; 237 dma_attr->dma_attr_sgllen = TX_SGL_SEGS; 238 dma_attr->dma_attr_granular = 1; 239 dma_attr->dma_attr_flags = 0; 240 241 /* 242 * Ingress Padding Boundary and Egress Status Page Size are set up by 243 * t4_fixup_host_params(). 244 */ 245 sge_control = t4_read_reg(sc, A_SGE_CONTROL); 246 sc->sge.pktshift = G_PKTSHIFT(sge_control); 247 sc->sge.stat_len = (sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64; 248 249 /* t4_nex uses FLM packed mode */ 250 sc->sge.fl_align = t4_fl_pkt_align(sc, true); 251 252 /* 253 * Device access and DMA attributes for rx buffers 254 */ 255 sc->sge.rxb_params.dip = sc->dip; 256 sc->sge.rxb_params.buf_size = rx_buf_size; 257 258 acc_attr = &sc->sge.rxb_params.acc_attr_rx; 259 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 260 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 261 262 dma_attr = &sc->sge.rxb_params.dma_attr_rx; 263 dma_attr->dma_attr_version = DMA_ATTR_V0; 264 dma_attr->dma_attr_addr_lo = 0; 265 dma_attr->dma_attr_addr_hi = UINT64_MAX; 266 dma_attr->dma_attr_count_max = UINT64_MAX; 267 /* 268 * Low 4 bits of an rx buffer address have a special meaning to the SGE 269 * and an rx buf cannot have an address with any of these bits set. 270 * FL_ALIGN is >= 32 so we're sure things are ok. 271 */ 272 dma_attr->dma_attr_align = sc->sge.fl_align; 273 dma_attr->dma_attr_burstsizes = 0xfff; 274 dma_attr->dma_attr_minxfer = 1; 275 dma_attr->dma_attr_maxxfer = UINT64_MAX; 276 dma_attr->dma_attr_seg = UINT64_MAX; 277 dma_attr->dma_attr_sgllen = 1; 278 dma_attr->dma_attr_granular = 1; 279 dma_attr->dma_attr_flags = 0; 280 281 sc->sge.rxbuf_cache = rxbuf_cache_create(&sc->sge.rxb_params); 282 283 /* 284 * A FL with <= fl_starve_thres buffers is starving and a periodic 285 * timer will attempt to refill it. This needs to be larger than the 286 * SGE's Egress Congestion Threshold. If it isn't, then we can get 287 * stuck waiting for new packets while the SGE is waiting for us to 288 * give it more Free List entries. (Note that the SGE's Egress 289 * Congestion Threshold is in units of 2 Free List pointers.) For T4, 290 * there was only a single field to control this. For T5 there's the 291 * original field which now only applies to Unpacked Mode Free List 292 * buffers and a new field which only applies to Packed Mode Free List 293 * buffers. 294 */ 295 296 sge_conm_ctrl = t4_read_reg(sc, A_SGE_CONM_CTRL); 297 switch (CHELSIO_CHIP_VERSION(sc->params.chip)) { 298 case CHELSIO_T4: 299 egress_threshold = G_EGRTHRESHOLD(sge_conm_ctrl); 300 break; 301 case CHELSIO_T5: 302 egress_threshold = G_EGRTHRESHOLDPACKING(sge_conm_ctrl); 303 break; 304 case CHELSIO_T6: 305 default: 306 egress_threshold = G_T6_EGRTHRESHOLDPACKING(sge_conm_ctrl); 307 } 308 sc->sge.fl_starve_threshold = 2*egress_threshold + 1; 309 310 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, rx_buf_size); 311 312 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, 313 V_THRESHOLD_0(p->counter_val[0]) | 314 V_THRESHOLD_1(p->counter_val[1]) | 315 V_THRESHOLD_2(p->counter_val[2]) | 316 V_THRESHOLD_3(p->counter_val[3])); 317 318 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, 319 V_TIMERVALUE0(us_to_core_ticks(sc, p->timer_val[0])) | 320 V_TIMERVALUE1(us_to_core_ticks(sc, p->timer_val[1]))); 321 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, 322 V_TIMERVALUE2(us_to_core_ticks(sc, p->timer_val[2])) | 323 V_TIMERVALUE3(us_to_core_ticks(sc, p->timer_val[3]))); 324 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, 325 V_TIMERVALUE4(us_to_core_ticks(sc, p->timer_val[4])) | 326 V_TIMERVALUE5(us_to_core_ticks(sc, p->timer_val[5]))); 327 } 328 329 /* 330 * Allocate and initialize the firmware event queue and the forwarded interrupt 331 * queues, if any. The adapter owns all these queues as they are not associated 332 * with any particular port. 333 * 334 * Returns errno on failure. Resources allocated up to that point may still be 335 * allocated. Caller is responsible for cleanup in case this function fails. 336 */ 337 int 338 t4_setup_adapter_queues(struct adapter *sc) 339 { 340 int rc; 341 342 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 343 344 /* 345 * Firmware event queue 346 */ 347 rc = alloc_fwq(sc); 348 if (rc != 0) 349 return (rc); 350 351 return (rc); 352 } 353 354 /* 355 * Idempotent 356 */ 357 int 358 t4_teardown_adapter_queues(struct adapter *sc) 359 { 360 361 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 362 363 (void) free_fwq(sc); 364 365 return (0); 366 } 367 368 static inline int 369 first_vector(struct port_info *pi) 370 { 371 struct adapter *sc = pi->adapter; 372 int rc = T4_EXTRA_INTR, i; 373 374 if (sc->intr_count == 1) 375 return (0); 376 377 for_each_port(sc, i) { 378 struct port_info *p = sc->port[i]; 379 380 if (i == pi->port_id) 381 break; 382 383 /* 384 * Not compiled with offload support and intr_count > 1. Only 385 * NIC queues exist and they'd better be taking direct 386 * interrupts. 387 */ 388 ASSERT(!(sc->flags & TAF_INTR_FWD)); 389 rc += p->nrxq; 390 } 391 return (rc); 392 } 393 394 /* 395 * Given an arbitrary "index," come up with an iq that can be used by other 396 * queues (of this port) for interrupt forwarding, SGE egress updates, etc. 397 * The iq returned is guaranteed to be something that takes direct interrupts. 398 */ 399 static struct sge_iq * 400 port_intr_iq(struct port_info *pi, int idx) 401 { 402 struct adapter *sc = pi->adapter; 403 struct sge *s = &sc->sge; 404 struct sge_iq *iq = NULL; 405 406 if (sc->intr_count == 1) 407 return (&sc->sge.fwq); 408 409 /* 410 * Not compiled with offload support and intr_count > 1. Only NIC 411 * queues exist and they'd better be taking direct interrupts. 412 */ 413 ASSERT(!(sc->flags & TAF_INTR_FWD)); 414 415 idx %= pi->nrxq; 416 iq = &s->rxq[pi->first_rxq + idx].iq; 417 418 return (iq); 419 } 420 421 int 422 t4_setup_port_queues(struct port_info *pi) 423 { 424 int rc = 0, i, intr_idx, j; 425 struct sge_rxq *rxq; 426 struct sge_txq *txq; 427 struct adapter *sc = pi->adapter; 428 struct driver_properties *p = &sc->props; 429 430 pi->ksp_config = setup_port_config_kstats(pi); 431 pi->ksp_info = setup_port_info_kstats(pi); 432 433 /* Interrupt vector to start from (when using multiple vectors) */ 434 intr_idx = first_vector(pi); 435 436 /* 437 * First pass over all rx queues (NIC and TOE): 438 * a) initialize iq and fl 439 * b) allocate queue iff it will take direct interrupts. 440 */ 441 442 for_each_rxq(pi, i, rxq) { 443 444 init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, p->qsize_rxq, 445 RX_IQ_ESIZE); 446 447 init_fl(&rxq->fl, p->qsize_rxq / 8); /* 8 bufs in each entry */ 448 449 if ((!(sc->flags & TAF_INTR_FWD)) || 450 (sc->intr_count > 1 && pi->nrxq)) { 451 rxq->iq.flags |= IQ_INTR; 452 rc = alloc_rxq(pi, rxq, intr_idx, i); 453 if (rc != 0) 454 goto done; 455 intr_idx++; 456 } 457 458 } 459 460 /* 461 * Second pass over all rx queues (NIC and TOE). The queues forwarding 462 * their interrupts are allocated now. 463 */ 464 j = 0; 465 for_each_rxq(pi, i, rxq) { 466 if (rxq->iq.flags & IQ_INTR) 467 continue; 468 469 intr_idx = port_intr_iq(pi, j)->abs_id; 470 471 rc = alloc_rxq(pi, rxq, intr_idx, i); 472 if (rc != 0) 473 goto done; 474 j++; 475 } 476 477 /* 478 * Now the tx queues. Only one pass needed. 479 */ 480 j = 0; 481 for_each_txq(pi, i, txq) { 482 uint16_t iqid; 483 484 iqid = port_intr_iq(pi, j)->cntxt_id; 485 init_eq(sc, &txq->eq, p->qsize_txq, pi->tx_chan, iqid); 486 rc = alloc_txq(pi, txq, i); 487 if (rc != 0) 488 goto done; 489 } 490 491 done: 492 if (rc != 0) 493 (void) t4_teardown_port_queues(pi); 494 495 return (rc); 496 } 497 498 /* 499 * Idempotent 500 */ 501 int 502 t4_teardown_port_queues(struct port_info *pi) 503 { 504 int i; 505 struct sge_rxq *rxq; 506 struct sge_txq *txq; 507 508 if (pi->ksp_config != NULL) { 509 kstat_delete(pi->ksp_config); 510 pi->ksp_config = NULL; 511 } 512 if (pi->ksp_info != NULL) { 513 kstat_delete(pi->ksp_info); 514 pi->ksp_info = NULL; 515 } 516 517 for_each_txq(pi, i, txq) { 518 (void) free_txq(pi, txq); 519 } 520 521 for_each_rxq(pi, i, rxq) { 522 if ((rxq->iq.flags & IQ_INTR) == 0) 523 (void) free_rxq(pi, rxq); 524 } 525 526 /* 527 * Then take down the rx queues that take direct interrupts. 528 */ 529 530 for_each_rxq(pi, i, rxq) { 531 if (rxq->iq.flags & IQ_INTR) 532 (void) free_rxq(pi, rxq); 533 } 534 535 return (0); 536 } 537 538 /* Deals with errors and forwarded interrupts */ 539 uint_t 540 t4_intr_all(caddr_t arg1, caddr_t arg2) 541 { 542 543 (void) t4_intr_err(arg1, arg2); 544 (void) t4_intr(arg1, arg2); 545 546 return (DDI_INTR_CLAIMED); 547 } 548 549 static void 550 t4_intr_rx_work(struct sge_iq *iq) 551 { 552 mblk_t *mp = NULL; 553 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */ 554 RXQ_LOCK(rxq); 555 if (!iq->polling) { 556 mp = t4_ring_rx(rxq, iq->qsize/8); 557 t4_write_reg(iq->adapter, MYPF_REG(A_SGE_PF_GTS), 558 V_INGRESSQID((u32)iq->cntxt_id) | 559 V_SEINTARM(iq->intr_next)); 560 } 561 RXQ_UNLOCK(rxq); 562 if (mp != NULL) { 563 mac_rx_ring(rxq->port->mh, rxq->ring_handle, mp, 564 rxq->ring_gen_num); 565 } 566 } 567 568 /* Deals with interrupts on the given ingress queue */ 569 /* ARGSUSED */ 570 uint_t 571 t4_intr(caddr_t arg1, caddr_t arg2) 572 { 573 struct sge_iq *iq = (struct sge_iq *)arg2; 574 int state; 575 576 /* 577 * Right now receive polling is only enabled for MSI-X and 578 * when we have enough msi-x vectors i.e no interrupt forwarding. 579 */ 580 if (iq->adapter->props.multi_rings) { 581 t4_intr_rx_work(iq); 582 } else { 583 state = atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY); 584 if (state == IQS_IDLE) { 585 (void) service_iq(iq, 0); 586 (void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE); 587 } 588 } 589 return (DDI_INTR_CLAIMED); 590 } 591 592 /* Deals with error interrupts */ 593 /* ARGSUSED */ 594 uint_t 595 t4_intr_err(caddr_t arg1, caddr_t arg2) 596 { 597 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 598 struct adapter *sc = (struct adapter *)arg1; 599 600 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0); 601 (void) t4_slow_intr_handler(sc); 602 603 return (DDI_INTR_CLAIMED); 604 } 605 606 /* 607 * t4_ring_rx - Process responses from an SGE response queue. 608 * 609 * This function processes responses from an SGE response queue up to the 610 * supplied budget. Responses include received packets as well as control 611 * messages from FW or HW. 612 * 613 * It returns a chain of mblks containing the received data, to be 614 * passed up to mac_rx_ring(). 615 */ 616 mblk_t * 617 t4_ring_rx(struct sge_rxq *rxq, int budget) 618 { 619 struct sge_iq *iq = &rxq->iq; 620 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */ 621 struct adapter *sc = iq->adapter; 622 struct rsp_ctrl *ctrl; 623 const struct rss_header *rss; 624 int ndescs = 0, fl_bufs_used = 0; 625 int rsp_type; 626 uint32_t lq; 627 mblk_t *mblk_head = NULL, **mblk_tail, *m; 628 struct cpl_rx_pkt *cpl; 629 uint32_t received_bytes = 0, pkt_len = 0; 630 bool csum_ok; 631 uint16_t err_vec; 632 633 mblk_tail = &mblk_head; 634 635 while (is_new_response(iq, &ctrl)) { 636 637 membar_consumer(); 638 639 m = NULL; 640 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen); 641 lq = be32_to_cpu(ctrl->pldbuflen_qid); 642 rss = (const void *)iq->cdesc; 643 644 switch (rsp_type) { 645 case X_RSPD_TYPE_FLBUF: 646 647 ASSERT(iq->flags & IQ_HAS_FL); 648 649 if (CPL_RX_PKT == rss->opcode) { 650 cpl = (void *)(rss + 1); 651 pkt_len = be16_to_cpu(cpl->len); 652 653 if (iq->polling && 654 ((received_bytes + pkt_len) > budget)) 655 goto done; 656 657 m = get_fl_payload(sc, fl, lq, &fl_bufs_used); 658 if (m == NULL) 659 goto done; 660 661 iq->intr_next = iq->intr_params; 662 m->b_rptr += sc->sge.pktshift; 663 if (sc->params.tp.rx_pkt_encap) { 664 /* Enabled only in T6 config file */ 665 err_vec = G_T6_COMPR_RXERR_VEC( 666 ntohs(cpl->err_vec)); 667 } else { 668 err_vec = ntohs(cpl->err_vec); 669 } 670 671 csum_ok = cpl->csum_calc && !err_vec; 672 673 /* TODO: what about cpl->ip_frag? */ 674 if (csum_ok && !cpl->ip_frag) { 675 mac_hcksum_set(m, 0, 0, 0, 0xffff, 676 HCK_FULLCKSUM_OK | HCK_FULLCKSUM | 677 HCK_IPV4_HDRCKSUM_OK); 678 rxq->rxcsum++; 679 } 680 rxq->rxpkts++; 681 rxq->rxbytes += pkt_len; 682 received_bytes += pkt_len; 683 684 *mblk_tail = m; 685 mblk_tail = &m->b_next; 686 687 break; 688 } 689 690 m = get_fl_payload(sc, fl, lq, &fl_bufs_used); 691 if (m == NULL) 692 goto done; 693 /* FALLTHROUGH */ 694 695 case X_RSPD_TYPE_CPL: 696 (void) t4_handle_cpl_msg(iq, rss, m); 697 break; 698 699 default: 700 break; 701 } 702 iq_next(iq); 703 ++ndescs; 704 if (!iq->polling && (ndescs == budget)) 705 break; 706 } 707 708 done: 709 710 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 711 V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) | 712 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); 713 714 if ((fl_bufs_used > 0) || (iq->flags & IQ_HAS_FL)) { 715 int starved; 716 FL_LOCK(fl); 717 fl->needed += fl_bufs_used; 718 starved = refill_fl(sc, fl, fl->cap / 8); 719 FL_UNLOCK(fl); 720 if (starved) 721 add_fl_to_sfl(sc, fl); 722 } 723 return (mblk_head); 724 } 725 726 /* 727 * Deals with anything and everything on the given ingress queue. 728 */ 729 static int 730 service_iq(struct sge_iq *iq, int budget) 731 { 732 struct sge_iq *q; 733 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */ 734 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */ 735 struct adapter *sc = iq->adapter; 736 struct rsp_ctrl *ctrl; 737 const struct rss_header *rss; 738 int ndescs = 0, limit, fl_bufs_used = 0; 739 int rsp_type; 740 uint32_t lq; 741 int starved; 742 mblk_t *m; 743 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql); 744 745 limit = budget ? budget : iq->qsize / 8; 746 747 /* 748 * We always come back and check the descriptor ring for new indirect 749 * interrupts and other responses after running a single handler. 750 */ 751 for (;;) { 752 while (is_new_response(iq, &ctrl)) { 753 754 membar_consumer(); 755 756 m = NULL; 757 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen); 758 lq = be32_to_cpu(ctrl->pldbuflen_qid); 759 rss = (const void *)iq->cdesc; 760 761 switch (rsp_type) { 762 case X_RSPD_TYPE_FLBUF: 763 764 ASSERT(iq->flags & IQ_HAS_FL); 765 766 m = get_fl_payload(sc, fl, lq, &fl_bufs_used); 767 if (m == NULL) { 768 /* 769 * Rearm the iq with a 770 * longer-than-default timer 771 */ 772 const uint32_t timer_idx = 773 V_QINTR_TIMER_IDX(SGE_NTIMERS-1); 774 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 775 V_CIDXINC(ndescs) | 776 V_INGRESSQID((u32)iq->cntxt_id) | 777 V_SEINTARM(timer_idx)); 778 if (fl_bufs_used > 0) { 779 ASSERT(iq->flags & IQ_HAS_FL); 780 FL_LOCK(fl); 781 fl->needed += fl_bufs_used; 782 starved = refill_fl(sc, fl, 783 fl->cap / 8); 784 FL_UNLOCK(fl); 785 if (starved) 786 add_fl_to_sfl(sc, fl); 787 } 788 return (0); 789 } 790 791 /* FALLTHRU */ 792 case X_RSPD_TYPE_CPL: 793 (void) t4_handle_cpl_msg(iq, rss, m); 794 break; 795 796 case X_RSPD_TYPE_INTR: 797 798 /* 799 * Interrupts should be forwarded only to queues 800 * that are not forwarding their interrupts. 801 * This means service_iq can recurse but only 1 802 * level deep. 803 */ 804 ASSERT(budget == 0); 805 806 q = sc->sge.iqmap[lq - sc->sge.iq_start]; 807 if (atomic_cas_uint(&q->state, IQS_IDLE, 808 IQS_BUSY) == IQS_IDLE) { 809 if (service_iq(q, q->qsize / 8) == 0) { 810 (void) atomic_cas_uint( 811 &q->state, IQS_BUSY, 812 IQS_IDLE); 813 } else { 814 STAILQ_INSERT_TAIL(&iql, q, 815 link); 816 } 817 } 818 break; 819 820 default: 821 break; 822 } 823 824 iq_next(iq); 825 if (++ndescs == limit) { 826 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 827 V_CIDXINC(ndescs) | 828 V_INGRESSQID(iq->cntxt_id) | 829 V_SEINTARM(V_QINTR_TIMER_IDX( 830 X_TIMERREG_UPDATE_CIDX))); 831 ndescs = 0; 832 833 if (fl_bufs_used > 0) { 834 ASSERT(iq->flags & IQ_HAS_FL); 835 FL_LOCK(fl); 836 fl->needed += fl_bufs_used; 837 (void) refill_fl(sc, fl, fl->cap / 8); 838 FL_UNLOCK(fl); 839 fl_bufs_used = 0; 840 } 841 842 if (budget != 0) 843 return (EINPROGRESS); 844 } 845 } 846 847 if (STAILQ_EMPTY(&iql) != 0) 848 break; 849 850 /* 851 * Process the head only, and send it to the back of the list if 852 * it's still not done. 853 */ 854 q = STAILQ_FIRST(&iql); 855 STAILQ_REMOVE_HEAD(&iql, link); 856 if (service_iq(q, q->qsize / 8) == 0) 857 (void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE); 858 else 859 STAILQ_INSERT_TAIL(&iql, q, link); 860 } 861 862 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) | 863 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next)); 864 865 if (iq->flags & IQ_HAS_FL) { 866 867 FL_LOCK(fl); 868 fl->needed += fl_bufs_used; 869 starved = refill_fl(sc, fl, fl->cap / 4); 870 FL_UNLOCK(fl); 871 if (starved != 0) 872 add_fl_to_sfl(sc, fl); 873 } 874 875 return (0); 876 } 877 878 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */ 879 #define TXPKTS_PKT_HDR ((\ 880 sizeof (struct ulp_txpkt) + \ 881 sizeof (struct ulptx_idata) + \ 882 sizeof (struct cpl_tx_pkt_core)) / 8) 883 884 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */ 885 #define TXPKTS_WR_HDR (\ 886 sizeof (struct fw_eth_tx_pkts_wr) / 8 + \ 887 TXPKTS_PKT_HDR) 888 889 /* Header of a tx WR, before SGL of first packet (in flits) */ 890 #define TXPKT_WR_HDR ((\ 891 sizeof (struct fw_eth_tx_pkt_wr) + \ 892 sizeof (struct cpl_tx_pkt_core)) / 8) 893 894 /* Header of a tx LSO WR, before SGL of first packet (in flits) */ 895 #define TXPKT_LSO_WR_HDR ((\ 896 sizeof (struct fw_eth_tx_pkt_wr) + \ 897 sizeof (struct cpl_tx_pkt_lso_core) + \ 898 sizeof (struct cpl_tx_pkt_core)) / 8) 899 900 mblk_t * 901 t4_eth_tx(void *arg, mblk_t *frame) 902 { 903 struct sge_txq *txq = (struct sge_txq *)arg; 904 struct port_info *pi = txq->port; 905 struct adapter *sc = pi->adapter; 906 struct sge_eq *eq = &txq->eq; 907 mblk_t *next_frame; 908 int rc, coalescing; 909 struct txpkts txpkts; 910 struct txinfo txinfo; 911 912 txpkts.npkt = 0; /* indicates there's nothing in txpkts */ 913 coalescing = 0; 914 915 TXQ_LOCK(txq); 916 if (eq->avail < 8) 917 (void) reclaim_tx_descs(txq, 8); 918 for (; frame; frame = next_frame) { 919 920 if (eq->avail < 8) 921 break; 922 923 next_frame = frame->b_next; 924 frame->b_next = NULL; 925 926 if (next_frame != NULL) 927 coalescing = 1; 928 929 rc = get_frame_txinfo(txq, &frame, &txinfo, coalescing); 930 if (rc != 0) { 931 if (rc == ENOMEM) { 932 933 /* Short of resources, suspend tx */ 934 935 frame->b_next = next_frame; 936 break; 937 } 938 939 /* 940 * Unrecoverable error for this frame, throw it 941 * away and move on to the next. 942 */ 943 944 freemsg(frame); 945 continue; 946 } 947 948 if (coalescing != 0 && 949 add_to_txpkts(txq, &txpkts, frame, &txinfo) == 0) { 950 951 /* Successfully absorbed into txpkts */ 952 953 write_ulp_cpl_sgl(pi, txq, &txpkts, &txinfo); 954 goto doorbell; 955 } 956 957 /* 958 * We weren't coalescing to begin with, or current frame could 959 * not be coalesced (add_to_txpkts flushes txpkts if a frame 960 * given to it can't be coalesced). Either way there should be 961 * nothing in txpkts. 962 */ 963 ASSERT(txpkts.npkt == 0); 964 965 /* We're sending out individual frames now */ 966 coalescing = 0; 967 968 if (eq->avail < 8) 969 (void) reclaim_tx_descs(txq, 8); 970 rc = write_txpkt_wr(pi, txq, frame, &txinfo); 971 if (rc != 0) { 972 973 /* Short of hardware descriptors, suspend tx */ 974 975 /* 976 * This is an unlikely but expensive failure. We've 977 * done all the hard work (DMA bindings etc.) and now we 978 * can't send out the frame. What's worse, we have to 979 * spend even more time freeing up everything in txinfo. 980 */ 981 txq->qfull++; 982 free_txinfo_resources(txq, &txinfo); 983 984 frame->b_next = next_frame; 985 break; 986 } 987 988 doorbell: 989 /* Fewer and fewer doorbells as the queue fills up */ 990 if (eq->pending >= (1 << (fls(eq->qsize - eq->avail) / 2))) { 991 txq->txbytes += txinfo.len; 992 txq->txpkts++; 993 ring_tx_db(sc, eq); 994 } 995 (void) reclaim_tx_descs(txq, 32); 996 } 997 998 if (txpkts.npkt > 0) 999 write_txpkts_wr(txq, &txpkts); 1000 1001 /* 1002 * frame not NULL means there was an error but we haven't thrown it 1003 * away. This can happen when we're short of tx descriptors (qfull) or 1004 * maybe even DMA handles (dma_hdl_failed). Either way, a credit flush 1005 * and reclaim will get things going again. 1006 * 1007 * If eq->avail is already 0 we know a credit flush was requested in the 1008 * WR that reduced it to 0 so we don't need another flush (we don't have 1009 * any descriptor for a flush WR anyway, duh). 1010 */ 1011 if (frame && eq->avail > 0) 1012 write_txqflush_wr(txq); 1013 1014 if (eq->pending != 0) 1015 ring_tx_db(sc, eq); 1016 1017 (void) reclaim_tx_descs(txq, eq->qsize); 1018 TXQ_UNLOCK(txq); 1019 1020 return (frame); 1021 } 1022 1023 static inline void 1024 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int8_t pktc_idx, 1025 int qsize, uint8_t esize) 1026 { 1027 ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS); 1028 ASSERT(pktc_idx < SGE_NCOUNTERS); /* -ve is ok, means don't use */ 1029 1030 iq->flags = 0; 1031 iq->adapter = sc; 1032 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx); 1033 iq->intr_pktc_idx = SGE_NCOUNTERS - 1; 1034 if (pktc_idx >= 0) { 1035 iq->intr_params |= F_QINTR_CNT_EN; 1036 iq->intr_pktc_idx = pktc_idx; 1037 } 1038 iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */ 1039 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */ 1040 } 1041 1042 static inline void 1043 init_fl(struct sge_fl *fl, uint16_t qsize) 1044 { 1045 1046 fl->qsize = qsize; 1047 fl->allocb_fail = 0; 1048 } 1049 1050 static inline void 1051 init_eq(struct adapter *sc, struct sge_eq *eq, uint16_t qsize, uint8_t tx_chan, 1052 uint16_t iqid) 1053 { 1054 struct sge *s = &sc->sge; 1055 uint32_t r; 1056 1057 ASSERT(tx_chan < NCHAN); 1058 1059 if (is_t5(sc->params.chip)) { 1060 r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF); 1061 r >>= S_QUEUESPERPAGEPF0 + 1062 (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf; 1063 s->s_qpp = r & M_QUEUESPERPAGEPF0; 1064 } 1065 1066 eq->flags = 0; 1067 eq->tx_chan = tx_chan; 1068 eq->iqid = iqid; 1069 eq->qsize = qsize; 1070 } 1071 1072 /* 1073 * Allocates the ring for an ingress queue and an optional freelist. If the 1074 * freelist is specified it will be allocated and then associated with the 1075 * ingress queue. 1076 * 1077 * Returns errno on failure. Resources allocated up to that point may still be 1078 * allocated. Caller is responsible for cleanup in case this function fails. 1079 * 1080 * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then 1081 * the intr_idx specifies the vector, starting from 0. Otherwise it specifies 1082 * the index of the queue to which its interrupts will be forwarded. 1083 */ 1084 static int 1085 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl, 1086 int intr_idx, int cong) 1087 { 1088 int rc, i, cntxt_id; 1089 size_t len; 1090 struct fw_iq_cmd c; 1091 struct adapter *sc = iq->adapter; 1092 uint32_t v = 0; 1093 1094 len = iq->qsize * iq->esize; 1095 rc = alloc_desc_ring(sc, len, DDI_DMA_READ, &iq->dhdl, &iq->ahdl, 1096 &iq->ba, (caddr_t *)&iq->desc); 1097 if (rc != 0) 1098 return (rc); 1099 1100 bzero(&c, sizeof (c)); 1101 c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | 1102 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) | 1103 V_FW_IQ_CMD_VFN(0)); 1104 1105 c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART | 1106 FW_LEN16(c)); 1107 1108 /* Special handling for firmware event queue */ 1109 if (iq == &sc->sge.fwq) 1110 v |= F_FW_IQ_CMD_IQASYNCH; 1111 1112 if (iq->flags & IQ_INTR) 1113 ASSERT(intr_idx < sc->intr_count); 1114 else 1115 v |= F_FW_IQ_CMD_IQANDST; 1116 v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx); 1117 1118 c.type_to_iqandstindex = cpu_to_be32(v | 1119 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | 1120 V_FW_IQ_CMD_VIID(pi->viid) | 1121 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT)); 1122 c.iqdroprss_to_iqesize = cpu_to_be16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | 1123 F_FW_IQ_CMD_IQGTSMODE | 1124 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) | 1125 V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4)); 1126 c.iqsize = cpu_to_be16(iq->qsize); 1127 c.iqaddr = cpu_to_be64(iq->ba); 1128 if (cong >= 0) { 1129 const uint32_t iq_type = 1130 cong ? FW_IQ_IQTYPE_NIC : FW_IQ_IQTYPE_OFLD; 1131 c.iqns_to_fl0congen = BE_32(F_FW_IQ_CMD_IQFLINTCONGEN | 1132 V_FW_IQ_CMD_IQTYPE(iq_type)); 1133 } 1134 1135 if (fl != NULL) { 1136 unsigned int chip_ver = CHELSIO_CHIP_VERSION(sc->params.chip); 1137 1138 mutex_init(&fl->lock, NULL, MUTEX_DRIVER, 1139 DDI_INTR_PRI(sc->intr_pri)); 1140 fl->flags |= FL_MTX; 1141 1142 len = fl->qsize * RX_FL_ESIZE; 1143 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &fl->dhdl, 1144 &fl->ahdl, &fl->ba, (caddr_t *)&fl->desc); 1145 if (rc != 0) 1146 return (rc); 1147 1148 /* Allocate space for one software descriptor per buffer. */ 1149 fl->cap = (fl->qsize - sc->sge.stat_len / RX_FL_ESIZE) * 8; 1150 fl->sdesc = kmem_zalloc(sizeof (struct fl_sdesc) * fl->cap, 1151 KM_SLEEP); 1152 fl->needed = fl->cap; 1153 fl->lowat = roundup(sc->sge.fl_starve_threshold, 8); 1154 1155 c.iqns_to_fl0congen |= 1156 cpu_to_be32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) | 1157 F_FW_IQ_CMD_FL0PACKEN | F_FW_IQ_CMD_FL0PADEN); 1158 if (cong >= 0) { 1159 c.iqns_to_fl0congen |= 1160 BE_32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) | 1161 F_FW_IQ_CMD_FL0CONGCIF | 1162 F_FW_IQ_CMD_FL0CONGEN); 1163 } 1164 1165 /* 1166 * In T6, for egress queue type FL there is internal overhead 1167 * of 16B for header going into FLM module. Hence the maximum 1168 * allowed burst size is 448 bytes. For T4/T5, the hardware 1169 * doesn't coalesce fetch requests if more than 64 bytes of 1170 * Free List pointers are provided, so we use a 128-byte Fetch 1171 * Burst Minimum there (T6 implements coalescing so we can use 1172 * the smaller 64-byte value there). 1173 */ 1174 1175 c.fl0dcaen_to_fl0cidxfthresh = cpu_to_be16( 1176 V_FW_IQ_CMD_FL0FBMIN(chip_ver <= CHELSIO_T5 ? 1177 X_FETCHBURSTMIN_128B : X_FETCHBURSTMIN_64B) | 1178 V_FW_IQ_CMD_FL0FBMAX(chip_ver <= CHELSIO_T5 ? 1179 X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B)); 1180 c.fl0size = cpu_to_be16(fl->qsize); 1181 c.fl0addr = cpu_to_be64(fl->ba); 1182 } 1183 1184 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1185 if (rc != 0) { 1186 cxgb_printf(sc->dip, CE_WARN, 1187 "failed to create ingress queue: %d", rc); 1188 return (rc); 1189 } 1190 1191 iq->cdesc = iq->desc; 1192 iq->cidx = 0; 1193 iq->gen = 1; 1194 iq->intr_next = iq->intr_params; 1195 iq->adapter = sc; 1196 iq->cntxt_id = be16_to_cpu(c.iqid); 1197 iq->abs_id = be16_to_cpu(c.physiqid); 1198 iq->flags |= IQ_ALLOCATED; 1199 mutex_init(&iq->lock, NULL, MUTEX_DRIVER, 1200 DDI_INTR_PRI(DDI_INTR_PRI(sc->intr_pri))); 1201 iq->polling = 0; 1202 1203 cntxt_id = iq->cntxt_id - sc->sge.iq_start; 1204 if (cntxt_id >= sc->sge.iqmap_sz) { 1205 panic("%s: iq->cntxt_id (%d) more than the max (%d)", __func__, 1206 cntxt_id, sc->sge.iqmap_sz - 1); 1207 } 1208 sc->sge.iqmap[cntxt_id] = iq; 1209 1210 if (fl != NULL) { 1211 fl->cntxt_id = be16_to_cpu(c.fl0id); 1212 fl->pidx = fl->cidx = 0; 1213 fl->copy_threshold = rx_copy_threshold; 1214 1215 cntxt_id = fl->cntxt_id - sc->sge.eq_start; 1216 if (cntxt_id >= sc->sge.eqmap_sz) { 1217 panic("%s: fl->cntxt_id (%d) more than the max (%d)", 1218 __func__, cntxt_id, sc->sge.eqmap_sz - 1); 1219 } 1220 sc->sge.eqmap[cntxt_id] = (void *)fl; 1221 1222 FL_LOCK(fl); 1223 (void) refill_fl(sc, fl, fl->lowat); 1224 FL_UNLOCK(fl); 1225 1226 iq->flags |= IQ_HAS_FL; 1227 } 1228 1229 if (is_t5(sc->params.chip) && cong >= 0) { 1230 uint32_t param, val; 1231 1232 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 1233 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) | 1234 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id); 1235 if (cong == 0) 1236 val = 1 << 19; 1237 else { 1238 val = 2 << 19; 1239 for (i = 0; i < 4; i++) { 1240 if (cong & (1 << i)) 1241 val |= 1 << (i << 2); 1242 } 1243 } 1244 1245 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1246 if (rc != 0) { 1247 /* report error but carry on */ 1248 cxgb_printf(sc->dip, CE_WARN, 1249 "failed to set congestion manager context for " 1250 "ingress queue %d: %d", iq->cntxt_id, rc); 1251 } 1252 } 1253 1254 /* Enable IQ interrupts */ 1255 iq->state = IQS_IDLE; 1256 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) | 1257 V_INGRESSQID(iq->cntxt_id)); 1258 1259 return (0); 1260 } 1261 1262 static int 1263 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl) 1264 { 1265 int rc; 1266 1267 if (iq != NULL) { 1268 struct adapter *sc = iq->adapter; 1269 dev_info_t *dip; 1270 1271 dip = pi ? pi->dip : sc->dip; 1272 if (iq->flags & IQ_ALLOCATED) { 1273 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, 1274 FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id, 1275 fl ? fl->cntxt_id : 0xffff, 0xffff); 1276 if (rc != 0) { 1277 cxgb_printf(dip, CE_WARN, 1278 "failed to free queue %p: %d", iq, rc); 1279 return (rc); 1280 } 1281 mutex_destroy(&iq->lock); 1282 iq->flags &= ~IQ_ALLOCATED; 1283 } 1284 1285 if (iq->desc != NULL) { 1286 (void) free_desc_ring(&iq->dhdl, &iq->ahdl); 1287 iq->desc = NULL; 1288 } 1289 1290 bzero(iq, sizeof (*iq)); 1291 } 1292 1293 if (fl != NULL) { 1294 if (fl->sdesc != NULL) { 1295 FL_LOCK(fl); 1296 free_fl_bufs(fl); 1297 FL_UNLOCK(fl); 1298 1299 kmem_free(fl->sdesc, sizeof (struct fl_sdesc) * 1300 fl->cap); 1301 fl->sdesc = NULL; 1302 } 1303 1304 if (fl->desc != NULL) { 1305 (void) free_desc_ring(&fl->dhdl, &fl->ahdl); 1306 fl->desc = NULL; 1307 } 1308 1309 if (fl->flags & FL_MTX) { 1310 mutex_destroy(&fl->lock); 1311 fl->flags &= ~FL_MTX; 1312 } 1313 1314 bzero(fl, sizeof (struct sge_fl)); 1315 } 1316 1317 return (0); 1318 } 1319 1320 static int 1321 alloc_fwq(struct adapter *sc) 1322 { 1323 int rc, intr_idx; 1324 struct sge_iq *fwq = &sc->sge.fwq; 1325 1326 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE); 1327 fwq->flags |= IQ_INTR; /* always */ 1328 intr_idx = sc->intr_count > 1 ? 1 : 0; 1329 rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1); 1330 if (rc != 0) { 1331 cxgb_printf(sc->dip, CE_WARN, 1332 "failed to create firmware event queue: %d.", rc); 1333 return (rc); 1334 } 1335 1336 return (0); 1337 } 1338 1339 static int 1340 free_fwq(struct adapter *sc) 1341 { 1342 1343 return (free_iq_fl(NULL, &sc->sge.fwq, NULL)); 1344 } 1345 1346 static int 1347 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int i) 1348 { 1349 int rc; 1350 1351 rxq->port = pi; 1352 rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx, 1353 t4_get_tp_ch_map(pi->adapter, pi->tx_chan)); 1354 if (rc != 0) 1355 return (rc); 1356 1357 rxq->ksp = setup_rxq_kstats(pi, rxq, i); 1358 1359 return (rc); 1360 } 1361 1362 static int 1363 free_rxq(struct port_info *pi, struct sge_rxq *rxq) 1364 { 1365 int rc; 1366 1367 if (rxq->ksp != NULL) { 1368 kstat_delete(rxq->ksp); 1369 rxq->ksp = NULL; 1370 } 1371 1372 rc = free_iq_fl(pi, &rxq->iq, &rxq->fl); 1373 if (rc == 0) 1374 bzero(&rxq->fl, sizeof (*rxq) - offsetof(struct sge_rxq, fl)); 1375 1376 return (rc); 1377 } 1378 1379 static int 1380 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1381 { 1382 int rc, cntxt_id; 1383 struct fw_eq_eth_cmd c; 1384 1385 bzero(&c, sizeof (c)); 1386 1387 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST | 1388 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) | 1389 V_FW_EQ_ETH_CMD_VFN(0)); 1390 c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC | 1391 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c)); 1392 c.autoequiqe_to_viid = BE_32(F_FW_EQ_ETH_CMD_AUTOEQUIQE | 1393 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(pi->viid)); 1394 c.fetchszm_to_iqid = 1395 BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1396 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | 1397 V_FW_EQ_ETH_CMD_IQID(eq->iqid)); 1398 c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1399 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1400 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1401 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize)); 1402 c.eqaddr = BE_64(eq->ba); 1403 1404 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1405 if (rc != 0) { 1406 cxgb_printf(pi->dip, CE_WARN, 1407 "failed to create Ethernet egress queue: %d", rc); 1408 return (rc); 1409 } 1410 eq->flags |= EQ_ALLOCATED; 1411 1412 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd)); 1413 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1414 if (cntxt_id >= sc->sge.eqmap_sz) 1415 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1416 cntxt_id, sc->sge.eqmap_sz - 1); 1417 sc->sge.eqmap[cntxt_id] = eq; 1418 1419 return (rc); 1420 } 1421 1422 static int 1423 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1424 { 1425 int rc; 1426 size_t len; 1427 1428 mutex_init(&eq->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(sc->intr_pri)); 1429 eq->flags |= EQ_MTX; 1430 1431 len = eq->qsize * EQ_ESIZE; 1432 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &eq->desc_dhdl, 1433 &eq->desc_ahdl, &eq->ba, (caddr_t *)&eq->desc); 1434 if (rc != 0) 1435 return (rc); 1436 1437 eq->cap = eq->qsize - sc->sge.stat_len / EQ_ESIZE; 1438 eq->spg = (void *)&eq->desc[eq->cap]; 1439 eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */ 1440 eq->pidx = eq->cidx = 0; 1441 eq->doorbells = sc->doorbells; 1442 1443 rc = eth_eq_alloc(sc, pi, eq); 1444 1445 if (eq->doorbells & (DOORBELL_UDB | DOORBELL_UDBWC | DOORBELL_WCWR)) { 1446 uint32_t s_qpp = sc->sge.s_qpp; 1447 uint32_t mask = (1 << s_qpp) - 1; 1448 volatile uint8_t *udb; 1449 1450 udb = (volatile uint8_t *)sc->reg1p + UDBS_DB_OFFSET; 1451 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */ 1452 eq->udb_qid = eq->cntxt_id & mask; /* id in page */ 1453 if (eq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE) 1454 eq->doorbells &= ~DOORBELL_WCWR; 1455 else { 1456 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */ 1457 eq->udb_qid = 0; 1458 } 1459 eq->udb = (volatile void *)udb; 1460 } 1461 1462 if (rc != 0) { 1463 cxgb_printf(sc->dip, CE_WARN, 1464 "failed to allocate egress queue: %d", rc); 1465 } 1466 1467 return (rc); 1468 } 1469 1470 static int 1471 free_eq(struct adapter *sc, struct sge_eq *eq) 1472 { 1473 int rc; 1474 1475 if (eq->flags & EQ_ALLOCATED) { 1476 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id); 1477 if (rc != 0) { 1478 cxgb_printf(sc->dip, CE_WARN, 1479 "failed to free egress queue: %d", rc); 1480 return (rc); 1481 } 1482 eq->flags &= ~EQ_ALLOCATED; 1483 } 1484 1485 if (eq->desc != NULL) { 1486 (void) free_desc_ring(&eq->desc_dhdl, &eq->desc_ahdl); 1487 eq->desc = NULL; 1488 } 1489 1490 if (eq->flags & EQ_MTX) 1491 mutex_destroy(&eq->lock); 1492 1493 bzero(eq, sizeof (*eq)); 1494 return (0); 1495 } 1496 1497 static int 1498 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx) 1499 { 1500 int rc, i; 1501 struct adapter *sc = pi->adapter; 1502 struct sge_eq *eq = &txq->eq; 1503 1504 rc = alloc_eq(sc, pi, eq); 1505 if (rc != 0) 1506 return (rc); 1507 1508 txq->port = pi; 1509 txq->sdesc = kmem_zalloc(sizeof (struct tx_sdesc) * eq->cap, KM_SLEEP); 1510 txq->txb_size = eq->qsize * tx_copy_threshold; 1511 rc = alloc_tx_copybuffer(sc, txq->txb_size, &txq->txb_dhdl, 1512 &txq->txb_ahdl, &txq->txb_ba, &txq->txb_va); 1513 if (rc == 0) 1514 txq->txb_avail = txq->txb_size; 1515 else 1516 txq->txb_avail = txq->txb_size = 0; 1517 1518 /* 1519 * TODO: is this too low? Worst case would need around 4 times qsize 1520 * (all tx descriptors filled to the brim with SGLs, with each entry in 1521 * the SGL coming from a distinct DMA handle). Increase tx_dhdl_total 1522 * if you see too many dma_hdl_failed. 1523 */ 1524 txq->tx_dhdl_total = eq->qsize * 2; 1525 txq->tx_dhdl = kmem_zalloc(sizeof (ddi_dma_handle_t) * 1526 txq->tx_dhdl_total, KM_SLEEP); 1527 for (i = 0; i < txq->tx_dhdl_total; i++) { 1528 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx, 1529 DDI_DMA_SLEEP, 0, &txq->tx_dhdl[i]); 1530 if (rc != DDI_SUCCESS) { 1531 cxgb_printf(sc->dip, CE_WARN, 1532 "%s: failed to allocate DMA handle (%d)", 1533 __func__, rc); 1534 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 1535 } 1536 txq->tx_dhdl_avail++; 1537 } 1538 1539 txq->ksp = setup_txq_kstats(pi, txq, idx); 1540 1541 return (rc); 1542 } 1543 1544 static int 1545 free_txq(struct port_info *pi, struct sge_txq *txq) 1546 { 1547 int i; 1548 struct adapter *sc = pi->adapter; 1549 struct sge_eq *eq = &txq->eq; 1550 1551 if (txq->ksp != NULL) { 1552 kstat_delete(txq->ksp); 1553 txq->ksp = NULL; 1554 } 1555 1556 if (txq->txb_va != NULL) { 1557 (void) free_desc_ring(&txq->txb_dhdl, &txq->txb_ahdl); 1558 txq->txb_va = NULL; 1559 } 1560 1561 if (txq->sdesc != NULL) { 1562 struct tx_sdesc *sd; 1563 ddi_dma_handle_t hdl; 1564 1565 TXQ_LOCK(txq); 1566 while (eq->cidx != eq->pidx) { 1567 sd = &txq->sdesc[eq->cidx]; 1568 1569 for (i = sd->hdls_used; i; i--) { 1570 hdl = txq->tx_dhdl[txq->tx_dhdl_cidx]; 1571 (void) ddi_dma_unbind_handle(hdl); 1572 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total) 1573 txq->tx_dhdl_cidx = 0; 1574 } 1575 1576 ASSERT(sd->m); 1577 freemsgchain(sd->m); 1578 1579 eq->cidx += sd->desc_used; 1580 if (eq->cidx >= eq->cap) 1581 eq->cidx -= eq->cap; 1582 1583 txq->txb_avail += txq->txb_used; 1584 } 1585 ASSERT(txq->tx_dhdl_cidx == txq->tx_dhdl_pidx); 1586 ASSERT(txq->txb_avail == txq->txb_size); 1587 TXQ_UNLOCK(txq); 1588 1589 kmem_free(txq->sdesc, sizeof (struct tx_sdesc) * eq->cap); 1590 txq->sdesc = NULL; 1591 } 1592 1593 if (txq->tx_dhdl != NULL) { 1594 for (i = 0; i < txq->tx_dhdl_total; i++) { 1595 if (txq->tx_dhdl[i] != NULL) 1596 ddi_dma_free_handle(&txq->tx_dhdl[i]); 1597 } 1598 kmem_free(txq->tx_dhdl, 1599 sizeof (ddi_dma_handle_t) * txq->tx_dhdl_total); 1600 txq->tx_dhdl = NULL; 1601 } 1602 1603 (void) free_eq(sc, &txq->eq); 1604 1605 bzero(txq, sizeof (*txq)); 1606 return (0); 1607 } 1608 1609 /* 1610 * Allocates a block of contiguous memory for DMA. Can be used to allocate 1611 * memory for descriptor rings or for tx/rx copy buffers. 1612 * 1613 * Caller does not have to clean up anything if this function fails, it cleans 1614 * up after itself. 1615 * 1616 * Caller provides the following: 1617 * len length of the block of memory to allocate. 1618 * flags DDI_DMA_* flags to use (CONSISTENT/STREAMING, READ/WRITE/RDWR) 1619 * acc_attr device access attributes for the allocation. 1620 * dma_attr DMA attributes for the allocation 1621 * 1622 * If the function is successful it fills up this information: 1623 * dma_hdl DMA handle for the allocated memory 1624 * acc_hdl access handle for the allocated memory 1625 * ba bus address of the allocated memory 1626 * va KVA of the allocated memory. 1627 */ 1628 static int 1629 alloc_dma_memory(struct adapter *sc, size_t len, int flags, 1630 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr, 1631 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 1632 uint64_t *pba, caddr_t *pva) 1633 { 1634 int rc; 1635 ddi_dma_handle_t dhdl; 1636 ddi_acc_handle_t ahdl; 1637 ddi_dma_cookie_t cookie; 1638 uint_t ccount; 1639 caddr_t va; 1640 size_t real_len; 1641 1642 *pva = NULL; 1643 1644 /* 1645 * DMA handle. 1646 */ 1647 rc = ddi_dma_alloc_handle(sc->dip, dma_attr, DDI_DMA_SLEEP, 0, &dhdl); 1648 if (rc != DDI_SUCCESS) { 1649 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 1650 } 1651 1652 /* 1653 * Memory suitable for DMA. 1654 */ 1655 rc = ddi_dma_mem_alloc(dhdl, len, acc_attr, 1656 flags & DDI_DMA_CONSISTENT ? DDI_DMA_CONSISTENT : DDI_DMA_STREAMING, 1657 DDI_DMA_SLEEP, 0, &va, &real_len, &ahdl); 1658 if (rc != DDI_SUCCESS) { 1659 ddi_dma_free_handle(&dhdl); 1660 return (ENOMEM); 1661 } 1662 1663 /* 1664 * DMA bindings. 1665 */ 1666 rc = ddi_dma_addr_bind_handle(dhdl, NULL, va, real_len, flags, NULL, 1667 NULL, &cookie, &ccount); 1668 if (rc != DDI_DMA_MAPPED) { 1669 ddi_dma_mem_free(&ahdl); 1670 ddi_dma_free_handle(&dhdl); 1671 return (ENOMEM); 1672 } 1673 if (ccount != 1) { 1674 /* unusable DMA mapping */ 1675 (void) free_desc_ring(&dhdl, &ahdl); 1676 return (ENOMEM); 1677 } 1678 1679 bzero(va, real_len); 1680 *dma_hdl = dhdl; 1681 *acc_hdl = ahdl; 1682 *pba = cookie.dmac_laddress; 1683 *pva = va; 1684 1685 return (0); 1686 } 1687 1688 static int 1689 free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl) 1690 { 1691 (void) ddi_dma_unbind_handle(*dhdl); 1692 ddi_dma_mem_free(ahdl); 1693 ddi_dma_free_handle(dhdl); 1694 1695 return (0); 1696 } 1697 1698 static int 1699 alloc_desc_ring(struct adapter *sc, size_t len, int rw, 1700 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 1701 uint64_t *pba, caddr_t *pva) 1702 { 1703 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_desc; 1704 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; 1705 1706 return (alloc_dma_memory(sc, len, DDI_DMA_CONSISTENT | rw, acc_attr, 1707 dma_attr, dma_hdl, acc_hdl, pba, pva)); 1708 } 1709 1710 static int 1711 free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl) 1712 { 1713 return (free_dma_memory(dhdl, ahdl)); 1714 } 1715 1716 static int 1717 alloc_tx_copybuffer(struct adapter *sc, size_t len, 1718 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 1719 uint64_t *pba, caddr_t *pva) 1720 { 1721 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_tx; 1722 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; /* NOT dma_attr_tx */ 1723 1724 return (alloc_dma_memory(sc, len, DDI_DMA_STREAMING | DDI_DMA_WRITE, 1725 acc_attr, dma_attr, dma_hdl, acc_hdl, pba, pva)); 1726 } 1727 1728 static inline bool 1729 is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl) 1730 { 1731 (void) ddi_dma_sync(iq->dhdl, (uintptr_t)iq->cdesc - 1732 (uintptr_t)iq->desc, iq->esize, DDI_DMA_SYNC_FORKERNEL); 1733 1734 *ctrl = (void *)((uintptr_t)iq->cdesc + 1735 (iq->esize - sizeof (struct rsp_ctrl))); 1736 1737 return ((((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen)); 1738 } 1739 1740 static inline void 1741 iq_next(struct sge_iq *iq) 1742 { 1743 iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize); 1744 if (++iq->cidx == iq->qsize - 1) { 1745 iq->cidx = 0; 1746 iq->gen ^= 1; 1747 iq->cdesc = iq->desc; 1748 } 1749 } 1750 1751 /* 1752 * Fill up the freelist by upto nbufs and maybe ring its doorbell. 1753 * 1754 * Returns non-zero to indicate that it should be added to the list of starving 1755 * freelists. 1756 */ 1757 static int 1758 refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs) 1759 { 1760 uint64_t *d = &fl->desc[fl->pidx]; 1761 struct fl_sdesc *sd = &fl->sdesc[fl->pidx]; 1762 1763 FL_LOCK_ASSERT_OWNED(fl); 1764 ASSERT(nbufs >= 0); 1765 1766 if (nbufs > fl->needed) 1767 nbufs = fl->needed; 1768 1769 while (nbufs--) { 1770 if (sd->rxb != NULL) { 1771 if (sd->rxb->ref_cnt == 1) { 1772 /* 1773 * Buffer is available for recycling. Two ways 1774 * this can happen: 1775 * 1776 * a) All the packets DMA'd into it last time 1777 * around were within the rx_copy_threshold 1778 * and no part of the buffer was ever passed 1779 * up (ref_cnt never went over 1). 1780 * 1781 * b) Packets DMA'd into the buffer were passed 1782 * up but have all been freed by the upper 1783 * layers by now (ref_cnt went over 1 but is 1784 * now back to 1). 1785 * 1786 * Either way the bus address in the descriptor 1787 * ring is already valid. 1788 */ 1789 ASSERT(*d == cpu_to_be64(sd->rxb->ba)); 1790 d++; 1791 goto recycled; 1792 } else { 1793 /* 1794 * Buffer still in use and we need a 1795 * replacement. But first release our reference 1796 * on the existing buffer. 1797 */ 1798 rxbuf_free(sd->rxb); 1799 } 1800 } 1801 1802 sd->rxb = rxbuf_alloc(sc->sge.rxbuf_cache, KM_NOSLEEP, 1); 1803 if (sd->rxb == NULL) 1804 break; 1805 *d++ = cpu_to_be64(sd->rxb->ba); 1806 1807 recycled: fl->pending++; 1808 sd++; 1809 fl->needed--; 1810 if (++fl->pidx == fl->cap) { 1811 fl->pidx = 0; 1812 sd = fl->sdesc; 1813 d = fl->desc; 1814 } 1815 } 1816 1817 if (fl->pending >= 8) 1818 ring_fl_db(sc, fl); 1819 1820 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING)); 1821 } 1822 1823 #ifndef TAILQ_FOREACH_SAFE 1824 #define TAILQ_FOREACH_SAFE(var, head, field, tvar) \ 1825 for ((var) = TAILQ_FIRST((head)); \ 1826 (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \ 1827 (var) = (tvar)) 1828 #endif 1829 1830 /* 1831 * Attempt to refill all starving freelists. 1832 */ 1833 static void 1834 refill_sfl(void *arg) 1835 { 1836 struct adapter *sc = arg; 1837 struct sge_fl *fl, *fl_temp; 1838 1839 mutex_enter(&sc->sfl_lock); 1840 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) { 1841 FL_LOCK(fl); 1842 (void) refill_fl(sc, fl, 64); 1843 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) { 1844 TAILQ_REMOVE(&sc->sfl, fl, link); 1845 fl->flags &= ~FL_STARVING; 1846 } 1847 FL_UNLOCK(fl); 1848 } 1849 1850 if (!TAILQ_EMPTY(&sc->sfl) != 0) 1851 sc->sfl_timer = timeout(refill_sfl, sc, drv_usectohz(100000)); 1852 mutex_exit(&sc->sfl_lock); 1853 } 1854 1855 static void 1856 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl) 1857 { 1858 mutex_enter(&sc->sfl_lock); 1859 FL_LOCK(fl); 1860 if ((fl->flags & FL_DOOMED) == 0) { 1861 if (TAILQ_EMPTY(&sc->sfl) != 0) { 1862 sc->sfl_timer = timeout(refill_sfl, sc, 1863 drv_usectohz(100000)); 1864 } 1865 fl->flags |= FL_STARVING; 1866 TAILQ_INSERT_TAIL(&sc->sfl, fl, link); 1867 } 1868 FL_UNLOCK(fl); 1869 mutex_exit(&sc->sfl_lock); 1870 } 1871 1872 static void 1873 free_fl_bufs(struct sge_fl *fl) 1874 { 1875 struct fl_sdesc *sd; 1876 unsigned int i; 1877 1878 FL_LOCK_ASSERT_OWNED(fl); 1879 1880 for (i = 0; i < fl->cap; i++) { 1881 sd = &fl->sdesc[i]; 1882 1883 if (sd->rxb != NULL) { 1884 rxbuf_free(sd->rxb); 1885 sd->rxb = NULL; 1886 } 1887 } 1888 } 1889 1890 /* 1891 * Note that fl->cidx and fl->offset are left unchanged in case of failure. 1892 */ 1893 static mblk_t * 1894 get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf, 1895 int *fl_bufs_used) 1896 { 1897 struct mblk_pair frame = {0}; 1898 struct rxbuf *rxb; 1899 mblk_t *m = NULL; 1900 uint_t nbuf = 0, len, copy, n; 1901 uint32_t cidx, offset, rcidx, roffset; 1902 1903 /* 1904 * The SGE won't pack a new frame into the current buffer if the entire 1905 * payload doesn't fit in the remaining space. Move on to the next buf 1906 * in that case. 1907 */ 1908 rcidx = fl->cidx; 1909 roffset = fl->offset; 1910 if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) { 1911 fl->offset = 0; 1912 if (++fl->cidx == fl->cap) 1913 fl->cidx = 0; 1914 nbuf++; 1915 } 1916 cidx = fl->cidx; 1917 offset = fl->offset; 1918 1919 len = G_RSPD_LEN(len_newbuf); /* pktshift + payload length */ 1920 copy = (len <= fl->copy_threshold); 1921 if (copy != 0) { 1922 frame.head = m = allocb(len, BPRI_HI); 1923 if (m == NULL) { 1924 fl->allocb_fail++; 1925 DTRACE_PROBE1(t4__fl_alloc_fail, struct sge_fl *, fl); 1926 fl->cidx = rcidx; 1927 fl->offset = roffset; 1928 return (NULL); 1929 } 1930 } 1931 1932 while (len) { 1933 rxb = fl->sdesc[cidx].rxb; 1934 n = min(len, rxb->buf_size - offset); 1935 1936 (void) ddi_dma_sync(rxb->dhdl, offset, n, 1937 DDI_DMA_SYNC_FORKERNEL); 1938 1939 if (copy != 0) 1940 bcopy(rxb->va + offset, m->b_wptr, n); 1941 else { 1942 m = desballoc((unsigned char *)rxb->va + offset, n, 1943 BPRI_HI, &rxb->freefunc); 1944 if (m == NULL) { 1945 fl->allocb_fail++; 1946 DTRACE_PROBE1(t4__fl_alloc_fail, 1947 struct sge_fl *, fl); 1948 if (frame.head) 1949 freemsgchain(frame.head); 1950 fl->cidx = rcidx; 1951 fl->offset = roffset; 1952 return (NULL); 1953 } 1954 atomic_inc_uint(&rxb->ref_cnt); 1955 if (frame.head != NULL) 1956 frame.tail->b_cont = m; 1957 else 1958 frame.head = m; 1959 frame.tail = m; 1960 } 1961 m->b_wptr += n; 1962 len -= n; 1963 offset += roundup(n, sc->sge.fl_align); 1964 ASSERT(offset <= rxb->buf_size); 1965 if (offset == rxb->buf_size) { 1966 offset = 0; 1967 if (++cidx == fl->cap) 1968 cidx = 0; 1969 nbuf++; 1970 } 1971 } 1972 1973 fl->cidx = cidx; 1974 fl->offset = offset; 1975 (*fl_bufs_used) += nbuf; 1976 1977 ASSERT(frame.head != NULL); 1978 return (frame.head); 1979 } 1980 1981 /* 1982 * We'll do immediate data tx for non-LSO, but only when not coalescing. We're 1983 * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes 1984 * of immediate data. 1985 */ 1986 #define IMM_LEN ( \ 1987 2 * EQ_ESIZE \ 1988 - sizeof (struct fw_eth_tx_pkt_wr) \ 1989 - sizeof (struct cpl_tx_pkt_core)) 1990 1991 /* 1992 * Returns non-zero on failure, no need to cleanup anything in that case. 1993 * 1994 * Note 1: We always try to pull up the mblk if required and return E2BIG only 1995 * if this fails. 1996 * 1997 * Note 2: We'll also pullup incoming mblk if HW_LSO is set and the first mblk 1998 * does not have the TCP header in it. 1999 */ 2000 static int 2001 get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, struct txinfo *txinfo, 2002 int sgl_only) 2003 { 2004 uint32_t flags = 0, len, n; 2005 mblk_t *m = *fp; 2006 int rc; 2007 2008 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb and dma_hdls */ 2009 2010 mac_hcksum_get(m, NULL, NULL, NULL, NULL, &flags); 2011 txinfo->flags = (flags & HCK_TX_FLAGS); 2012 2013 mac_lso_get(m, &txinfo->mss, &flags); 2014 txinfo->flags |= (flags & HW_LSO_FLAGS); 2015 2016 if (flags & HW_LSO) 2017 sgl_only = 1; /* Do not allow immediate data with LSO */ 2018 2019 /* 2020 * If checksum or segmentation offloads are requested, gather 2021 * information about the sizes and types of headers in the packet. 2022 */ 2023 if (txinfo->flags != 0) { 2024 mac_ether_offload_info(m, &txinfo->meoi); 2025 } else { 2026 bzero(&txinfo->meoi, sizeof (txinfo->meoi)); 2027 } 2028 2029 start: txinfo->nsegs = 0; 2030 txinfo->hdls_used = 0; 2031 txinfo->txb_used = 0; 2032 txinfo->len = 0; 2033 2034 /* total length and a rough estimate of # of segments */ 2035 n = 0; 2036 for (; m; m = m->b_cont) { 2037 len = MBLKL(m); 2038 n += (len / PAGE_SIZE) + 1; 2039 txinfo->len += len; 2040 } 2041 m = *fp; 2042 2043 if (n >= TX_SGL_SEGS || (flags & HW_LSO && MBLKL(m) < 50)) { 2044 txq->pullup_early++; 2045 m = msgpullup(*fp, -1); 2046 if (m == NULL) { 2047 txq->pullup_failed++; 2048 return (E2BIG); /* (*fp) left as it was */ 2049 } 2050 freemsg(*fp); 2051 *fp = m; 2052 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags); 2053 } 2054 2055 if (txinfo->len <= IMM_LEN && !sgl_only) 2056 return (0); /* nsegs = 0 tells caller to use imm. tx */ 2057 2058 if (txinfo->len <= txq->copy_threshold && 2059 copy_into_txb(txq, m, txinfo->len, txinfo) == 0) 2060 goto done; 2061 2062 for (; m; m = m->b_cont) { 2063 2064 len = MBLKL(m); 2065 2066 /* Use tx copy buffer if this mblk is small enough */ 2067 if (len <= txq->copy_threshold && 2068 copy_into_txb(txq, m, len, txinfo) == 0) 2069 continue; 2070 2071 /* Add DMA bindings for this mblk to the SGL */ 2072 rc = add_mblk(txq, txinfo, m, len); 2073 2074 if (rc == E2BIG || 2075 (txinfo->nsegs == TX_SGL_SEGS && m->b_cont)) { 2076 2077 txq->pullup_late++; 2078 m = msgpullup(*fp, -1); 2079 if (m != NULL) { 2080 free_txinfo_resources(txq, txinfo); 2081 freemsg(*fp); 2082 *fp = m; 2083 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags); 2084 goto start; 2085 } 2086 2087 txq->pullup_failed++; 2088 rc = E2BIG; 2089 } 2090 2091 if (rc != 0) { 2092 free_txinfo_resources(txq, txinfo); 2093 return (rc); 2094 } 2095 } 2096 2097 ASSERT(txinfo->nsegs > 0 && txinfo->nsegs <= TX_SGL_SEGS); 2098 2099 done: 2100 2101 /* 2102 * Store the # of flits required to hold this frame's SGL in nflits. An 2103 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by 2104 * multiple (len0 + len1, addr0, addr1) tuples. If addr1 is not used 2105 * then len1 must be set to 0. 2106 */ 2107 n = txinfo->nsegs - 1; 2108 txinfo->nflits = (3 * n) / 2 + (n & 1) + 2; 2109 if (n & 1) 2110 txinfo->sgl.sge[n / 2].len[1] = cpu_to_be32(0); 2111 2112 txinfo->sgl.cmd_nsge = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_DSGL) | 2113 V_ULPTX_NSGE(txinfo->nsegs)); 2114 2115 return (0); 2116 } 2117 2118 static inline int 2119 fits_in_txb(struct sge_txq *txq, int len, int *waste) 2120 { 2121 if (txq->txb_avail < len) 2122 return (0); 2123 2124 if (txq->txb_next + len <= txq->txb_size) { 2125 *waste = 0; 2126 return (1); 2127 } 2128 2129 *waste = txq->txb_size - txq->txb_next; 2130 2131 return (txq->txb_avail - *waste < len ? 0 : 1); 2132 } 2133 2134 #define TXB_CHUNK 64 2135 2136 /* 2137 * Copies the specified # of bytes into txq's tx copy buffer and updates txinfo 2138 * and txq to indicate resources used. Caller has to make sure that those many 2139 * bytes are available in the mblk chain (b_cont linked). 2140 */ 2141 static inline int 2142 copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, struct txinfo *txinfo) 2143 { 2144 int waste, n; 2145 2146 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb */ 2147 2148 if (!fits_in_txb(txq, len, &waste)) { 2149 txq->txb_full++; 2150 return (ENOMEM); 2151 } 2152 2153 if (waste != 0) { 2154 ASSERT((waste & (TXB_CHUNK - 1)) == 0); 2155 txinfo->txb_used += waste; 2156 txq->txb_avail -= waste; 2157 txq->txb_next = 0; 2158 } 2159 2160 for (n = 0; n < len; m = m->b_cont) { 2161 bcopy(m->b_rptr, txq->txb_va + txq->txb_next + n, MBLKL(m)); 2162 n += MBLKL(m); 2163 } 2164 2165 add_seg(txinfo, txq->txb_ba + txq->txb_next, len); 2166 2167 n = roundup(len, TXB_CHUNK); 2168 txinfo->txb_used += n; 2169 txq->txb_avail -= n; 2170 txq->txb_next += n; 2171 ASSERT(txq->txb_next <= txq->txb_size); 2172 if (txq->txb_next == txq->txb_size) 2173 txq->txb_next = 0; 2174 2175 return (0); 2176 } 2177 2178 static inline void 2179 add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len) 2180 { 2181 ASSERT(txinfo->nsegs < TX_SGL_SEGS); /* must have room */ 2182 2183 if (txinfo->nsegs != 0) { 2184 int idx = txinfo->nsegs - 1; 2185 txinfo->sgl.sge[idx / 2].len[idx & 1] = cpu_to_be32(len); 2186 txinfo->sgl.sge[idx / 2].addr[idx & 1] = cpu_to_be64(ba); 2187 } else { 2188 txinfo->sgl.len0 = cpu_to_be32(len); 2189 txinfo->sgl.addr0 = cpu_to_be64(ba); 2190 } 2191 txinfo->nsegs++; 2192 } 2193 2194 /* 2195 * This function cleans up any partially allocated resources when it fails so 2196 * there's nothing for the caller to clean up in that case. 2197 * 2198 * EIO indicates permanent failure. Caller should drop the frame containing 2199 * this mblk and continue. 2200 * 2201 * E2BIG indicates that the SGL length for this mblk exceeds the hardware 2202 * limit. Caller should pull up the frame before trying to send it out. 2203 * (This error means our pullup_early heuristic did not work for this frame) 2204 * 2205 * ENOMEM indicates a temporary shortage of resources (DMA handles, other DMA 2206 * resources, etc.). Caller should suspend the tx queue and wait for reclaim to 2207 * free up resources. 2208 */ 2209 static inline int 2210 add_mblk(struct sge_txq *txq, struct txinfo *txinfo, mblk_t *m, int len) 2211 { 2212 ddi_dma_handle_t dhdl; 2213 ddi_dma_cookie_t cookie; 2214 uint_t ccount = 0; 2215 int rc; 2216 2217 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate dhdls */ 2218 2219 if (txq->tx_dhdl_avail == 0) { 2220 txq->dma_hdl_failed++; 2221 return (ENOMEM); 2222 } 2223 2224 dhdl = txq->tx_dhdl[txq->tx_dhdl_pidx]; 2225 rc = ddi_dma_addr_bind_handle(dhdl, NULL, (caddr_t)m->b_rptr, len, 2226 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie, 2227 &ccount); 2228 if (rc != DDI_DMA_MAPPED) { 2229 txq->dma_map_failed++; 2230 2231 ASSERT(rc != DDI_DMA_INUSE && rc != DDI_DMA_PARTIAL_MAP); 2232 2233 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EIO); 2234 } 2235 2236 if (ccount + txinfo->nsegs > TX_SGL_SEGS) { 2237 (void) ddi_dma_unbind_handle(dhdl); 2238 return (E2BIG); 2239 } 2240 2241 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size); 2242 while (--ccount) { 2243 ddi_dma_nextcookie(dhdl, &cookie); 2244 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size); 2245 } 2246 2247 if (++txq->tx_dhdl_pidx == txq->tx_dhdl_total) 2248 txq->tx_dhdl_pidx = 0; 2249 txq->tx_dhdl_avail--; 2250 txinfo->hdls_used++; 2251 2252 return (0); 2253 } 2254 2255 /* 2256 * Releases all the txq resources used up in the specified txinfo. 2257 */ 2258 static void 2259 free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo) 2260 { 2261 int n; 2262 2263 TXQ_LOCK_ASSERT_OWNED(txq); /* dhdls, txb */ 2264 2265 n = txinfo->txb_used; 2266 if (n > 0) { 2267 txq->txb_avail += n; 2268 if (n <= txq->txb_next) 2269 txq->txb_next -= n; 2270 else { 2271 n -= txq->txb_next; 2272 txq->txb_next = txq->txb_size - n; 2273 } 2274 } 2275 2276 for (n = txinfo->hdls_used; n > 0; n--) { 2277 if (txq->tx_dhdl_pidx > 0) 2278 txq->tx_dhdl_pidx--; 2279 else 2280 txq->tx_dhdl_pidx = txq->tx_dhdl_total - 1; 2281 txq->tx_dhdl_avail++; 2282 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_pidx]); 2283 } 2284 } 2285 2286 /* 2287 * Returns 0 to indicate that m has been accepted into a coalesced tx work 2288 * request. It has either been folded into txpkts or txpkts was flushed and m 2289 * has started a new coalesced work request (as the first frame in a fresh 2290 * txpkts). 2291 * 2292 * Returns non-zero to indicate a failure - caller is responsible for 2293 * transmitting m, if there was anything in txpkts it has been flushed. 2294 */ 2295 static int 2296 add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m, 2297 struct txinfo *txinfo) 2298 { 2299 struct sge_eq *eq = &txq->eq; 2300 int can_coalesce; 2301 struct tx_sdesc *txsd; 2302 uint8_t flits; 2303 2304 TXQ_LOCK_ASSERT_OWNED(txq); 2305 2306 if (txpkts->npkt > 0) { 2307 flits = TXPKTS_PKT_HDR + txinfo->nflits; 2308 can_coalesce = (txinfo->flags & HW_LSO) == 0 && 2309 txpkts->nflits + flits <= TX_WR_FLITS && 2310 txpkts->nflits + flits <= eq->avail * 8 && 2311 txpkts->plen + txinfo->len < 65536; 2312 2313 if (can_coalesce != 0) { 2314 txpkts->tail->b_next = m; 2315 txpkts->tail = m; 2316 txpkts->npkt++; 2317 txpkts->nflits += flits; 2318 txpkts->plen += txinfo->len; 2319 2320 txsd = &txq->sdesc[eq->pidx]; 2321 txsd->txb_used += txinfo->txb_used; 2322 txsd->hdls_used += txinfo->hdls_used; 2323 2324 return (0); 2325 } 2326 2327 /* 2328 * Couldn't coalesce m into txpkts. The first order of business 2329 * is to send txpkts on its way. Then we'll revisit m. 2330 */ 2331 write_txpkts_wr(txq, txpkts); 2332 } 2333 2334 /* 2335 * Check if we can start a new coalesced tx work request with m as 2336 * the first packet in it. 2337 */ 2338 2339 ASSERT(txpkts->npkt == 0); 2340 ASSERT(txinfo->len < 65536); 2341 2342 flits = TXPKTS_WR_HDR + txinfo->nflits; 2343 can_coalesce = (txinfo->flags & HW_LSO) == 0 && 2344 flits <= eq->avail * 8 && flits <= TX_WR_FLITS; 2345 2346 if (can_coalesce == 0) 2347 return (EINVAL); 2348 2349 /* 2350 * Start a fresh coalesced tx WR with m as the first frame in it. 2351 */ 2352 txpkts->tail = m; 2353 txpkts->npkt = 1; 2354 txpkts->nflits = flits; 2355 txpkts->flitp = &eq->desc[eq->pidx].flit[2]; 2356 txpkts->plen = txinfo->len; 2357 2358 txsd = &txq->sdesc[eq->pidx]; 2359 txsd->m = m; 2360 txsd->txb_used = txinfo->txb_used; 2361 txsd->hdls_used = txinfo->hdls_used; 2362 2363 return (0); 2364 } 2365 2366 /* 2367 * Note that write_txpkts_wr can never run out of hardware descriptors (but 2368 * write_txpkt_wr can). add_to_txpkts ensures that a frame is accepted for 2369 * coalescing only if sufficient hardware descriptors are available. 2370 */ 2371 static void 2372 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts) 2373 { 2374 struct sge_eq *eq = &txq->eq; 2375 struct fw_eth_tx_pkts_wr *wr; 2376 struct tx_sdesc *txsd; 2377 uint32_t ctrl; 2378 uint16_t ndesc; 2379 2380 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */ 2381 2382 ndesc = howmany(txpkts->nflits, 8); 2383 2384 wr = (void *)&eq->desc[eq->pidx]; 2385 wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR) | 2386 V_FW_WR_IMMDLEN(0)); /* immdlen does not matter in this WR */ 2387 ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2)); 2388 if (eq->avail == ndesc) 2389 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ; 2390 wr->equiq_to_len16 = cpu_to_be32(ctrl); 2391 wr->plen = cpu_to_be16(txpkts->plen); 2392 wr->npkt = txpkts->npkt; 2393 wr->r3 = wr->type = 0; 2394 2395 /* Everything else already written */ 2396 2397 txsd = &txq->sdesc[eq->pidx]; 2398 txsd->desc_used = ndesc; 2399 2400 txq->txb_used += txsd->txb_used / TXB_CHUNK; 2401 txq->hdl_used += txsd->hdls_used; 2402 2403 ASSERT(eq->avail >= ndesc); 2404 2405 eq->pending += ndesc; 2406 eq->avail -= ndesc; 2407 eq->pidx += ndesc; 2408 if (eq->pidx >= eq->cap) 2409 eq->pidx -= eq->cap; 2410 2411 txq->txpkts_pkts += txpkts->npkt; 2412 txq->txpkts_wrs++; 2413 txpkts->npkt = 0; /* emptied */ 2414 } 2415 2416 typedef enum { 2417 COS_SUCCESS, /* ctrl flit contains proper bits for csum offload */ 2418 COS_IGNORE, /* no csum offload requested */ 2419 COS_FAIL, /* csum offload requested, but pkt data missing */ 2420 } csum_offload_status_t; 2421 /* 2422 * Build a ctrl1 flit for checksum offload in CPL_TX_PKT_XT command 2423 */ 2424 static csum_offload_status_t 2425 csum_to_ctrl(const struct txinfo *txinfo, uint32_t chip_version, 2426 uint64_t *ctrlp) 2427 { 2428 const mac_ether_offload_info_t *meoi = &txinfo->meoi; 2429 const uint32_t tx_flags = txinfo->flags; 2430 const boolean_t needs_l3_csum = (tx_flags & HW_LSO) != 0 || 2431 (tx_flags & HCK_IPV4_HDRCKSUM) != 0; 2432 const boolean_t needs_l4_csum = (tx_flags & HW_LSO) != 0 || 2433 (tx_flags & (HCK_FULLCKSUM | HCK_PARTIALCKSUM)) != 0; 2434 2435 /* 2436 * Default to disabling any checksumming both for cases where it is not 2437 * requested, but also if we cannot appropriately interrogate the 2438 * required information from the packet. 2439 */ 2440 uint64_t ctrl = F_TXPKT_L4CSUM_DIS | F_TXPKT_IPCSUM_DIS; 2441 if (!needs_l3_csum && !needs_l4_csum) { 2442 *ctrlp = ctrl; 2443 return (COS_IGNORE); 2444 } 2445 2446 if (needs_l3_csum) { 2447 /* Only IPv4 checksums are supported (for L3) */ 2448 if ((meoi->meoi_flags & MEOI_L3INFO_SET) == 0 || 2449 meoi->meoi_l3proto != ETHERTYPE_IP) { 2450 *ctrlp = ctrl; 2451 return (COS_FAIL); 2452 } 2453 ctrl &= ~F_TXPKT_IPCSUM_DIS; 2454 } 2455 2456 if (needs_l4_csum) { 2457 /* 2458 * We need at least all of the L3 header to make decisions about 2459 * the contained L4 protocol. If not all of the L4 information 2460 * is present, we will leave it to the NIC to checksum all it is 2461 * able to. 2462 */ 2463 if ((meoi->meoi_flags & MEOI_L3INFO_SET) == 0) { 2464 *ctrlp = ctrl; 2465 return (COS_FAIL); 2466 } 2467 2468 /* 2469 * Since we are parsing the packet anyways, make the checksum 2470 * decision based on the L4 protocol, rather than using the 2471 * Generic TCP/UDP checksum using start & end offsets in the 2472 * packet (like requested with PARTIALCKSUM). 2473 */ 2474 int csum_type = -1; 2475 if (meoi->meoi_l3proto == ETHERTYPE_IP && 2476 meoi->meoi_l4proto == IPPROTO_TCP) { 2477 csum_type = TX_CSUM_TCPIP; 2478 } else if (meoi->meoi_l3proto == ETHERTYPE_IPV6 && 2479 meoi->meoi_l4proto == IPPROTO_TCP) { 2480 csum_type = TX_CSUM_TCPIP6; 2481 } else if (meoi->meoi_l3proto == ETHERTYPE_IP && 2482 meoi->meoi_l4proto == IPPROTO_UDP) { 2483 csum_type = TX_CSUM_UDPIP; 2484 } else if (meoi->meoi_l3proto == ETHERTYPE_IPV6 && 2485 meoi->meoi_l4proto == IPPROTO_UDP) { 2486 csum_type = TX_CSUM_UDPIP6; 2487 } else { 2488 *ctrlp = ctrl; 2489 return (COS_FAIL); 2490 } 2491 2492 ASSERT(csum_type != -1); 2493 ctrl &= ~F_TXPKT_L4CSUM_DIS; 2494 ctrl |= V_TXPKT_CSUM_TYPE(csum_type); 2495 } 2496 2497 if ((ctrl & F_TXPKT_IPCSUM_DIS) == 0 && 2498 (ctrl & F_TXPKT_L4CSUM_DIS) != 0) { 2499 /* 2500 * If only the IPv4 checksum is requested, we need to set an 2501 * appropriate type in the command for it. 2502 */ 2503 ctrl |= V_TXPKT_CSUM_TYPE(TX_CSUM_IP); 2504 } 2505 2506 ASSERT(ctrl != (F_TXPKT_L4CSUM_DIS | F_TXPKT_IPCSUM_DIS)); 2507 2508 /* 2509 * Fill in the requisite L2/L3 header length data. 2510 * 2511 * The Ethernet header length is recorded as 'size - 14 bytes' 2512 */ 2513 const uint8_t eth_len = meoi->meoi_l2hlen - 14; 2514 if (chip_version >= CHELSIO_T6) { 2515 ctrl |= V_T6_TXPKT_ETHHDR_LEN(eth_len); 2516 } else { 2517 ctrl |= V_TXPKT_ETHHDR_LEN(eth_len); 2518 } 2519 ctrl |= V_TXPKT_IPHDR_LEN(meoi->meoi_l3hlen); 2520 2521 *ctrlp = ctrl; 2522 return (COS_SUCCESS); 2523 } 2524 2525 static int 2526 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m, 2527 struct txinfo *txinfo) 2528 { 2529 struct sge_eq *eq = &txq->eq; 2530 struct fw_eth_tx_pkt_wr *wr; 2531 struct cpl_tx_pkt_core *cpl; 2532 uint32_t ctrl; /* used in many unrelated places */ 2533 uint64_t ctrl1; 2534 int nflits, ndesc; 2535 struct tx_sdesc *txsd; 2536 caddr_t dst; 2537 const mac_ether_offload_info_t *meoi = &txinfo->meoi; 2538 2539 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */ 2540 2541 /* 2542 * Do we have enough flits to send this frame out? 2543 */ 2544 ctrl = sizeof (struct cpl_tx_pkt_core); 2545 if (txinfo->flags & HW_LSO) { 2546 nflits = TXPKT_LSO_WR_HDR; 2547 ctrl += sizeof (struct cpl_tx_pkt_lso_core); 2548 } else { 2549 nflits = TXPKT_WR_HDR; 2550 } 2551 if (txinfo->nsegs > 0) 2552 nflits += txinfo->nflits; 2553 else { 2554 nflits += howmany(txinfo->len, 8); 2555 ctrl += txinfo->len; 2556 } 2557 ndesc = howmany(nflits, 8); 2558 if (ndesc > eq->avail) 2559 return (ENOMEM); 2560 2561 /* Firmware work request header */ 2562 wr = (void *)&eq->desc[eq->pidx]; 2563 wr->op_immdlen = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | 2564 V_FW_WR_IMMDLEN(ctrl)); 2565 ctrl = V_FW_WR_LEN16(howmany(nflits, 2)); 2566 if (eq->avail == ndesc) 2567 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ; 2568 wr->equiq_to_len16 = cpu_to_be32(ctrl); 2569 wr->r3 = 0; 2570 2571 if (txinfo->flags & HW_LSO && 2572 (meoi->meoi_flags & MEOI_L4INFO_SET) != 0 && 2573 meoi->meoi_l4proto == IPPROTO_TCP) { 2574 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); 2575 2576 ctrl = V_LSO_OPCODE((u32)CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | 2577 F_LSO_LAST_SLICE; 2578 2579 if (meoi->meoi_l2hlen > sizeof (struct ether_header)) { 2580 /* 2581 * This presently assumes a standard VLAN header, 2582 * without support for Q-in-Q. 2583 */ 2584 ctrl |= V_LSO_ETHHDR_LEN(1); 2585 } 2586 2587 switch (meoi->meoi_l3proto) { 2588 case ETHERTYPE_IPV6: 2589 ctrl |= F_LSO_IPV6; 2590 /* FALLTHROUGH */ 2591 case ETHERTYPE_IP: 2592 ctrl |= V_LSO_IPHDR_LEN(meoi->meoi_l3hlen / 4); 2593 break; 2594 default: 2595 break; 2596 } 2597 2598 ctrl |= V_LSO_TCPHDR_LEN(meoi->meoi_l4hlen / 4); 2599 2600 lso->lso_ctrl = cpu_to_be32(ctrl); 2601 lso->ipid_ofst = cpu_to_be16(0); 2602 lso->mss = cpu_to_be16(txinfo->mss); 2603 lso->seqno_offset = cpu_to_be32(0); 2604 if (is_t4(pi->adapter->params.chip)) 2605 lso->len = cpu_to_be32(txinfo->len); 2606 else 2607 lso->len = cpu_to_be32(V_LSO_T5_XFER_SIZE(txinfo->len)); 2608 2609 cpl = (void *)(lso + 1); 2610 2611 txq->tso_wrs++; 2612 } else { 2613 cpl = (void *)(wr + 1); 2614 } 2615 2616 /* Checksum offload */ 2617 switch (csum_to_ctrl(txinfo, 2618 CHELSIO_CHIP_VERSION(pi->adapter->params.chip), &ctrl1)) { 2619 case COS_SUCCESS: 2620 txq->txcsum++; 2621 break; 2622 case COS_FAIL: 2623 /* 2624 * Packet will be going out with checksums which are probably 2625 * wrong but there is little we can do now. 2626 */ 2627 txq->csum_failed++; 2628 break; 2629 default: 2630 break; 2631 } 2632 2633 /* CPL header */ 2634 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | 2635 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf)); 2636 cpl->pack = 0; 2637 cpl->len = cpu_to_be16(txinfo->len); 2638 cpl->ctrl1 = cpu_to_be64(ctrl1); 2639 2640 /* Software descriptor */ 2641 txsd = &txq->sdesc[eq->pidx]; 2642 txsd->m = m; 2643 txsd->txb_used = txinfo->txb_used; 2644 txsd->hdls_used = txinfo->hdls_used; 2645 /* LINTED: E_ASSIGN_NARROW_CONV */ 2646 txsd->desc_used = ndesc; 2647 2648 txq->txb_used += txinfo->txb_used / TXB_CHUNK; 2649 txq->hdl_used += txinfo->hdls_used; 2650 2651 eq->pending += ndesc; 2652 eq->avail -= ndesc; 2653 eq->pidx += ndesc; 2654 if (eq->pidx >= eq->cap) 2655 eq->pidx -= eq->cap; 2656 2657 /* SGL */ 2658 dst = (void *)(cpl + 1); 2659 if (txinfo->nsegs > 0) { 2660 txq->sgl_wrs++; 2661 copy_to_txd(eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8); 2662 2663 /* Need to zero-pad to a 16 byte boundary if not on one */ 2664 if ((uintptr_t)dst & 0xf) 2665 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 2666 *(uint64_t *)dst = 0; 2667 2668 } else { 2669 txq->imm_wrs++; 2670 #ifdef DEBUG 2671 ctrl = txinfo->len; 2672 #endif 2673 for (; m; m = m->b_cont) { 2674 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m)); 2675 #ifdef DEBUG 2676 ctrl -= MBLKL(m); 2677 #endif 2678 } 2679 ASSERT(ctrl == 0); 2680 } 2681 2682 txq->txpkt_wrs++; 2683 return (0); 2684 } 2685 2686 static inline void 2687 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq, 2688 struct txpkts *txpkts, struct txinfo *txinfo) 2689 { 2690 struct ulp_txpkt *ulpmc; 2691 struct ulptx_idata *ulpsc; 2692 struct cpl_tx_pkt_core *cpl; 2693 uintptr_t flitp, start, end; 2694 uint64_t ctrl; 2695 caddr_t dst; 2696 2697 ASSERT(txpkts->npkt > 0); 2698 2699 start = (uintptr_t)txq->eq.desc; 2700 end = (uintptr_t)txq->eq.spg; 2701 2702 /* Checksum offload */ 2703 switch (csum_to_ctrl(txinfo, 2704 CHELSIO_CHIP_VERSION(pi->adapter->params.chip), &ctrl)) { 2705 case COS_SUCCESS: 2706 txq->txcsum++; 2707 break; 2708 case COS_FAIL: 2709 /* 2710 * Packet will be going out with checksums which are probably 2711 * wrong but there is little we can do now. 2712 */ 2713 txq->csum_failed++; 2714 break; 2715 default: 2716 break; 2717 } 2718 2719 /* 2720 * The previous packet's SGL must have ended at a 16 byte boundary (this 2721 * is required by the firmware/hardware). It follows that flitp cannot 2722 * wrap around between the ULPTX master command and ULPTX subcommand (8 2723 * bytes each), and that it can not wrap around in the middle of the 2724 * cpl_tx_pkt_core either. 2725 */ 2726 flitp = (uintptr_t)txpkts->flitp; 2727 ASSERT((flitp & 0xf) == 0); 2728 2729 /* ULP master command */ 2730 ulpmc = (void *)flitp; 2731 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); 2732 ulpmc->len = htonl(howmany(sizeof (*ulpmc) + sizeof (*ulpsc) + 2733 sizeof (*cpl) + 8 * txinfo->nflits, 16)); 2734 2735 /* ULP subcommand */ 2736 ulpsc = (void *)(ulpmc + 1); 2737 ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) | 2738 F_ULP_TX_SC_MORE); 2739 ulpsc->len = cpu_to_be32(sizeof (struct cpl_tx_pkt_core)); 2740 2741 flitp += sizeof (*ulpmc) + sizeof (*ulpsc); 2742 if (flitp == end) 2743 flitp = start; 2744 2745 /* CPL_TX_PKT_XT */ 2746 cpl = (void *)flitp; 2747 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | 2748 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf)); 2749 cpl->pack = 0; 2750 cpl->len = cpu_to_be16(txinfo->len); 2751 cpl->ctrl1 = cpu_to_be64(ctrl); 2752 2753 flitp += sizeof (*cpl); 2754 if (flitp == end) 2755 flitp = start; 2756 2757 /* SGL for this frame */ 2758 dst = (caddr_t)flitp; 2759 copy_to_txd(&txq->eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8); 2760 flitp = (uintptr_t)dst; 2761 2762 /* Zero pad and advance to a 16 byte boundary if not already at one. */ 2763 if (flitp & 0xf) { 2764 2765 /* no matter what, flitp should be on an 8 byte boundary */ 2766 ASSERT((flitp & 0x7) == 0); 2767 2768 *(uint64_t *)flitp = 0; 2769 flitp += sizeof (uint64_t); 2770 txpkts->nflits++; 2771 } 2772 2773 if (flitp == end) 2774 flitp = start; 2775 2776 txpkts->flitp = (void *)flitp; 2777 } 2778 2779 static inline void 2780 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) 2781 { 2782 if ((uintptr_t)(*to) + len <= (uintptr_t)eq->spg) { 2783 bcopy(from, *to, len); 2784 (*to) += len; 2785 } else { 2786 int portion = (uintptr_t)eq->spg - (uintptr_t)(*to); 2787 2788 bcopy(from, *to, portion); 2789 from += portion; 2790 portion = len - portion; /* remaining */ 2791 bcopy(from, (void *)eq->desc, portion); 2792 (*to) = (caddr_t)eq->desc + portion; 2793 } 2794 } 2795 2796 static inline void 2797 ring_tx_db(struct adapter *sc, struct sge_eq *eq) 2798 { 2799 int val, db_mode; 2800 t4_doorbells_t db = eq->doorbells; 2801 2802 if (eq->pending > 1) 2803 db &= ~DOORBELL_WCWR; 2804 2805 if (eq->pending > eq->pidx) { 2806 int offset = eq->cap - (eq->pending - eq->pidx); 2807 2808 /* pidx has wrapped around since last doorbell */ 2809 2810 (void) ddi_dma_sync(eq->desc_dhdl, 2811 offset * sizeof (struct tx_desc), 0, 2812 DDI_DMA_SYNC_FORDEV); 2813 (void) ddi_dma_sync(eq->desc_dhdl, 2814 0, eq->pidx * sizeof (struct tx_desc), 2815 DDI_DMA_SYNC_FORDEV); 2816 } else if (eq->pending > 0) { 2817 (void) ddi_dma_sync(eq->desc_dhdl, 2818 (eq->pidx - eq->pending) * sizeof (struct tx_desc), 2819 eq->pending * sizeof (struct tx_desc), 2820 DDI_DMA_SYNC_FORDEV); 2821 } 2822 2823 membar_producer(); 2824 2825 if (is_t4(sc->params.chip)) 2826 val = V_PIDX(eq->pending); 2827 else 2828 val = V_PIDX_T5(eq->pending); 2829 2830 db_mode = (1 << (ffs(db) - 1)); 2831 switch (db_mode) { 2832 case DOORBELL_UDB: 2833 *eq->udb = LE_32(V_QID(eq->udb_qid) | val); 2834 break; 2835 2836 case DOORBELL_WCWR: 2837 { 2838 volatile uint64_t *dst, *src; 2839 int i; 2840 /* 2841 * Queues whose 128B doorbell segment fits in 2842 * the page do not use relative qid 2843 * (udb_qid is always 0). Only queues with 2844 * doorbell segments can do WCWR. 2845 */ 2846 ASSERT(eq->udb_qid == 0 && eq->pending == 1); 2847 2848 dst = (volatile void *)((uintptr_t)eq->udb + 2849 UDBS_WR_OFFSET - UDBS_DB_OFFSET); 2850 i = eq->pidx ? eq->pidx - 1 : eq->cap - 1; 2851 src = (void *)&eq->desc[i]; 2852 while (src != (void *)&eq->desc[i + 1]) { 2853 *dst++ = *src++; 2854 } 2855 membar_producer(); 2856 break; 2857 } 2858 2859 case DOORBELL_UDBWC: 2860 *eq->udb = LE_32(V_QID(eq->udb_qid) | val); 2861 membar_producer(); 2862 break; 2863 2864 case DOORBELL_KDB: 2865 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), 2866 V_QID(eq->cntxt_id) | val); 2867 break; 2868 } 2869 2870 eq->pending = 0; 2871 } 2872 2873 static int 2874 reclaim_tx_descs(struct sge_txq *txq, int howmany) 2875 { 2876 struct tx_sdesc *txsd; 2877 uint_t cidx, can_reclaim, reclaimed, txb_freed, hdls_freed; 2878 struct sge_eq *eq = &txq->eq; 2879 2880 EQ_LOCK_ASSERT_OWNED(eq); 2881 2882 cidx = eq->spg->cidx; /* stable snapshot */ 2883 cidx = be16_to_cpu(cidx); 2884 2885 if (cidx >= eq->cidx) 2886 can_reclaim = cidx - eq->cidx; 2887 else 2888 can_reclaim = cidx + eq->cap - eq->cidx; 2889 2890 if (can_reclaim == 0) 2891 return (0); 2892 2893 txb_freed = hdls_freed = reclaimed = 0; 2894 do { 2895 int ndesc; 2896 2897 txsd = &txq->sdesc[eq->cidx]; 2898 ndesc = txsd->desc_used; 2899 2900 /* Firmware doesn't return "partial" credits. */ 2901 ASSERT(can_reclaim >= ndesc); 2902 2903 /* 2904 * We always keep mblk around, even for immediate data. If mblk 2905 * is NULL, this has to be the software descriptor for a credit 2906 * flush work request. 2907 */ 2908 if (txsd->m != NULL) 2909 freemsgchain(txsd->m); 2910 #ifdef DEBUG 2911 else { 2912 ASSERT(txsd->txb_used == 0); 2913 ASSERT(txsd->hdls_used == 0); 2914 ASSERT(ndesc == 1); 2915 } 2916 #endif 2917 2918 txb_freed += txsd->txb_used; 2919 hdls_freed += txsd->hdls_used; 2920 reclaimed += ndesc; 2921 2922 eq->cidx += ndesc; 2923 if (eq->cidx >= eq->cap) 2924 eq->cidx -= eq->cap; 2925 2926 can_reclaim -= ndesc; 2927 2928 } while (can_reclaim && reclaimed < howmany); 2929 2930 eq->avail += reclaimed; 2931 ASSERT(eq->avail < eq->cap); /* avail tops out at (cap - 1) */ 2932 2933 txq->txb_avail += txb_freed; 2934 2935 txq->tx_dhdl_avail += hdls_freed; 2936 ASSERT(txq->tx_dhdl_avail <= txq->tx_dhdl_total); 2937 for (; hdls_freed; hdls_freed--) { 2938 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_cidx]); 2939 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total) 2940 txq->tx_dhdl_cidx = 0; 2941 } 2942 2943 return (reclaimed); 2944 } 2945 2946 static void 2947 write_txqflush_wr(struct sge_txq *txq) 2948 { 2949 struct sge_eq *eq = &txq->eq; 2950 struct fw_eq_flush_wr *wr; 2951 struct tx_sdesc *txsd; 2952 2953 EQ_LOCK_ASSERT_OWNED(eq); 2954 ASSERT(eq->avail > 0); 2955 2956 wr = (void *)&eq->desc[eq->pidx]; 2957 bzero(wr, sizeof (*wr)); 2958 wr->opcode = FW_EQ_FLUSH_WR; 2959 wr->equiq_to_len16 = cpu_to_be32(V_FW_WR_LEN16(sizeof (*wr) / 16) | 2960 F_FW_WR_EQUEQ | F_FW_WR_EQUIQ); 2961 2962 txsd = &txq->sdesc[eq->pidx]; 2963 txsd->m = NULL; 2964 txsd->txb_used = 0; 2965 txsd->hdls_used = 0; 2966 txsd->desc_used = 1; 2967 2968 eq->pending++; 2969 eq->avail--; 2970 if (++eq->pidx == eq->cap) 2971 eq->pidx = 0; 2972 } 2973 2974 static int 2975 t4_handle_cpl_msg(struct sge_iq *iq, const struct rss_header *rss, mblk_t *mp) 2976 { 2977 const uint8_t opcode = rss->opcode; 2978 2979 DTRACE_PROBE4(t4__cpl_msg, struct sge_iq *, iq, uint8_t, opcode, 2980 const struct rss_header *, rss, mblk_t *, mp); 2981 2982 switch (opcode) { 2983 case CPL_FW4_MSG: 2984 case CPL_FW6_MSG: 2985 ASSERT3P(mp, ==, NULL); 2986 return (t4_handle_fw_msg(iq, rss)); 2987 case CPL_SGE_EGR_UPDATE: 2988 return (handle_sge_egr_update(iq, rss, mp)); 2989 case CPL_RX_PKT: 2990 return (t4_eth_rx(iq, rss, mp)); 2991 default: 2992 cxgb_printf(iq->adapter->dip, CE_WARN, 2993 "unhandled CPL opcode 0x%02x", opcode); 2994 if (mp != NULL) { 2995 freemsg(mp); 2996 } 2997 return (0); 2998 } 2999 } 3000 3001 static int 3002 t4_handle_fw_msg(struct sge_iq *iq, const struct rss_header *rss) 3003 { 3004 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1); 3005 const uint8_t msg_type = cpl->type; 3006 const struct rss_header *rss2; 3007 struct adapter *sc = iq->adapter; 3008 3009 DTRACE_PROBE3(t4__fw_msg, struct sge_iq *, iq, uint8_t, msg_type, 3010 const struct rss_header *, rss); 3011 3012 switch (msg_type) { 3013 case FW_TYPE_RSSCPL: /* also synonym for FW6_TYPE_RSSCPL */ 3014 rss2 = (const struct rss_header *)&cpl->data[0]; 3015 return (t4_handle_cpl_msg(iq, rss2, NULL)); 3016 case FW6_TYPE_CMD_RPL: 3017 return (t4_handle_fw_rpl(sc, &cpl->data[0])); 3018 default: 3019 cxgb_printf(sc->dip, CE_WARN, 3020 "unhandled fw_msg type 0x%02x", msg_type); 3021 return (0); 3022 } 3023 } 3024 3025 static int 3026 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m) 3027 { 3028 bool csum_ok; 3029 uint16_t err_vec; 3030 struct sge_rxq *rxq = (void *)iq; 3031 struct mblk_pair chain = {0}; 3032 struct adapter *sc = iq->adapter; 3033 const struct cpl_rx_pkt *cpl = (const void *)(rss + 1); 3034 3035 iq->intr_next = iq->intr_params; 3036 3037 m->b_rptr += sc->sge.pktshift; 3038 3039 /* Compressed error vector is enabled for T6 only */ 3040 if (sc->params.tp.rx_pkt_encap) 3041 /* It is enabled only in T6 config file */ 3042 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec)); 3043 else 3044 err_vec = ntohs(cpl->err_vec); 3045 3046 csum_ok = cpl->csum_calc && !err_vec; 3047 /* TODO: what about cpl->ip_frag? */ 3048 if (csum_ok && !cpl->ip_frag) { 3049 mac_hcksum_set(m, 0, 0, 0, 0xffff, 3050 HCK_FULLCKSUM_OK | HCK_FULLCKSUM | 3051 HCK_IPV4_HDRCKSUM_OK); 3052 rxq->rxcsum++; 3053 } 3054 3055 /* Add to the chain that we'll send up */ 3056 if (chain.head != NULL) 3057 chain.tail->b_next = m; 3058 else 3059 chain.head = m; 3060 chain.tail = m; 3061 3062 t4_mac_rx(rxq->port, rxq, chain.head); 3063 3064 rxq->rxpkts++; 3065 rxq->rxbytes += be16_to_cpu(cpl->len); 3066 return (0); 3067 } 3068 3069 #define FL_HW_IDX(idx) ((idx) >> 3) 3070 3071 static inline void 3072 ring_fl_db(struct adapter *sc, struct sge_fl *fl) 3073 { 3074 int desc_start, desc_last, ndesc; 3075 uint32_t v = sc->params.arch.sge_fl_db; 3076 3077 ndesc = FL_HW_IDX(fl->pending); 3078 3079 /* Hold back one credit if pidx = cidx */ 3080 if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx)) 3081 ndesc--; 3082 3083 /* 3084 * There are chances of ndesc modified above (to avoid pidx = cidx). 3085 * If there is nothing to post, return. 3086 */ 3087 if (ndesc <= 0) 3088 return; 3089 3090 desc_last = FL_HW_IDX(fl->pidx); 3091 3092 if (fl->pidx < fl->pending) { 3093 /* There was a wrap */ 3094 desc_start = FL_HW_IDX(fl->pidx + fl->cap - fl->pending); 3095 3096 /* From desc_start to the end of list */ 3097 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 0, 3098 DDI_DMA_SYNC_FORDEV); 3099 3100 /* From start of list to the desc_last */ 3101 if (desc_last != 0) 3102 (void) ddi_dma_sync(fl->dhdl, 0, desc_last * 3103 RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV); 3104 } else { 3105 /* There was no wrap, sync from start_desc to last_desc */ 3106 desc_start = FL_HW_IDX(fl->pidx - fl->pending); 3107 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 3108 ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV); 3109 } 3110 3111 if (is_t4(sc->params.chip)) 3112 v |= V_PIDX(ndesc); 3113 else 3114 v |= V_PIDX_T5(ndesc); 3115 v |= V_QID(fl->cntxt_id) | V_PIDX(ndesc); 3116 3117 membar_producer(); 3118 3119 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v); 3120 3121 /* 3122 * Update pending count: 3123 * Deduct the number of descriptors posted 3124 */ 3125 fl->pending -= ndesc * 8; 3126 } 3127 3128 static void 3129 tx_reclaim_task(void *arg) 3130 { 3131 struct sge_txq *txq = arg; 3132 3133 TXQ_LOCK(txq); 3134 reclaim_tx_descs(txq, txq->eq.qsize); 3135 TXQ_UNLOCK(txq); 3136 } 3137 3138 /* ARGSUSED */ 3139 static int 3140 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss, 3141 mblk_t *m) 3142 { 3143 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1); 3144 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid)); 3145 struct adapter *sc = iq->adapter; 3146 struct sge *s = &sc->sge; 3147 struct sge_eq *eq; 3148 struct sge_txq *txq; 3149 3150 txq = (void *)s->eqmap[qid - s->eq_start]; 3151 eq = &txq->eq; 3152 txq->qflush++; 3153 t4_mac_tx_update(txq->port, txq); 3154 3155 ddi_taskq_dispatch(sc->tq[eq->tx_chan], tx_reclaim_task, 3156 (void *)txq, DDI_NOSLEEP); 3157 3158 return (0); 3159 } 3160 3161 int 3162 t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count, 3163 int flags) 3164 { 3165 int i, rc; 3166 3167 txmaps->map_total = count; 3168 txmaps->map_avail = txmaps->map_cidx = txmaps->map_pidx = 0; 3169 3170 txmaps->map = kmem_zalloc(sizeof (ddi_dma_handle_t) * 3171 txmaps->map_total, flags); 3172 3173 for (i = 0; i < count; i++) { 3174 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx, 3175 DDI_DMA_SLEEP, 0, &txmaps->map[i]); 3176 if (rc != DDI_SUCCESS) { 3177 cxgb_printf(sc->dip, CE_WARN, 3178 "%s: failed to allocate DMA handle (%d)", 3179 __func__, rc); 3180 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 3181 } 3182 txmaps->map_avail++; 3183 } 3184 3185 return (0); 3186 } 3187 3188 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG) 3189 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR) 3190 #define KS_U_SET(x, y) kstatp->x.value.ul = (y) 3191 #define KS_U_FROM(x, y) kstatp->x.value.ul = (y)->x 3192 #define KS_C_SET(x, ...) \ 3193 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__) 3194 3195 /* 3196 * cxgbe:X:config 3197 */ 3198 struct cxgbe_port_config_kstats { 3199 kstat_named_t idx; 3200 kstat_named_t nrxq; 3201 kstat_named_t ntxq; 3202 kstat_named_t first_rxq; 3203 kstat_named_t first_txq; 3204 kstat_named_t controller; 3205 kstat_named_t factory_mac_address; 3206 }; 3207 3208 /* 3209 * cxgbe:X:info 3210 */ 3211 struct cxgbe_port_info_kstats { 3212 kstat_named_t transceiver; 3213 kstat_named_t rx_ovflow0; 3214 kstat_named_t rx_ovflow1; 3215 kstat_named_t rx_ovflow2; 3216 kstat_named_t rx_ovflow3; 3217 kstat_named_t rx_trunc0; 3218 kstat_named_t rx_trunc1; 3219 kstat_named_t rx_trunc2; 3220 kstat_named_t rx_trunc3; 3221 kstat_named_t tx_pause; 3222 kstat_named_t rx_pause; 3223 }; 3224 3225 static kstat_t * 3226 setup_port_config_kstats(struct port_info *pi) 3227 { 3228 kstat_t *ksp; 3229 struct cxgbe_port_config_kstats *kstatp; 3230 int ndata; 3231 dev_info_t *pdip = ddi_get_parent(pi->dip); 3232 uint8_t *ma = &pi->hw_addr[0]; 3233 3234 ndata = sizeof (struct cxgbe_port_config_kstats) / 3235 sizeof (kstat_named_t); 3236 3237 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "config", 3238 "net", KSTAT_TYPE_NAMED, ndata, 0); 3239 if (ksp == NULL) { 3240 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats."); 3241 return (NULL); 3242 } 3243 3244 kstatp = (struct cxgbe_port_config_kstats *)ksp->ks_data; 3245 3246 KS_UINIT(idx); 3247 KS_UINIT(nrxq); 3248 KS_UINIT(ntxq); 3249 KS_UINIT(first_rxq); 3250 KS_UINIT(first_txq); 3251 KS_CINIT(controller); 3252 KS_CINIT(factory_mac_address); 3253 3254 KS_U_SET(idx, pi->port_id); 3255 KS_U_SET(nrxq, pi->nrxq); 3256 KS_U_SET(ntxq, pi->ntxq); 3257 KS_U_SET(first_rxq, pi->first_rxq); 3258 KS_U_SET(first_txq, pi->first_txq); 3259 KS_C_SET(controller, "%s%d", ddi_driver_name(pdip), 3260 ddi_get_instance(pdip)); 3261 KS_C_SET(factory_mac_address, "%02X%02X%02X%02X%02X%02X", 3262 ma[0], ma[1], ma[2], ma[3], ma[4], ma[5]); 3263 3264 /* Do NOT set ksp->ks_update. These kstats do not change. */ 3265 3266 /* Install the kstat */ 3267 ksp->ks_private = (void *)pi; 3268 kstat_install(ksp); 3269 3270 return (ksp); 3271 } 3272 3273 static kstat_t * 3274 setup_port_info_kstats(struct port_info *pi) 3275 { 3276 kstat_t *ksp; 3277 struct cxgbe_port_info_kstats *kstatp; 3278 int ndata; 3279 3280 ndata = sizeof (struct cxgbe_port_info_kstats) / sizeof (kstat_named_t); 3281 3282 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "info", 3283 "net", KSTAT_TYPE_NAMED, ndata, 0); 3284 if (ksp == NULL) { 3285 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats."); 3286 return (NULL); 3287 } 3288 3289 kstatp = (struct cxgbe_port_info_kstats *)ksp->ks_data; 3290 3291 KS_CINIT(transceiver); 3292 KS_UINIT(rx_ovflow0); 3293 KS_UINIT(rx_ovflow1); 3294 KS_UINIT(rx_ovflow2); 3295 KS_UINIT(rx_ovflow3); 3296 KS_UINIT(rx_trunc0); 3297 KS_UINIT(rx_trunc1); 3298 KS_UINIT(rx_trunc2); 3299 KS_UINIT(rx_trunc3); 3300 KS_UINIT(tx_pause); 3301 KS_UINIT(rx_pause); 3302 3303 /* Install the kstat */ 3304 ksp->ks_update = update_port_info_kstats; 3305 ksp->ks_private = (void *)pi; 3306 kstat_install(ksp); 3307 3308 return (ksp); 3309 } 3310 3311 static int 3312 update_port_info_kstats(kstat_t *ksp, int rw) 3313 { 3314 struct cxgbe_port_info_kstats *kstatp = 3315 (struct cxgbe_port_info_kstats *)ksp->ks_data; 3316 struct port_info *pi = ksp->ks_private; 3317 static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX", 3318 "active TWINAX", "LRM" }; 3319 uint32_t bgmap; 3320 3321 if (rw == KSTAT_WRITE) 3322 return (0); 3323 3324 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 3325 KS_C_SET(transceiver, "unplugged"); 3326 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 3327 KS_C_SET(transceiver, "unknown"); 3328 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 3329 KS_C_SET(transceiver, "unsupported"); 3330 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str)) 3331 KS_C_SET(transceiver, "%s", mod_str[pi->mod_type]); 3332 else 3333 KS_C_SET(transceiver, "type %d", pi->mod_type); 3334 3335 #define GET_STAT(name) t4_read_reg64(pi->adapter, \ 3336 PORT_REG(pi->port_id, A_MPS_PORT_STAT_##name##_L)) 3337 #define GET_STAT_COM(name) t4_read_reg64(pi->adapter, \ 3338 A_MPS_STAT_##name##_L) 3339 3340 bgmap = G_NUMPORTS(t4_read_reg(pi->adapter, A_MPS_CMN_CTL)); 3341 if (bgmap == 0) 3342 bgmap = (pi->port_id == 0) ? 0xf : 0; 3343 else if (bgmap == 1) 3344 bgmap = (pi->port_id < 2) ? (3 << (2 * pi->port_id)) : 0; 3345 else 3346 bgmap = 1; 3347 3348 KS_U_SET(rx_ovflow0, (bgmap & 1) ? 3349 GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0); 3350 KS_U_SET(rx_ovflow1, (bgmap & 2) ? 3351 GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0); 3352 KS_U_SET(rx_ovflow2, (bgmap & 4) ? 3353 GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0); 3354 KS_U_SET(rx_ovflow3, (bgmap & 8) ? 3355 GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0); 3356 KS_U_SET(rx_trunc0, (bgmap & 1) ? 3357 GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0); 3358 KS_U_SET(rx_trunc1, (bgmap & 2) ? 3359 GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0); 3360 KS_U_SET(rx_trunc2, (bgmap & 4) ? 3361 GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0); 3362 KS_U_SET(rx_trunc3, (bgmap & 8) ? 3363 GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0); 3364 3365 KS_U_SET(tx_pause, GET_STAT(TX_PORT_PAUSE)); 3366 KS_U_SET(rx_pause, GET_STAT(RX_PORT_PAUSE)); 3367 3368 return (0); 3369 3370 } 3371 3372 /* 3373 * cxgbe:X:rxqY 3374 */ 3375 struct rxq_kstats { 3376 kstat_named_t rxcsum; 3377 kstat_named_t rxpkts; 3378 kstat_named_t rxbytes; 3379 kstat_named_t nomem; 3380 }; 3381 3382 static kstat_t * 3383 setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, int idx) 3384 { 3385 struct kstat *ksp; 3386 struct rxq_kstats *kstatp; 3387 int ndata; 3388 char str[16]; 3389 3390 ndata = sizeof (struct rxq_kstats) / sizeof (kstat_named_t); 3391 (void) snprintf(str, sizeof (str), "rxq%u", idx); 3392 3393 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "rxq", 3394 KSTAT_TYPE_NAMED, ndata, 0); 3395 if (ksp == NULL) { 3396 cxgb_printf(pi->dip, CE_WARN, 3397 "%s: failed to initialize rxq kstats for queue %d.", 3398 __func__, idx); 3399 return (NULL); 3400 } 3401 3402 kstatp = (struct rxq_kstats *)ksp->ks_data; 3403 3404 KS_UINIT(rxcsum); 3405 KS_UINIT(rxpkts); 3406 KS_UINIT(rxbytes); 3407 KS_UINIT(nomem); 3408 3409 ksp->ks_update = update_rxq_kstats; 3410 ksp->ks_private = (void *)rxq; 3411 kstat_install(ksp); 3412 3413 return (ksp); 3414 } 3415 3416 static int 3417 update_rxq_kstats(kstat_t *ksp, int rw) 3418 { 3419 struct rxq_kstats *kstatp = (struct rxq_kstats *)ksp->ks_data; 3420 struct sge_rxq *rxq = ksp->ks_private; 3421 3422 if (rw == KSTAT_WRITE) 3423 return (0); 3424 3425 KS_U_FROM(rxcsum, rxq); 3426 KS_U_FROM(rxpkts, rxq); 3427 KS_U_FROM(rxbytes, rxq); 3428 KS_U_FROM(nomem, rxq); 3429 3430 return (0); 3431 } 3432 3433 /* 3434 * cxgbe:X:txqY 3435 */ 3436 struct txq_kstats { 3437 kstat_named_t txcsum; 3438 kstat_named_t tso_wrs; 3439 kstat_named_t imm_wrs; 3440 kstat_named_t sgl_wrs; 3441 kstat_named_t txpkt_wrs; 3442 kstat_named_t txpkts_wrs; 3443 kstat_named_t txpkts_pkts; 3444 kstat_named_t txb_used; 3445 kstat_named_t hdl_used; 3446 kstat_named_t txb_full; 3447 kstat_named_t dma_hdl_failed; 3448 kstat_named_t dma_map_failed; 3449 kstat_named_t qfull; 3450 kstat_named_t qflush; 3451 kstat_named_t pullup_early; 3452 kstat_named_t pullup_late; 3453 kstat_named_t pullup_failed; 3454 kstat_named_t csum_failed; 3455 }; 3456 3457 static kstat_t * 3458 setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, int idx) 3459 { 3460 struct kstat *ksp; 3461 struct txq_kstats *kstatp; 3462 int ndata; 3463 char str[16]; 3464 3465 ndata = sizeof (struct txq_kstats) / sizeof (kstat_named_t); 3466 (void) snprintf(str, sizeof (str), "txq%u", idx); 3467 3468 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "txq", 3469 KSTAT_TYPE_NAMED, ndata, 0); 3470 if (ksp == NULL) { 3471 cxgb_printf(pi->dip, CE_WARN, 3472 "%s: failed to initialize txq kstats for queue %d.", 3473 __func__, idx); 3474 return (NULL); 3475 } 3476 3477 kstatp = (struct txq_kstats *)ksp->ks_data; 3478 3479 KS_UINIT(txcsum); 3480 KS_UINIT(tso_wrs); 3481 KS_UINIT(imm_wrs); 3482 KS_UINIT(sgl_wrs); 3483 KS_UINIT(txpkt_wrs); 3484 KS_UINIT(txpkts_wrs); 3485 KS_UINIT(txpkts_pkts); 3486 KS_UINIT(txb_used); 3487 KS_UINIT(hdl_used); 3488 KS_UINIT(txb_full); 3489 KS_UINIT(dma_hdl_failed); 3490 KS_UINIT(dma_map_failed); 3491 KS_UINIT(qfull); 3492 KS_UINIT(qflush); 3493 KS_UINIT(pullup_early); 3494 KS_UINIT(pullup_late); 3495 KS_UINIT(pullup_failed); 3496 KS_UINIT(csum_failed); 3497 3498 ksp->ks_update = update_txq_kstats; 3499 ksp->ks_private = (void *)txq; 3500 kstat_install(ksp); 3501 3502 return (ksp); 3503 } 3504 3505 static int 3506 update_txq_kstats(kstat_t *ksp, int rw) 3507 { 3508 struct txq_kstats *kstatp = (struct txq_kstats *)ksp->ks_data; 3509 struct sge_txq *txq = ksp->ks_private; 3510 3511 if (rw == KSTAT_WRITE) 3512 return (0); 3513 3514 KS_U_FROM(txcsum, txq); 3515 KS_U_FROM(tso_wrs, txq); 3516 KS_U_FROM(imm_wrs, txq); 3517 KS_U_FROM(sgl_wrs, txq); 3518 KS_U_FROM(txpkt_wrs, txq); 3519 KS_U_FROM(txpkts_wrs, txq); 3520 KS_U_FROM(txpkts_pkts, txq); 3521 KS_U_FROM(txb_used, txq); 3522 KS_U_FROM(hdl_used, txq); 3523 KS_U_FROM(txb_full, txq); 3524 KS_U_FROM(dma_hdl_failed, txq); 3525 KS_U_FROM(dma_map_failed, txq); 3526 KS_U_FROM(qfull, txq); 3527 KS_U_FROM(qflush, txq); 3528 KS_U_FROM(pullup_early, txq); 3529 KS_U_FROM(pullup_late, txq); 3530 KS_U_FROM(pullup_failed, txq); 3531 KS_U_FROM(csum_failed, txq); 3532 3533 return (0); 3534 } 3535 3536 static int rxbuf_ctor(void *, void *, int); 3537 static void rxbuf_dtor(void *, void *); 3538 3539 static kmem_cache_t * 3540 rxbuf_cache_create(struct rxbuf_cache_params *p) 3541 { 3542 char name[32]; 3543 3544 (void) snprintf(name, sizeof (name), "%s%d_rxbuf_cache", 3545 ddi_driver_name(p->dip), ddi_get_instance(p->dip)); 3546 3547 return kmem_cache_create(name, sizeof (struct rxbuf), _CACHE_LINE_SIZE, 3548 rxbuf_ctor, rxbuf_dtor, NULL, p, NULL, 0); 3549 } 3550 3551 /* 3552 * If ref_cnt is more than 1 then those many calls to rxbuf_free will 3553 * have to be made before the rxb is released back to the kmem_cache. 3554 */ 3555 static struct rxbuf * 3556 rxbuf_alloc(kmem_cache_t *cache, int kmflags, uint_t ref_cnt) 3557 { 3558 struct rxbuf *rxb; 3559 3560 ASSERT(ref_cnt > 0); 3561 3562 rxb = kmem_cache_alloc(cache, kmflags); 3563 if (rxb != NULL) { 3564 rxb->ref_cnt = ref_cnt; 3565 rxb->cache = cache; 3566 } 3567 3568 return (rxb); 3569 } 3570 3571 /* 3572 * This is normally called via the rxb's freefunc, when an mblk referencing the 3573 * rxb is freed. 3574 */ 3575 static void 3576 rxbuf_free(struct rxbuf *rxb) 3577 { 3578 if (atomic_dec_uint_nv(&rxb->ref_cnt) == 0) 3579 kmem_cache_free(rxb->cache, rxb); 3580 } 3581 3582 static int 3583 rxbuf_ctor(void *arg1, void *arg2, int kmflag) 3584 { 3585 struct rxbuf *rxb = arg1; 3586 struct rxbuf_cache_params *p = arg2; 3587 size_t real_len; 3588 ddi_dma_cookie_t cookie; 3589 uint_t ccount = 0; 3590 int (*callback)(caddr_t); 3591 int rc = ENOMEM; 3592 3593 if ((kmflag & KM_NOSLEEP) != 0) 3594 callback = DDI_DMA_DONTWAIT; 3595 else 3596 callback = DDI_DMA_SLEEP; 3597 3598 rc = ddi_dma_alloc_handle(p->dip, &p->dma_attr_rx, callback, 0, 3599 &rxb->dhdl); 3600 if (rc != DDI_SUCCESS) 3601 return (rc == DDI_DMA_BADATTR ? EINVAL : ENOMEM); 3602 3603 rc = ddi_dma_mem_alloc(rxb->dhdl, p->buf_size, &p->acc_attr_rx, 3604 DDI_DMA_STREAMING, callback, 0, &rxb->va, &real_len, &rxb->ahdl); 3605 if (rc != DDI_SUCCESS) { 3606 rc = ENOMEM; 3607 goto fail1; 3608 } 3609 3610 rc = ddi_dma_addr_bind_handle(rxb->dhdl, NULL, rxb->va, p->buf_size, 3611 DDI_DMA_READ | DDI_DMA_STREAMING, NULL, NULL, &cookie, &ccount); 3612 if (rc != DDI_DMA_MAPPED) { 3613 if (rc == DDI_DMA_INUSE) 3614 rc = EBUSY; 3615 else if (rc == DDI_DMA_TOOBIG) 3616 rc = E2BIG; 3617 else 3618 rc = ENOMEM; 3619 goto fail2; 3620 } 3621 3622 if (ccount != 1) { 3623 rc = E2BIG; 3624 goto fail3; 3625 } 3626 3627 rxb->ref_cnt = 0; 3628 rxb->buf_size = p->buf_size; 3629 rxb->freefunc.free_arg = (caddr_t)rxb; 3630 rxb->freefunc.free_func = rxbuf_free; 3631 rxb->ba = cookie.dmac_laddress; 3632 3633 return (0); 3634 3635 fail3: (void) ddi_dma_unbind_handle(rxb->dhdl); 3636 fail2: ddi_dma_mem_free(&rxb->ahdl); 3637 fail1: ddi_dma_free_handle(&rxb->dhdl); 3638 return (rc); 3639 } 3640 3641 static void 3642 rxbuf_dtor(void *arg1, void *arg2) 3643 { 3644 struct rxbuf *rxb = arg1; 3645 3646 (void) ddi_dma_unbind_handle(rxb->dhdl); 3647 ddi_dma_mem_free(&rxb->ahdl); 3648 ddi_dma_free_handle(&rxb->dhdl); 3649 } 3650