1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * This file is part of the Chelsio T4 support code. 14 * 15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved. 16 * 17 * This program is distributed in the hope that it will be useful, but WITHOUT 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this 20 * release for licensing terms and conditions. 21 */ 22 23 /* 24 * Copyright 2021 Oxide Computer Company 25 */ 26 27 #include <sys/ddi.h> 28 #include <sys/sunddi.h> 29 #include <sys/sunndi.h> 30 #include <sys/atomic.h> 31 #include <sys/dlpi.h> 32 #include <sys/pattr.h> 33 #include <sys/strsubr.h> 34 #include <sys/stream.h> 35 #include <sys/strsun.h> 36 #include <inet/ip.h> 37 #include <inet/tcp.h> 38 39 #include "version.h" 40 #include "common/common.h" 41 #include "common/t4_msg.h" 42 #include "common/t4_regs.h" 43 #include "common/t4_regs_values.h" 44 45 /* TODO: Tune. */ 46 int rx_buf_size = 8192; 47 int tx_copy_threshold = 256; 48 uint16_t rx_copy_threshold = 256; 49 50 /* Used to track coalesced tx work request */ 51 struct txpkts { 52 mblk_t *tail; /* head is in the software descriptor */ 53 uint64_t *flitp; /* ptr to flit where next pkt should start */ 54 uint8_t npkt; /* # of packets in this work request */ 55 uint8_t nflits; /* # of flits used by this work request */ 56 uint16_t plen; /* total payload (sum of all packets) */ 57 }; 58 59 /* All information needed to tx a frame */ 60 struct txinfo { 61 uint32_t len; /* Total length of frame */ 62 uint32_t flags; /* Checksum and LSO flags */ 63 uint32_t mss; /* MSS for LSO */ 64 uint8_t nsegs; /* # of segments in the SGL, 0 means imm. tx */ 65 uint8_t nflits; /* # of flits needed for the SGL */ 66 uint8_t hdls_used; /* # of DMA handles used */ 67 uint32_t txb_used; /* txb_space used */ 68 struct ulptx_sgl sgl __attribute__((aligned(8))); 69 struct ulptx_sge_pair reserved[TX_SGL_SEGS / 2]; 70 }; 71 72 static int service_iq(struct sge_iq *iq, int budget); 73 static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, 74 int8_t pktc_idx, int qsize, uint8_t esize); 75 static inline void init_fl(struct sge_fl *fl, uint16_t qsize); 76 static inline void init_eq(struct adapter *sc, struct sge_eq *eq, 77 uint16_t eqtype, uint16_t qsize,uint8_t tx_chan, uint16_t iqid); 78 static int alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, 79 struct sge_fl *fl, int intr_idx, int cong); 80 static int free_iq_fl(struct port_info *pi, struct sge_iq *iq, 81 struct sge_fl *fl); 82 static int alloc_fwq(struct adapter *sc); 83 static int free_fwq(struct adapter *sc); 84 #ifdef TCP_OFFLOAD_ENABLE 85 static int alloc_mgmtq(struct adapter *sc); 86 #endif 87 static int alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, 88 int i); 89 static int free_rxq(struct port_info *pi, struct sge_rxq *rxq); 90 #ifdef TCP_OFFLOAD_ENABLE 91 static int alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq, 92 int intr_idx); 93 static int free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq); 94 #endif 95 static int ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq); 96 static int eth_eq_alloc(struct adapter *sc, struct port_info *pi, 97 struct sge_eq *eq); 98 #ifdef TCP_OFFLOAD_ENABLE 99 static int ofld_eq_alloc(struct adapter *sc, struct port_info *pi, 100 struct sge_eq *eq); 101 #endif 102 static int alloc_eq(struct adapter *sc, struct port_info *pi, 103 struct sge_eq *eq); 104 static int free_eq(struct adapter *sc, struct sge_eq *eq); 105 #ifdef TCP_OFFLOAD_ENABLE 106 static int alloc_wrq(struct adapter *sc, struct port_info *pi, 107 struct sge_wrq *wrq, int idx); 108 static int free_wrq(struct adapter *sc, struct sge_wrq *wrq); 109 #endif 110 static int alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx); 111 static int free_txq(struct port_info *pi, struct sge_txq *txq); 112 static int alloc_dma_memory(struct adapter *sc, size_t len, int flags, 113 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr, 114 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 115 caddr_t *pva); 116 static int free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl); 117 static int alloc_desc_ring(struct adapter *sc, size_t len, int rw, 118 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 119 caddr_t *pva); 120 static int free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl); 121 static int alloc_tx_copybuffer(struct adapter *sc, size_t len, 122 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 123 caddr_t *pva); 124 static inline bool is_new_response(const struct sge_iq *iq, 125 struct rsp_ctrl **ctrl); 126 static inline void iq_next(struct sge_iq *iq); 127 static int refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs); 128 static void refill_sfl(void *arg); 129 static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl); 130 static void free_fl_bufs(struct sge_fl *fl); 131 static mblk_t *get_fl_payload(struct adapter *sc, struct sge_fl *fl, 132 uint32_t len_newbuf, int *fl_bufs_used); 133 static int get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, 134 struct txinfo *txinfo, int sgl_only); 135 static inline int fits_in_txb(struct sge_txq *txq, int len, int *waste); 136 static inline int copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, 137 struct txinfo *txinfo); 138 static inline void add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len); 139 static inline int add_mblk(struct sge_txq *txq, struct txinfo *txinfo, 140 mblk_t *m, int len); 141 static void free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo); 142 static int add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m, 143 struct txinfo *txinfo); 144 static void write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts); 145 static int write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m, 146 struct txinfo *txinfo); 147 static inline void write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq, 148 struct txpkts *txpkts, struct txinfo *txinfo); 149 static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, 150 int len); 151 static inline void ring_tx_db(struct adapter *sc, struct sge_eq *eq); 152 static int reclaim_tx_descs(struct sge_txq *txq, int howmany); 153 static void write_txqflush_wr(struct sge_txq *txq); 154 static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, 155 mblk_t *m); 156 static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl); 157 static kstat_t *setup_port_config_kstats(struct port_info *pi); 158 static kstat_t *setup_port_info_kstats(struct port_info *pi); 159 static kstat_t *setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, 160 int idx); 161 static int update_rxq_kstats(kstat_t *ksp, int rw); 162 static int update_port_info_kstats(kstat_t *ksp, int rw); 163 static kstat_t *setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, 164 int idx); 165 static int update_txq_kstats(kstat_t *ksp, int rw); 166 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *, 167 mblk_t *); 168 static int handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, 169 mblk_t *m); 170 171 static inline int 172 reclaimable(struct sge_eq *eq) 173 { 174 unsigned int cidx; 175 176 cidx = eq->spg->cidx; /* stable snapshot */ 177 cidx = be16_to_cpu(cidx); 178 179 if (cidx >= eq->cidx) 180 return (cidx - eq->cidx); 181 else 182 return (cidx + eq->cap - eq->cidx); 183 } 184 185 void 186 t4_sge_init(struct adapter *sc) 187 { 188 struct driver_properties *p = &sc->props; 189 ddi_dma_attr_t *dma_attr; 190 ddi_device_acc_attr_t *acc_attr; 191 uint32_t sge_control, sge_conm_ctrl; 192 int egress_threshold; 193 194 /* 195 * Device access and DMA attributes for descriptor rings 196 */ 197 acc_attr = &sc->sge.acc_attr_desc; 198 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 199 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 200 acc_attr->devacc_attr_dataorder = DDI_STRICTORDER_ACC; 201 202 dma_attr = &sc->sge.dma_attr_desc; 203 dma_attr->dma_attr_version = DMA_ATTR_V0; 204 dma_attr->dma_attr_addr_lo = 0; 205 dma_attr->dma_attr_addr_hi = UINT64_MAX; 206 dma_attr->dma_attr_count_max = UINT64_MAX; 207 dma_attr->dma_attr_align = 512; 208 dma_attr->dma_attr_burstsizes = 0xfff; 209 dma_attr->dma_attr_minxfer = 1; 210 dma_attr->dma_attr_maxxfer = UINT64_MAX; 211 dma_attr->dma_attr_seg = UINT64_MAX; 212 dma_attr->dma_attr_sgllen = 1; 213 dma_attr->dma_attr_granular = 1; 214 dma_attr->dma_attr_flags = 0; 215 216 /* 217 * Device access and DMA attributes for tx buffers 218 */ 219 acc_attr = &sc->sge.acc_attr_tx; 220 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 221 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 222 223 dma_attr = &sc->sge.dma_attr_tx; 224 dma_attr->dma_attr_version = DMA_ATTR_V0; 225 dma_attr->dma_attr_addr_lo = 0; 226 dma_attr->dma_attr_addr_hi = UINT64_MAX; 227 dma_attr->dma_attr_count_max = UINT64_MAX; 228 dma_attr->dma_attr_align = 1; 229 dma_attr->dma_attr_burstsizes = 0xfff; 230 dma_attr->dma_attr_minxfer = 1; 231 dma_attr->dma_attr_maxxfer = UINT64_MAX; 232 dma_attr->dma_attr_seg = UINT64_MAX; 233 dma_attr->dma_attr_sgllen = TX_SGL_SEGS; 234 dma_attr->dma_attr_granular = 1; 235 dma_attr->dma_attr_flags = 0; 236 237 /* 238 * Ingress Padding Boundary and Egress Status Page Size are set up by 239 * t4_fixup_host_params(). 240 */ 241 sge_control = t4_read_reg(sc, A_SGE_CONTROL); 242 sc->sge.pktshift = G_PKTSHIFT(sge_control); 243 sc->sge.stat_len = (sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64; 244 245 /* t4_nex uses FLM packed mode */ 246 sc->sge.fl_align = t4_fl_pkt_align(sc, true); 247 248 /* 249 * Device access and DMA attributes for rx buffers 250 */ 251 sc->sge.rxb_params.dip = sc->dip; 252 sc->sge.rxb_params.buf_size = rx_buf_size; 253 254 acc_attr = &sc->sge.rxb_params.acc_attr_rx; 255 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 256 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 257 258 dma_attr = &sc->sge.rxb_params.dma_attr_rx; 259 dma_attr->dma_attr_version = DMA_ATTR_V0; 260 dma_attr->dma_attr_addr_lo = 0; 261 dma_attr->dma_attr_addr_hi = UINT64_MAX; 262 dma_attr->dma_attr_count_max = UINT64_MAX; 263 /* 264 * Low 4 bits of an rx buffer address have a special meaning to the SGE 265 * and an rx buf cannot have an address with any of these bits set. 266 * FL_ALIGN is >= 32 so we're sure things are ok. 267 */ 268 dma_attr->dma_attr_align = sc->sge.fl_align; 269 dma_attr->dma_attr_burstsizes = 0xfff; 270 dma_attr->dma_attr_minxfer = 1; 271 dma_attr->dma_attr_maxxfer = UINT64_MAX; 272 dma_attr->dma_attr_seg = UINT64_MAX; 273 dma_attr->dma_attr_sgllen = 1; 274 dma_attr->dma_attr_granular = 1; 275 dma_attr->dma_attr_flags = 0; 276 277 sc->sge.rxbuf_cache = rxbuf_cache_create(&sc->sge.rxb_params); 278 279 /* 280 * A FL with <= fl_starve_thres buffers is starving and a periodic 281 * timer will attempt to refill it. This needs to be larger than the 282 * SGE's Egress Congestion Threshold. If it isn't, then we can get 283 * stuck waiting for new packets while the SGE is waiting for us to 284 * give it more Free List entries. (Note that the SGE's Egress 285 * Congestion Threshold is in units of 2 Free List pointers.) For T4, 286 * there was only a single field to control this. For T5 there's the 287 * original field which now only applies to Unpacked Mode Free List 288 * buffers and a new field which only applies to Packed Mode Free List 289 * buffers. 290 */ 291 292 sge_conm_ctrl = t4_read_reg(sc, A_SGE_CONM_CTRL); 293 switch (CHELSIO_CHIP_VERSION(sc->params.chip)) { 294 case CHELSIO_T4: 295 egress_threshold = G_EGRTHRESHOLD(sge_conm_ctrl); 296 break; 297 case CHELSIO_T5: 298 egress_threshold = G_EGRTHRESHOLDPACKING(sge_conm_ctrl); 299 break; 300 case CHELSIO_T6: 301 default: 302 egress_threshold = G_T6_EGRTHRESHOLDPACKING(sge_conm_ctrl); 303 } 304 sc->sge.fl_starve_threshold = 2*egress_threshold + 1; 305 306 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, rx_buf_size); 307 308 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, 309 V_THRESHOLD_0(p->counter_val[0]) | 310 V_THRESHOLD_1(p->counter_val[1]) | 311 V_THRESHOLD_2(p->counter_val[2]) | 312 V_THRESHOLD_3(p->counter_val[3])); 313 314 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, 315 V_TIMERVALUE0(us_to_core_ticks(sc, p->timer_val[0])) | 316 V_TIMERVALUE1(us_to_core_ticks(sc, p->timer_val[1]))); 317 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, 318 V_TIMERVALUE2(us_to_core_ticks(sc, p->timer_val[2])) | 319 V_TIMERVALUE3(us_to_core_ticks(sc, p->timer_val[3]))); 320 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, 321 V_TIMERVALUE4(us_to_core_ticks(sc, p->timer_val[4])) | 322 V_TIMERVALUE5(us_to_core_ticks(sc, p->timer_val[5]))); 323 324 (void) t4_register_cpl_handler(sc, CPL_FW4_MSG, handle_fw_rpl); 325 (void) t4_register_cpl_handler(sc, CPL_FW6_MSG, handle_fw_rpl); 326 (void) t4_register_cpl_handler(sc, CPL_SGE_EGR_UPDATE, handle_sge_egr_update); 327 (void) t4_register_cpl_handler(sc, CPL_RX_PKT, t4_eth_rx); 328 (void) t4_register_fw_msg_handler(sc, FW6_TYPE_CMD_RPL, 329 t4_handle_fw_rpl); 330 } 331 332 /* 333 * Allocate and initialize the firmware event queue and the forwarded interrupt 334 * queues, if any. The adapter owns all these queues as they are not associated 335 * with any particular port. 336 * 337 * Returns errno on failure. Resources allocated up to that point may still be 338 * allocated. Caller is responsible for cleanup in case this function fails. 339 */ 340 int 341 t4_setup_adapter_queues(struct adapter *sc) 342 { 343 int rc; 344 345 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 346 347 /* 348 * Firmware event queue 349 */ 350 rc = alloc_fwq(sc); 351 if (rc != 0) 352 return (rc); 353 354 #ifdef TCP_OFFLOAD_ENABLE 355 /* 356 * Management queue. This is just a control queue that uses the fwq as 357 * its associated iq. 358 */ 359 rc = alloc_mgmtq(sc); 360 #endif 361 362 return (rc); 363 } 364 365 /* 366 * Idempotent 367 */ 368 int 369 t4_teardown_adapter_queues(struct adapter *sc) 370 { 371 372 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 373 374 (void) free_fwq(sc); 375 376 return (0); 377 } 378 379 static inline int 380 first_vector(struct port_info *pi) 381 { 382 struct adapter *sc = pi->adapter; 383 int rc = T4_EXTRA_INTR, i; 384 385 if (sc->intr_count == 1) 386 return (0); 387 388 for_each_port(sc, i) { 389 struct port_info *p = sc->port[i]; 390 391 if (i == pi->port_id) 392 break; 393 394 #ifdef TCP_OFFLOAD_ENABLE 395 if (!(sc->flags & INTR_FWD)) 396 rc += p->nrxq + p->nofldrxq; 397 else 398 rc += max(p->nrxq, p->nofldrxq); 399 #else 400 /* 401 * Not compiled with offload support and intr_count > 1. Only 402 * NIC queues exist and they'd better be taking direct 403 * interrupts. 404 */ 405 ASSERT(!(sc->flags & INTR_FWD)); 406 rc += p->nrxq; 407 #endif 408 } 409 return (rc); 410 } 411 412 /* 413 * Given an arbitrary "index," come up with an iq that can be used by other 414 * queues (of this port) for interrupt forwarding, SGE egress updates, etc. 415 * The iq returned is guaranteed to be something that takes direct interrupts. 416 */ 417 static struct sge_iq * 418 port_intr_iq(struct port_info *pi, int idx) 419 { 420 struct adapter *sc = pi->adapter; 421 struct sge *s = &sc->sge; 422 struct sge_iq *iq = NULL; 423 424 if (sc->intr_count == 1) 425 return (&sc->sge.fwq); 426 427 #ifdef TCP_OFFLOAD_ENABLE 428 if (!(sc->flags & INTR_FWD)) { 429 idx %= pi->nrxq + pi->nofldrxq; 430 431 if (idx >= pi->nrxq) { 432 idx -= pi->nrxq; 433 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq; 434 } else 435 iq = &s->rxq[pi->first_rxq + idx].iq; 436 437 } else { 438 idx %= max(pi->nrxq, pi->nofldrxq); 439 440 if (pi->nrxq >= pi->nofldrxq) 441 iq = &s->rxq[pi->first_rxq + idx].iq; 442 else 443 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq; 444 } 445 #else 446 /* 447 * Not compiled with offload support and intr_count > 1. Only NIC 448 * queues exist and they'd better be taking direct interrupts. 449 */ 450 ASSERT(!(sc->flags & INTR_FWD)); 451 452 idx %= pi->nrxq; 453 iq = &s->rxq[pi->first_rxq + idx].iq; 454 #endif 455 456 return (iq); 457 } 458 459 int 460 t4_setup_port_queues(struct port_info *pi) 461 { 462 int rc = 0, i, intr_idx, j; 463 struct sge_rxq *rxq; 464 struct sge_txq *txq; 465 #ifdef TCP_OFFLOAD_ENABLE 466 int iqid; 467 struct sge_wrq *ctrlq; 468 struct sge_ofld_rxq *ofld_rxq; 469 struct sge_wrq *ofld_txq; 470 #endif 471 struct adapter *sc = pi->adapter; 472 struct driver_properties *p = &sc->props; 473 474 pi->ksp_config = setup_port_config_kstats(pi); 475 pi->ksp_info = setup_port_info_kstats(pi); 476 477 /* Interrupt vector to start from (when using multiple vectors) */ 478 intr_idx = first_vector(pi); 479 480 /* 481 * First pass over all rx queues (NIC and TOE): 482 * a) initialize iq and fl 483 * b) allocate queue iff it will take direct interrupts. 484 */ 485 486 for_each_rxq(pi, i, rxq) { 487 488 init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, p->qsize_rxq, 489 RX_IQ_ESIZE); 490 491 init_fl(&rxq->fl, p->qsize_rxq / 8); /* 8 bufs in each entry */ 492 493 if ((!(sc->flags & INTR_FWD)) 494 #ifdef TCP_OFFLOAD_ENABLE 495 || (sc->intr_count > 1 && pi->nrxq >= pi->nofldrxq) 496 #else 497 || (sc->intr_count > 1 && pi->nrxq) 498 #endif 499 ) { 500 rxq->iq.flags |= IQ_INTR; 501 rc = alloc_rxq(pi, rxq, intr_idx, i); 502 if (rc != 0) 503 goto done; 504 intr_idx++; 505 } 506 507 } 508 509 #ifdef TCP_OFFLOAD_ENABLE 510 for_each_ofld_rxq(pi, i, ofld_rxq) { 511 512 init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, 513 p->qsize_rxq, RX_IQ_ESIZE); 514 515 init_fl(&ofld_rxq->fl, p->qsize_rxq / 8); 516 517 if (!(sc->flags & INTR_FWD) || 518 (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) { 519 ofld_rxq->iq.flags = IQ_INTR; 520 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx); 521 if (rc != 0) 522 goto done; 523 524 intr_idx++; 525 } 526 } 527 #endif 528 529 /* 530 * Second pass over all rx queues (NIC and TOE). The queues forwarding 531 * their interrupts are allocated now. 532 */ 533 j = 0; 534 for_each_rxq(pi, i, rxq) { 535 if (rxq->iq.flags & IQ_INTR) 536 continue; 537 538 intr_idx = port_intr_iq(pi, j)->abs_id; 539 540 rc = alloc_rxq(pi, rxq, intr_idx, i); 541 if (rc != 0) 542 goto done; 543 j++; 544 } 545 546 #ifdef TCP_OFFLOAD_ENABLE 547 for_each_ofld_rxq(pi, i, ofld_rxq) { 548 if (ofld_rxq->iq.flags & IQ_INTR) 549 continue; 550 551 intr_idx = port_intr_iq(pi, j)->abs_id; 552 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx); 553 if (rc != 0) 554 goto done; 555 j++; 556 } 557 #endif 558 /* 559 * Now the tx queues. Only one pass needed. 560 */ 561 j = 0; 562 for_each_txq(pi, i, txq) { 563 uint16_t iqid; 564 565 iqid = port_intr_iq(pi, j)->cntxt_id; 566 init_eq(sc, &txq->eq, EQ_ETH, p->qsize_txq, pi->tx_chan, iqid); 567 rc = alloc_txq(pi, txq, i); 568 if (rc != 0) 569 goto done; 570 } 571 572 #ifdef TCP_OFFLOAD_ENABLE 573 for_each_ofld_txq(pi, i, ofld_txq) { 574 uint16_t iqid; 575 576 iqid = port_intr_iq(pi, j)->cntxt_id; 577 init_eq(sc, &ofld_txq->eq, EQ_OFLD, p->qsize_txq, pi->tx_chan, 578 iqid); 579 rc = alloc_wrq(sc, pi, ofld_txq, i); 580 if (rc != 0) 581 goto done; 582 } 583 584 /* 585 * Finally, the control queue. 586 */ 587 ctrlq = &sc->sge.ctrlq[pi->port_id]; 588 iqid = port_intr_iq(pi, 0)->cntxt_id; 589 init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, iqid); 590 rc = alloc_wrq(sc, pi, ctrlq, 0); 591 #endif 592 593 done: 594 if (rc != 0) 595 (void) t4_teardown_port_queues(pi); 596 597 return (rc); 598 } 599 600 /* 601 * Idempotent 602 */ 603 int 604 t4_teardown_port_queues(struct port_info *pi) 605 { 606 int i; 607 struct sge_rxq *rxq; 608 struct sge_txq *txq; 609 #ifdef TCP_OFFLOAD_ENABLE 610 struct adapter *sc = pi->adapter; 611 struct sge_ofld_rxq *ofld_rxq; 612 struct sge_wrq *ofld_txq; 613 #endif 614 615 if (pi->ksp_config != NULL) { 616 kstat_delete(pi->ksp_config); 617 pi->ksp_config = NULL; 618 } 619 if (pi->ksp_info != NULL) { 620 kstat_delete(pi->ksp_info); 621 pi->ksp_info = NULL; 622 } 623 624 #ifdef TCP_OFFLOAD_ENABLE 625 (void) free_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 626 #endif 627 628 for_each_txq(pi, i, txq) { 629 (void) free_txq(pi, txq); 630 } 631 632 #ifdef TCP_OFFLOAD_ENABLE 633 for_each_ofld_txq(pi, i, ofld_txq) { 634 (void) free_wrq(sc, ofld_txq); 635 } 636 637 for_each_ofld_rxq(pi, i, ofld_rxq) { 638 if ((ofld_rxq->iq.flags & IQ_INTR) == 0) 639 (void) free_ofld_rxq(pi, ofld_rxq); 640 } 641 #endif 642 643 for_each_rxq(pi, i, rxq) { 644 if ((rxq->iq.flags & IQ_INTR) == 0) 645 (void) free_rxq(pi, rxq); 646 } 647 648 /* 649 * Then take down the rx queues that take direct interrupts. 650 */ 651 652 for_each_rxq(pi, i, rxq) { 653 if (rxq->iq.flags & IQ_INTR) 654 (void) free_rxq(pi, rxq); 655 } 656 657 #ifdef TCP_OFFLOAD_ENABLE 658 for_each_ofld_rxq(pi, i, ofld_rxq) { 659 if (ofld_rxq->iq.flags & IQ_INTR) 660 (void) free_ofld_rxq(pi, ofld_rxq); 661 } 662 #endif 663 664 return (0); 665 } 666 667 /* Deals with errors and forwarded interrupts */ 668 uint_t 669 t4_intr_all(caddr_t arg1, caddr_t arg2) 670 { 671 672 (void) t4_intr_err(arg1, arg2); 673 (void) t4_intr(arg1, arg2); 674 675 return (DDI_INTR_CLAIMED); 676 } 677 678 static void 679 t4_intr_rx_work(struct sge_iq *iq) 680 { 681 mblk_t *mp = NULL; 682 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */ 683 RXQ_LOCK(rxq); 684 if (!iq->polling) { 685 mp = t4_ring_rx(rxq, iq->qsize/8); 686 t4_write_reg(iq->adapter, MYPF_REG(A_SGE_PF_GTS), 687 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next)); 688 } 689 RXQ_UNLOCK(rxq); 690 if (mp != NULL) 691 mac_rx_ring(rxq->port->mh, rxq->ring_handle, mp, 692 rxq->ring_gen_num); 693 } 694 695 /* Deals with interrupts on the given ingress queue */ 696 /* ARGSUSED */ 697 uint_t 698 t4_intr(caddr_t arg1, caddr_t arg2) 699 { 700 struct sge_iq *iq = (struct sge_iq *)arg2; 701 int state; 702 703 /* Right now receive polling is only enabled for MSI-X and 704 * when we have enough msi-x vectors i.e no interrupt forwarding. 705 */ 706 if (iq->adapter->props.multi_rings) { 707 t4_intr_rx_work(iq); 708 } else { 709 state = atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY); 710 if (state == IQS_IDLE) { 711 (void) service_iq(iq, 0); 712 (void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE); 713 } 714 } 715 return (DDI_INTR_CLAIMED); 716 } 717 718 /* Deals with error interrupts */ 719 /* ARGSUSED */ 720 uint_t 721 t4_intr_err(caddr_t arg1, caddr_t arg2) 722 { 723 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 724 struct adapter *sc = (struct adapter *)arg1; 725 726 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0); 727 (void) t4_slow_intr_handler(sc); 728 729 return (DDI_INTR_CLAIMED); 730 } 731 732 /* 733 * t4_ring_rx - Process responses from an SGE response queue. 734 * 735 * This function processes responses from an SGE response queue up to the supplied budget. 736 * Responses include received packets as well as control messages from FW 737 * or HW. 738 * It returns a chain of mblks containing the received data, to be 739 * passed up to mac_ring_rx(). 740 */ 741 mblk_t * 742 t4_ring_rx(struct sge_rxq *rxq, int budget) 743 { 744 struct sge_iq *iq = &rxq->iq; 745 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */ 746 struct adapter *sc = iq->adapter; 747 struct rsp_ctrl *ctrl; 748 const struct rss_header *rss; 749 int ndescs = 0, fl_bufs_used = 0; 750 int rsp_type; 751 uint32_t lq; 752 mblk_t *mblk_head = NULL, **mblk_tail, *m; 753 struct cpl_rx_pkt *cpl; 754 uint32_t received_bytes = 0, pkt_len = 0; 755 bool csum_ok; 756 uint16_t err_vec; 757 758 mblk_tail = &mblk_head; 759 760 while (is_new_response(iq, &ctrl)) { 761 762 membar_consumer(); 763 764 m = NULL; 765 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen); 766 lq = be32_to_cpu(ctrl->pldbuflen_qid); 767 rss = (const void *)iq->cdesc; 768 769 switch (rsp_type) { 770 case X_RSPD_TYPE_FLBUF: 771 772 ASSERT(iq->flags & IQ_HAS_FL); 773 774 if (CPL_RX_PKT == rss->opcode) { 775 cpl = (void *)(rss + 1); 776 pkt_len = be16_to_cpu(cpl->len); 777 778 if (iq->polling && ((received_bytes + pkt_len) > budget)) 779 goto done; 780 781 m = get_fl_payload(sc, fl, lq, &fl_bufs_used); 782 if (m == NULL) 783 goto done; 784 785 iq->intr_next = iq->intr_params; 786 m->b_rptr += sc->sge.pktshift; 787 if (sc->params.tp.rx_pkt_encap) 788 /* It is enabled only in T6 config file */ 789 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec)); 790 else 791 err_vec = ntohs(cpl->err_vec); 792 793 csum_ok = cpl->csum_calc && !err_vec; 794 795 /* TODO: what about cpl->ip_frag? */ 796 if (csum_ok && !cpl->ip_frag) { 797 mac_hcksum_set(m, 0, 0, 0, 0xffff, 798 HCK_FULLCKSUM_OK | HCK_FULLCKSUM | 799 HCK_IPV4_HDRCKSUM_OK); 800 rxq->rxcsum++; 801 } 802 rxq->rxpkts++; 803 rxq->rxbytes += pkt_len; 804 received_bytes += pkt_len; 805 806 *mblk_tail = m; 807 mblk_tail = &m->b_next; 808 809 break; 810 } 811 812 m = get_fl_payload(sc, fl, lq, &fl_bufs_used); 813 if (m == NULL) 814 goto done; 815 /* FALLTHROUGH */ 816 817 case X_RSPD_TYPE_CPL: 818 ASSERT(rss->opcode < NUM_CPL_CMDS); 819 sc->cpl_handler[rss->opcode](iq, rss, m); 820 break; 821 822 default: 823 break; 824 } 825 iq_next(iq); 826 ++ndescs; 827 if (!iq->polling && (ndescs == budget)) 828 break; 829 } 830 831 done: 832 833 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 834 V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) | 835 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); 836 837 if ((fl_bufs_used > 0) || (iq->flags & IQ_HAS_FL)) { 838 int starved; 839 FL_LOCK(fl); 840 fl->needed += fl_bufs_used; 841 starved = refill_fl(sc, fl, fl->cap / 8); 842 FL_UNLOCK(fl); 843 if (starved) 844 add_fl_to_sfl(sc, fl); 845 } 846 return (mblk_head); 847 } 848 849 /* 850 * Deals with anything and everything on the given ingress queue. 851 */ 852 static int 853 service_iq(struct sge_iq *iq, int budget) 854 { 855 struct sge_iq *q; 856 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */ 857 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */ 858 struct adapter *sc = iq->adapter; 859 struct rsp_ctrl *ctrl; 860 const struct rss_header *rss; 861 int ndescs = 0, limit, fl_bufs_used = 0; 862 int rsp_type; 863 uint32_t lq; 864 int starved; 865 mblk_t *m; 866 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql); 867 868 limit = budget ? budget : iq->qsize / 8; 869 870 /* 871 * We always come back and check the descriptor ring for new indirect 872 * interrupts and other responses after running a single handler. 873 */ 874 for (;;) { 875 while (is_new_response(iq, &ctrl)) { 876 877 membar_consumer(); 878 879 m = NULL; 880 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen); 881 lq = be32_to_cpu(ctrl->pldbuflen_qid); 882 rss = (const void *)iq->cdesc; 883 884 switch (rsp_type) { 885 case X_RSPD_TYPE_FLBUF: 886 887 ASSERT(iq->flags & IQ_HAS_FL); 888 889 m = get_fl_payload(sc, fl, lq, &fl_bufs_used); 890 if (m == NULL) { 891 /* 892 * Rearm the iq with a 893 * longer-than-default timer 894 */ 895 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) | 896 V_INGRESSQID((u32)iq->cntxt_id) | 897 V_SEINTARM(V_QINTR_TIMER_IDX(SGE_NTIMERS-1))); 898 if (fl_bufs_used > 0) { 899 ASSERT(iq->flags & IQ_HAS_FL); 900 FL_LOCK(fl); 901 fl->needed += fl_bufs_used; 902 starved = refill_fl(sc, fl, fl->cap / 8); 903 FL_UNLOCK(fl); 904 if (starved) 905 add_fl_to_sfl(sc, fl); 906 } 907 return (0); 908 } 909 910 /* FALLTHRU */ 911 case X_RSPD_TYPE_CPL: 912 913 ASSERT(rss->opcode < NUM_CPL_CMDS); 914 sc->cpl_handler[rss->opcode](iq, rss, m); 915 break; 916 917 case X_RSPD_TYPE_INTR: 918 919 /* 920 * Interrupts should be forwarded only to queues 921 * that are not forwarding their interrupts. 922 * This means service_iq can recurse but only 1 923 * level deep. 924 */ 925 ASSERT(budget == 0); 926 927 q = sc->sge.iqmap[lq - sc->sge.iq_start]; 928 if (atomic_cas_uint(&q->state, IQS_IDLE, 929 IQS_BUSY) == IQS_IDLE) { 930 if (service_iq(q, q->qsize / 8) == 0) { 931 (void) atomic_cas_uint( 932 &q->state, IQS_BUSY, 933 IQS_IDLE); 934 } else { 935 STAILQ_INSERT_TAIL(&iql, q, 936 link); 937 } 938 } 939 break; 940 941 default: 942 break; 943 } 944 945 iq_next(iq); 946 if (++ndescs == limit) { 947 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 948 V_CIDXINC(ndescs) | 949 V_INGRESSQID(iq->cntxt_id) | 950 V_SEINTARM(V_QINTR_TIMER_IDX( 951 X_TIMERREG_UPDATE_CIDX))); 952 ndescs = 0; 953 954 if (fl_bufs_used > 0) { 955 ASSERT(iq->flags & IQ_HAS_FL); 956 FL_LOCK(fl); 957 fl->needed += fl_bufs_used; 958 (void) refill_fl(sc, fl, fl->cap / 8); 959 FL_UNLOCK(fl); 960 fl_bufs_used = 0; 961 } 962 963 if (budget != 0) 964 return (EINPROGRESS); 965 } 966 } 967 968 if (STAILQ_EMPTY(&iql) != 0) 969 break; 970 971 /* 972 * Process the head only, and send it to the back of the list if 973 * it's still not done. 974 */ 975 q = STAILQ_FIRST(&iql); 976 STAILQ_REMOVE_HEAD(&iql, link); 977 if (service_iq(q, q->qsize / 8) == 0) 978 (void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE); 979 else 980 STAILQ_INSERT_TAIL(&iql, q, link); 981 } 982 983 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) | 984 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next)); 985 986 if (iq->flags & IQ_HAS_FL) { 987 988 FL_LOCK(fl); 989 fl->needed += fl_bufs_used; 990 starved = refill_fl(sc, fl, fl->cap / 4); 991 FL_UNLOCK(fl); 992 if (starved != 0) 993 add_fl_to_sfl(sc, fl); 994 } 995 996 return (0); 997 } 998 999 #ifdef TCP_OFFLOAD_ENABLE 1000 int 1001 t4_mgmt_tx(struct adapter *sc, mblk_t *m) 1002 { 1003 return (t4_wrq_tx(sc, &sc->sge.mgmtq, m)); 1004 } 1005 1006 /* 1007 * Doesn't fail. Holds on to work requests it can't send right away. 1008 */ 1009 int 1010 t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, mblk_t *m0) 1011 { 1012 struct sge_eq *eq = &wrq->eq; 1013 struct mblk_pair *wr_list = &wrq->wr_list; 1014 int can_reclaim; 1015 caddr_t dst; 1016 mblk_t *wr, *next; 1017 1018 TXQ_LOCK_ASSERT_OWNED(wrq); 1019 #ifdef TCP_OFFLOAD_ENABLE 1020 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_OFLD || 1021 (eq->flags & EQ_TYPEMASK) == EQ_CTRL); 1022 #else 1023 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_CTRL); 1024 #endif 1025 1026 if (m0 != NULL) { 1027 if (wr_list->head != NULL) 1028 wr_list->tail->b_next = m0; 1029 else 1030 wr_list->head = m0; 1031 while (m0->b_next) 1032 m0 = m0->b_next; 1033 wr_list->tail = m0; 1034 } 1035 1036 can_reclaim = reclaimable(eq); 1037 eq->cidx += can_reclaim; 1038 eq->avail += can_reclaim; 1039 if (eq->cidx >= eq->cap) 1040 eq->cidx -= eq->cap; 1041 1042 for (wr = wr_list->head; wr; wr = next) { 1043 int ndesc, len = 0; 1044 mblk_t *m; 1045 1046 next = wr->b_next; 1047 wr->b_next = NULL; 1048 1049 for (m = wr; m; m = m->b_cont) 1050 len += MBLKL(m); 1051 1052 ASSERT(len > 0 && (len & 0x7) == 0); 1053 ASSERT(len <= SGE_MAX_WR_LEN); 1054 1055 ndesc = howmany(len, EQ_ESIZE); 1056 if (eq->avail < ndesc) { 1057 wr->b_next = next; 1058 wrq->no_desc++; 1059 break; 1060 } 1061 1062 dst = (void *)&eq->desc[eq->pidx]; 1063 for (m = wr; m; m = m->b_cont) 1064 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m)); 1065 1066 eq->pidx += ndesc; 1067 eq->avail -= ndesc; 1068 if (eq->pidx >= eq->cap) 1069 eq->pidx -= eq->cap; 1070 1071 eq->pending += ndesc; 1072 if (eq->pending > 16) 1073 ring_tx_db(sc, eq); 1074 1075 wrq->tx_wrs++; 1076 freemsg(wr); 1077 1078 if (eq->avail < 8) { 1079 can_reclaim = reclaimable(eq); 1080 eq->cidx += can_reclaim; 1081 eq->avail += can_reclaim; 1082 if (eq->cidx >= eq->cap) 1083 eq->cidx -= eq->cap; 1084 } 1085 } 1086 1087 if (eq->pending != 0) 1088 ring_tx_db(sc, eq); 1089 1090 if (wr == NULL) 1091 wr_list->head = wr_list->tail = NULL; 1092 else { 1093 wr_list->head = wr; 1094 1095 ASSERT(wr_list->tail->b_next == NULL); 1096 } 1097 1098 return (0); 1099 } 1100 #endif 1101 1102 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */ 1103 #define TXPKTS_PKT_HDR ((\ 1104 sizeof (struct ulp_txpkt) + \ 1105 sizeof (struct ulptx_idata) + \ 1106 sizeof (struct cpl_tx_pkt_core)) / 8) 1107 1108 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */ 1109 #define TXPKTS_WR_HDR (\ 1110 sizeof (struct fw_eth_tx_pkts_wr) / 8 + \ 1111 TXPKTS_PKT_HDR) 1112 1113 /* Header of a tx WR, before SGL of first packet (in flits) */ 1114 #define TXPKT_WR_HDR ((\ 1115 sizeof (struct fw_eth_tx_pkt_wr) + \ 1116 sizeof (struct cpl_tx_pkt_core)) / 8) 1117 1118 /* Header of a tx LSO WR, before SGL of first packet (in flits) */ 1119 #define TXPKT_LSO_WR_HDR ((\ 1120 sizeof (struct fw_eth_tx_pkt_wr) + \ 1121 sizeof(struct cpl_tx_pkt_lso_core) + \ 1122 sizeof (struct cpl_tx_pkt_core)) / 8) 1123 1124 mblk_t * 1125 t4_eth_tx(void *arg, mblk_t *frame) 1126 { 1127 struct sge_txq *txq = (struct sge_txq *) arg; 1128 struct port_info *pi = txq->port; 1129 struct adapter *sc = pi->adapter; 1130 struct sge_eq *eq = &txq->eq; 1131 mblk_t *next_frame; 1132 int rc, coalescing; 1133 struct txpkts txpkts; 1134 struct txinfo txinfo; 1135 1136 txpkts.npkt = 0; /* indicates there's nothing in txpkts */ 1137 coalescing = 0; 1138 1139 TXQ_LOCK(txq); 1140 if (eq->avail < 8) 1141 (void) reclaim_tx_descs(txq, 8); 1142 for (; frame; frame = next_frame) { 1143 1144 if (eq->avail < 8) 1145 break; 1146 1147 next_frame = frame->b_next; 1148 frame->b_next = NULL; 1149 1150 if (next_frame != NULL) 1151 coalescing = 1; 1152 1153 rc = get_frame_txinfo(txq, &frame, &txinfo, coalescing); 1154 if (rc != 0) { 1155 if (rc == ENOMEM) { 1156 1157 /* Short of resources, suspend tx */ 1158 1159 frame->b_next = next_frame; 1160 break; 1161 } 1162 1163 /* 1164 * Unrecoverable error for this frame, throw it 1165 * away and move on to the next. 1166 */ 1167 1168 freemsg(frame); 1169 continue; 1170 } 1171 1172 if (coalescing != 0 && 1173 add_to_txpkts(txq, &txpkts, frame, &txinfo) == 0) { 1174 1175 /* Successfully absorbed into txpkts */ 1176 1177 write_ulp_cpl_sgl(pi, txq, &txpkts, &txinfo); 1178 goto doorbell; 1179 } 1180 1181 /* 1182 * We weren't coalescing to begin with, or current frame could 1183 * not be coalesced (add_to_txpkts flushes txpkts if a frame 1184 * given to it can't be coalesced). Either way there should be 1185 * nothing in txpkts. 1186 */ 1187 ASSERT(txpkts.npkt == 0); 1188 1189 /* We're sending out individual frames now */ 1190 coalescing = 0; 1191 1192 if (eq->avail < 8) 1193 (void) reclaim_tx_descs(txq, 8); 1194 rc = write_txpkt_wr(pi, txq, frame, &txinfo); 1195 if (rc != 0) { 1196 1197 /* Short of hardware descriptors, suspend tx */ 1198 1199 /* 1200 * This is an unlikely but expensive failure. We've 1201 * done all the hard work (DMA bindings etc.) and now we 1202 * can't send out the frame. What's worse, we have to 1203 * spend even more time freeing up everything in txinfo. 1204 */ 1205 txq->qfull++; 1206 free_txinfo_resources(txq, &txinfo); 1207 1208 frame->b_next = next_frame; 1209 break; 1210 } 1211 1212 doorbell: 1213 /* Fewer and fewer doorbells as the queue fills up */ 1214 if (eq->pending >= (1 << (fls(eq->qsize - eq->avail) / 2))) { 1215 txq->txbytes += txinfo.len; 1216 txq->txpkts++; 1217 ring_tx_db(sc, eq); 1218 } 1219 (void) reclaim_tx_descs(txq, 32); 1220 } 1221 1222 if (txpkts.npkt > 0) 1223 write_txpkts_wr(txq, &txpkts); 1224 1225 /* 1226 * frame not NULL means there was an error but we haven't thrown it 1227 * away. This can happen when we're short of tx descriptors (qfull) or 1228 * maybe even DMA handles (dma_hdl_failed). Either way, a credit flush 1229 * and reclaim will get things going again. 1230 * 1231 * If eq->avail is already 0 we know a credit flush was requested in the 1232 * WR that reduced it to 0 so we don't need another flush (we don't have 1233 * any descriptor for a flush WR anyway, duh). 1234 */ 1235 if (frame && eq->avail > 0) 1236 write_txqflush_wr(txq); 1237 1238 if (eq->pending != 0) 1239 ring_tx_db(sc, eq); 1240 1241 (void) reclaim_tx_descs(txq, eq->qsize); 1242 TXQ_UNLOCK(txq); 1243 1244 return (frame); 1245 } 1246 1247 static inline void 1248 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int8_t pktc_idx, 1249 int qsize, uint8_t esize) 1250 { 1251 ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS); 1252 ASSERT(pktc_idx < SGE_NCOUNTERS); /* -ve is ok, means don't use */ 1253 1254 iq->flags = 0; 1255 iq->adapter = sc; 1256 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx); 1257 iq->intr_pktc_idx = SGE_NCOUNTERS - 1; 1258 if (pktc_idx >= 0) { 1259 iq->intr_params |= F_QINTR_CNT_EN; 1260 iq->intr_pktc_idx = pktc_idx; 1261 } 1262 iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */ 1263 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */ 1264 } 1265 1266 static inline void 1267 init_fl(struct sge_fl *fl, uint16_t qsize) 1268 { 1269 1270 fl->qsize = qsize; 1271 fl->allocb_fail = 0; 1272 } 1273 1274 static inline void 1275 init_eq(struct adapter *sc, struct sge_eq *eq, uint16_t eqtype, uint16_t qsize, 1276 uint8_t tx_chan, uint16_t iqid) 1277 { 1278 struct sge *s = &sc->sge; 1279 uint32_t r; 1280 1281 ASSERT(tx_chan < NCHAN); 1282 ASSERT(eqtype <= EQ_TYPEMASK); 1283 1284 if (is_t5(sc->params.chip)) { 1285 r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF); 1286 r >>= S_QUEUESPERPAGEPF0 + 1287 (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf; 1288 s->s_qpp = r & M_QUEUESPERPAGEPF0; 1289 } 1290 1291 eq->flags = eqtype & EQ_TYPEMASK; 1292 eq->tx_chan = tx_chan; 1293 eq->iqid = iqid; 1294 eq->qsize = qsize; 1295 } 1296 1297 /* 1298 * Allocates the ring for an ingress queue and an optional freelist. If the 1299 * freelist is specified it will be allocated and then associated with the 1300 * ingress queue. 1301 * 1302 * Returns errno on failure. Resources allocated up to that point may still be 1303 * allocated. Caller is responsible for cleanup in case this function fails. 1304 * 1305 * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then 1306 * the intr_idx specifies the vector, starting from 0. Otherwise it specifies 1307 * the index of the queue to which its interrupts will be forwarded. 1308 */ 1309 static int 1310 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl, 1311 int intr_idx, int cong) 1312 { 1313 int rc, i, cntxt_id; 1314 size_t len; 1315 struct fw_iq_cmd c; 1316 struct adapter *sc = iq->adapter; 1317 uint32_t v = 0; 1318 1319 len = iq->qsize * iq->esize; 1320 rc = alloc_desc_ring(sc, len, DDI_DMA_READ, &iq->dhdl, &iq->ahdl, 1321 &iq->ba, (caddr_t *)&iq->desc); 1322 if (rc != 0) 1323 return (rc); 1324 1325 bzero(&c, sizeof (c)); 1326 c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | 1327 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) | 1328 V_FW_IQ_CMD_VFN(0)); 1329 1330 c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART | 1331 FW_LEN16(c)); 1332 1333 /* Special handling for firmware event queue */ 1334 if (iq == &sc->sge.fwq) 1335 v |= F_FW_IQ_CMD_IQASYNCH; 1336 1337 if (iq->flags & IQ_INTR) 1338 ASSERT(intr_idx < sc->intr_count); 1339 else 1340 v |= F_FW_IQ_CMD_IQANDST; 1341 v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx); 1342 1343 c.type_to_iqandstindex = cpu_to_be32(v | 1344 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | 1345 V_FW_IQ_CMD_VIID(pi->viid) | 1346 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT)); 1347 c.iqdroprss_to_iqesize = cpu_to_be16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | 1348 F_FW_IQ_CMD_IQGTSMODE | 1349 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) | 1350 V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4)); 1351 c.iqsize = cpu_to_be16(iq->qsize); 1352 c.iqaddr = cpu_to_be64(iq->ba); 1353 if (cong >= 0) 1354 c.iqns_to_fl0congen = BE_32(F_FW_IQ_CMD_IQFLINTCONGEN | 1355 V_FW_IQ_CMD_IQTYPE(cong ? 1356 FW_IQ_IQTYPE_NIC : FW_IQ_IQTYPE_OFLD)); 1357 1358 if (fl != NULL) { 1359 unsigned int chip_ver = CHELSIO_CHIP_VERSION(sc->params.chip); 1360 1361 mutex_init(&fl->lock, NULL, MUTEX_DRIVER, 1362 DDI_INTR_PRI(sc->intr_pri)); 1363 fl->flags |= FL_MTX; 1364 1365 len = fl->qsize * RX_FL_ESIZE; 1366 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &fl->dhdl, 1367 &fl->ahdl, &fl->ba, (caddr_t *)&fl->desc); 1368 if (rc != 0) 1369 return (rc); 1370 1371 /* Allocate space for one software descriptor per buffer. */ 1372 fl->cap = (fl->qsize - sc->sge.stat_len / RX_FL_ESIZE) * 8; 1373 fl->sdesc = kmem_zalloc(sizeof (struct fl_sdesc) * fl->cap, 1374 KM_SLEEP); 1375 fl->needed = fl->cap; 1376 fl->lowat = roundup(sc->sge.fl_starve_threshold, 8); 1377 1378 c.iqns_to_fl0congen |= 1379 cpu_to_be32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) | 1380 F_FW_IQ_CMD_FL0PACKEN | F_FW_IQ_CMD_FL0PADEN); 1381 if (cong >= 0) { 1382 c.iqns_to_fl0congen |= 1383 BE_32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) | 1384 F_FW_IQ_CMD_FL0CONGCIF | 1385 F_FW_IQ_CMD_FL0CONGEN); 1386 } 1387 1388 /* In T6, for egress queue type FL there is internal overhead 1389 * of 16B for header going into FLM module. Hence the maximum 1390 * allowed burst size is 448 bytes. For T4/T5, the hardware 1391 * doesn't coalesce fetch requests if more than 64 bytes of 1392 * Free List pointers are provided, so we use a 128-byte Fetch 1393 * Burst Minimum there (T6 implements coalescing so we can use 1394 * the smaller 64-byte value there). 1395 */ 1396 1397 c.fl0dcaen_to_fl0cidxfthresh = 1398 cpu_to_be16(V_FW_IQ_CMD_FL0FBMIN(chip_ver <= CHELSIO_T5 1399 ? X_FETCHBURSTMIN_128B 1400 : X_FETCHBURSTMIN_64B) | 1401 V_FW_IQ_CMD_FL0FBMAX(chip_ver <= CHELSIO_T5 1402 ? X_FETCHBURSTMAX_512B 1403 : X_FETCHBURSTMAX_256B)); 1404 c.fl0size = cpu_to_be16(fl->qsize); 1405 c.fl0addr = cpu_to_be64(fl->ba); 1406 } 1407 1408 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1409 if (rc != 0) { 1410 cxgb_printf(sc->dip, CE_WARN, 1411 "failed to create ingress queue: %d", rc); 1412 return (rc); 1413 } 1414 1415 iq->cdesc = iq->desc; 1416 iq->cidx = 0; 1417 iq->gen = 1; 1418 iq->intr_next = iq->intr_params; 1419 iq->adapter = sc; 1420 iq->cntxt_id = be16_to_cpu(c.iqid); 1421 iq->abs_id = be16_to_cpu(c.physiqid); 1422 iq->flags |= IQ_ALLOCATED; 1423 mutex_init(&iq->lock, NULL, 1424 MUTEX_DRIVER, DDI_INTR_PRI(DDI_INTR_PRI(sc->intr_pri))); 1425 iq->polling = 0; 1426 1427 cntxt_id = iq->cntxt_id - sc->sge.iq_start; 1428 if (cntxt_id >= sc->sge.niq) { 1429 panic("%s: iq->cntxt_id (%d) more than the max (%d)", __func__, 1430 cntxt_id, sc->sge.niq - 1); 1431 } 1432 sc->sge.iqmap[cntxt_id] = iq; 1433 1434 if (fl != NULL) { 1435 fl->cntxt_id = be16_to_cpu(c.fl0id); 1436 fl->pidx = fl->cidx = 0; 1437 fl->copy_threshold = rx_copy_threshold; 1438 1439 cntxt_id = fl->cntxt_id - sc->sge.eq_start; 1440 if (cntxt_id >= sc->sge.neq) { 1441 panic("%s: fl->cntxt_id (%d) more than the max (%d)", 1442 __func__, cntxt_id, sc->sge.neq - 1); 1443 } 1444 sc->sge.eqmap[cntxt_id] = (void *)fl; 1445 1446 FL_LOCK(fl); 1447 (void) refill_fl(sc, fl, fl->lowat); 1448 FL_UNLOCK(fl); 1449 1450 iq->flags |= IQ_HAS_FL; 1451 } 1452 1453 if (is_t5(sc->params.chip) && cong >= 0) { 1454 uint32_t param, val; 1455 1456 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 1457 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) | 1458 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id); 1459 if (cong == 0) 1460 val = 1 << 19; 1461 else { 1462 val = 2 << 19; 1463 for (i = 0; i < 4; i++) { 1464 if (cong & (1 << i)) 1465 val |= 1 << (i << 2); 1466 } 1467 } 1468 1469 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1470 if (rc != 0) { 1471 /* report error but carry on */ 1472 cxgb_printf(sc->dip, CE_WARN, 1473 "failed to set congestion manager context for " 1474 "ingress queue %d: %d", iq->cntxt_id, rc); 1475 } 1476 } 1477 1478 /* Enable IQ interrupts */ 1479 iq->state = IQS_IDLE; 1480 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) | 1481 V_INGRESSQID(iq->cntxt_id)); 1482 1483 return (0); 1484 } 1485 1486 static int 1487 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl) 1488 { 1489 int rc; 1490 1491 if (iq != NULL) { 1492 struct adapter *sc = iq->adapter; 1493 dev_info_t *dip; 1494 1495 dip = pi ? pi->dip : sc->dip; 1496 if (iq->flags & IQ_ALLOCATED) { 1497 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, 1498 FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id, 1499 fl ? fl->cntxt_id : 0xffff, 0xffff); 1500 if (rc != 0) { 1501 cxgb_printf(dip, CE_WARN, 1502 "failed to free queue %p: %d", iq, rc); 1503 return (rc); 1504 } 1505 mutex_destroy(&iq->lock); 1506 iq->flags &= ~IQ_ALLOCATED; 1507 } 1508 1509 if (iq->desc != NULL) { 1510 (void) free_desc_ring(&iq->dhdl, &iq->ahdl); 1511 iq->desc = NULL; 1512 } 1513 1514 bzero(iq, sizeof (*iq)); 1515 } 1516 1517 if (fl != NULL) { 1518 if (fl->sdesc != NULL) { 1519 FL_LOCK(fl); 1520 free_fl_bufs(fl); 1521 FL_UNLOCK(fl); 1522 1523 kmem_free(fl->sdesc, sizeof (struct fl_sdesc) * 1524 fl->cap); 1525 fl->sdesc = NULL; 1526 } 1527 1528 if (fl->desc != NULL) { 1529 (void) free_desc_ring(&fl->dhdl, &fl->ahdl); 1530 fl->desc = NULL; 1531 } 1532 1533 if (fl->flags & FL_MTX) { 1534 mutex_destroy(&fl->lock); 1535 fl->flags &= ~FL_MTX; 1536 } 1537 1538 bzero(fl, sizeof (struct sge_fl)); 1539 } 1540 1541 return (0); 1542 } 1543 1544 static int 1545 alloc_fwq(struct adapter *sc) 1546 { 1547 int rc, intr_idx; 1548 struct sge_iq *fwq = &sc->sge.fwq; 1549 1550 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE); 1551 fwq->flags |= IQ_INTR; /* always */ 1552 intr_idx = sc->intr_count > 1 ? 1 : 0; 1553 rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1); 1554 if (rc != 0) { 1555 cxgb_printf(sc->dip, CE_WARN, 1556 "failed to create firmware event queue: %d.", rc); 1557 return (rc); 1558 } 1559 1560 return (0); 1561 } 1562 1563 static int 1564 free_fwq(struct adapter *sc) 1565 { 1566 1567 return (free_iq_fl(NULL, &sc->sge.fwq, NULL)); 1568 } 1569 1570 #ifdef TCP_OFFLOAD_ENABLE 1571 static int 1572 alloc_mgmtq(struct adapter *sc) 1573 { 1574 int rc; 1575 struct sge_wrq *mgmtq = &sc->sge.mgmtq; 1576 1577 init_eq(sc, &mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan, 1578 sc->sge.fwq.cntxt_id); 1579 rc = alloc_wrq(sc, NULL, mgmtq, 0); 1580 if (rc != 0) { 1581 cxgb_printf(sc->dip, CE_WARN, 1582 "failed to create management queue: %d\n", rc); 1583 return (rc); 1584 } 1585 1586 return (0); 1587 } 1588 #endif 1589 1590 static int 1591 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int i) 1592 { 1593 int rc; 1594 1595 rxq->port = pi; 1596 rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx, 1597 t4_get_tp_ch_map(pi->adapter, pi->tx_chan)); 1598 if (rc != 0) 1599 return (rc); 1600 1601 rxq->ksp = setup_rxq_kstats(pi, rxq, i); 1602 1603 return (rc); 1604 } 1605 1606 static int 1607 free_rxq(struct port_info *pi, struct sge_rxq *rxq) 1608 { 1609 int rc; 1610 1611 if (rxq->ksp != NULL) { 1612 kstat_delete(rxq->ksp); 1613 rxq->ksp = NULL; 1614 } 1615 1616 rc = free_iq_fl(pi, &rxq->iq, &rxq->fl); 1617 if (rc == 0) 1618 bzero(&rxq->fl, sizeof (*rxq) - offsetof(struct sge_rxq, fl)); 1619 1620 return (rc); 1621 } 1622 1623 #ifdef TCP_OFFLOAD_ENABLE 1624 static int 1625 alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq, 1626 int intr_idx) 1627 { 1628 int rc; 1629 1630 rc = alloc_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx, 1631 t4_get_tp_ch_map(pi->adapter, pi->tx_chan)); 1632 if (rc != 0) 1633 return (rc); 1634 1635 return (rc); 1636 } 1637 1638 static int 1639 free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq) 1640 { 1641 int rc; 1642 1643 rc = free_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl); 1644 if (rc == 0) 1645 bzero(&ofld_rxq->fl, sizeof (*ofld_rxq) - 1646 offsetof(struct sge_ofld_rxq, fl)); 1647 1648 return (rc); 1649 } 1650 #endif 1651 1652 static int 1653 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) 1654 { 1655 int rc, cntxt_id; 1656 struct fw_eq_ctrl_cmd c; 1657 1658 bzero(&c, sizeof (c)); 1659 1660 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST | 1661 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) | 1662 V_FW_EQ_CTRL_CMD_VFN(0)); 1663 c.alloc_to_len16 = BE_32(F_FW_EQ_CTRL_CMD_ALLOC | 1664 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c)); 1665 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* TODO */ 1666 c.physeqid_pkd = BE_32(0); 1667 c.fetchszm_to_iqid = 1668 BE_32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1669 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) | 1670 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid)); 1671 c.dcaen_to_eqsize = 1672 BE_32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1673 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1674 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1675 V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize)); 1676 c.eqaddr = BE_64(eq->ba); 1677 1678 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1679 if (rc != 0) { 1680 cxgb_printf(sc->dip, CE_WARN, 1681 "failed to create control queue %d: %d", eq->tx_chan, rc); 1682 return (rc); 1683 } 1684 eq->flags |= EQ_ALLOCATED; 1685 1686 eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(BE_32(c.cmpliqid_eqid)); 1687 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1688 if (cntxt_id >= sc->sge.neq) 1689 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1690 cntxt_id, sc->sge.neq - 1); 1691 sc->sge.eqmap[cntxt_id] = eq; 1692 1693 return (rc); 1694 } 1695 1696 static int 1697 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1698 { 1699 int rc, cntxt_id; 1700 struct fw_eq_eth_cmd c; 1701 1702 bzero(&c, sizeof (c)); 1703 1704 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST | 1705 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) | 1706 V_FW_EQ_ETH_CMD_VFN(0)); 1707 c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC | 1708 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c)); 1709 c.autoequiqe_to_viid = BE_32(F_FW_EQ_ETH_CMD_AUTOEQUIQE | 1710 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(pi->viid)); 1711 c.fetchszm_to_iqid = 1712 BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1713 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | 1714 V_FW_EQ_ETH_CMD_IQID(eq->iqid)); 1715 c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1716 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1717 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1718 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize)); 1719 c.eqaddr = BE_64(eq->ba); 1720 1721 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1722 if (rc != 0) { 1723 cxgb_printf(pi->dip, CE_WARN, 1724 "failed to create Ethernet egress queue: %d", rc); 1725 return (rc); 1726 } 1727 eq->flags |= EQ_ALLOCATED; 1728 1729 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd)); 1730 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1731 if (cntxt_id >= sc->sge.neq) 1732 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1733 cntxt_id, sc->sge.neq - 1); 1734 sc->sge.eqmap[cntxt_id] = eq; 1735 1736 return (rc); 1737 } 1738 1739 #ifdef TCP_OFFLOAD_ENABLE 1740 static int 1741 ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1742 { 1743 int rc, cntxt_id; 1744 struct fw_eq_ofld_cmd c; 1745 1746 bzero(&c, sizeof (c)); 1747 1748 c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST | 1749 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) | 1750 V_FW_EQ_OFLD_CMD_VFN(0)); 1751 c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC | 1752 F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c)); 1753 c.fetchszm_to_iqid = 1754 htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1755 V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) | 1756 F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid)); 1757 c.dcaen_to_eqsize = 1758 BE_32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1759 V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1760 V_FW_EQ_OFLD_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1761 V_FW_EQ_OFLD_CMD_EQSIZE(eq->qsize)); 1762 c.eqaddr = BE_64(eq->ba); 1763 1764 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1765 if (rc != 0) { 1766 cxgb_printf(pi->dip, CE_WARN, 1767 "failed to create egress queue for TCP offload: %d", rc); 1768 return (rc); 1769 } 1770 eq->flags |= EQ_ALLOCATED; 1771 1772 eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(BE_32(c.eqid_pkd)); 1773 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1774 if (cntxt_id >= sc->sge.neq) 1775 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1776 cntxt_id, sc->sge.neq - 1); 1777 sc->sge.eqmap[cntxt_id] = eq; 1778 1779 return (rc); 1780 } 1781 #endif 1782 1783 static int 1784 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1785 { 1786 int rc; 1787 size_t len; 1788 1789 mutex_init(&eq->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(sc->intr_pri)); 1790 eq->flags |= EQ_MTX; 1791 1792 len = eq->qsize * EQ_ESIZE; 1793 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &eq->desc_dhdl, 1794 &eq->desc_ahdl, &eq->ba, (caddr_t *)&eq->desc); 1795 if (rc != 0) 1796 return (rc); 1797 1798 eq->cap = eq->qsize - sc->sge.stat_len / EQ_ESIZE; 1799 eq->spg = (void *)&eq->desc[eq->cap]; 1800 eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */ 1801 eq->pidx = eq->cidx = 0; 1802 eq->doorbells = sc->doorbells; 1803 1804 switch (eq->flags & EQ_TYPEMASK) { 1805 case EQ_CTRL: 1806 rc = ctrl_eq_alloc(sc, eq); 1807 break; 1808 1809 case EQ_ETH: 1810 rc = eth_eq_alloc(sc, pi, eq); 1811 break; 1812 1813 #ifdef TCP_OFFLOAD_ENABLE 1814 case EQ_OFLD: 1815 rc = ofld_eq_alloc(sc, pi, eq); 1816 break; 1817 #endif 1818 1819 default: 1820 panic("%s: invalid eq type %d.", __func__, 1821 eq->flags & EQ_TYPEMASK); 1822 } 1823 1824 if (eq->doorbells & 1825 (DOORBELL_UDB | DOORBELL_UDBWC | DOORBELL_WCWR)) { 1826 uint32_t s_qpp = sc->sge.s_qpp; 1827 uint32_t mask = (1 << s_qpp) - 1; 1828 volatile uint8_t *udb; 1829 1830 udb = (volatile uint8_t *)sc->reg1p + UDBS_DB_OFFSET; 1831 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */ 1832 eq->udb_qid = eq->cntxt_id & mask; /* id in page */ 1833 if (eq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE) 1834 eq->doorbells &= ~DOORBELL_WCWR; 1835 else { 1836 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */ 1837 eq->udb_qid = 0; 1838 } 1839 eq->udb = (volatile void *)udb; 1840 } 1841 1842 if (rc != 0) { 1843 cxgb_printf(sc->dip, CE_WARN, 1844 "failed to allocate egress queue(%d): %d", 1845 eq->flags & EQ_TYPEMASK, rc); 1846 } 1847 1848 return (rc); 1849 } 1850 1851 static int 1852 free_eq(struct adapter *sc, struct sge_eq *eq) 1853 { 1854 int rc; 1855 1856 if (eq->flags & EQ_ALLOCATED) { 1857 switch (eq->flags & EQ_TYPEMASK) { 1858 case EQ_CTRL: 1859 rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0, 1860 eq->cntxt_id); 1861 break; 1862 1863 case EQ_ETH: 1864 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, 1865 eq->cntxt_id); 1866 break; 1867 1868 #ifdef TCP_OFFLOAD_ENABLE 1869 case EQ_OFLD: 1870 rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0, 1871 eq->cntxt_id); 1872 break; 1873 #endif 1874 1875 default: 1876 panic("%s: invalid eq type %d.", __func__, 1877 eq->flags & EQ_TYPEMASK); 1878 } 1879 if (rc != 0) { 1880 cxgb_printf(sc->dip, CE_WARN, 1881 "failed to free egress queue (%d): %d", 1882 eq->flags & EQ_TYPEMASK, rc); 1883 return (rc); 1884 } 1885 eq->flags &= ~EQ_ALLOCATED; 1886 } 1887 1888 if (eq->desc != NULL) { 1889 (void) free_desc_ring(&eq->desc_dhdl, &eq->desc_ahdl); 1890 eq->desc = NULL; 1891 } 1892 1893 if (eq->flags & EQ_MTX) 1894 mutex_destroy(&eq->lock); 1895 1896 bzero(eq, sizeof (*eq)); 1897 return (0); 1898 } 1899 1900 #ifdef TCP_OFFLOAD_ENABLE 1901 /* ARGSUSED */ 1902 static int 1903 alloc_wrq(struct adapter *sc, struct port_info *pi, struct sge_wrq *wrq, 1904 int idx) 1905 { 1906 int rc; 1907 1908 rc = alloc_eq(sc, pi, &wrq->eq); 1909 if (rc != 0) 1910 return (rc); 1911 1912 wrq->adapter = sc; 1913 wrq->wr_list.head = NULL; 1914 wrq->wr_list.tail = NULL; 1915 1916 /* 1917 * TODO: use idx to figure out what kind of wrq this is and install 1918 * useful kstats for it. 1919 */ 1920 1921 return (rc); 1922 } 1923 1924 static int 1925 free_wrq(struct adapter *sc, struct sge_wrq *wrq) 1926 { 1927 int rc; 1928 1929 rc = free_eq(sc, &wrq->eq); 1930 if (rc != 0) 1931 return (rc); 1932 1933 bzero(wrq, sizeof (*wrq)); 1934 return (0); 1935 } 1936 #endif 1937 1938 static int 1939 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx) 1940 { 1941 int rc, i; 1942 struct adapter *sc = pi->adapter; 1943 struct sge_eq *eq = &txq->eq; 1944 1945 rc = alloc_eq(sc, pi, eq); 1946 if (rc != 0) 1947 return (rc); 1948 1949 txq->port = pi; 1950 txq->sdesc = kmem_zalloc(sizeof (struct tx_sdesc) * eq->cap, KM_SLEEP); 1951 txq->txb_size = eq->qsize * tx_copy_threshold; 1952 rc = alloc_tx_copybuffer(sc, txq->txb_size, &txq->txb_dhdl, 1953 &txq->txb_ahdl, &txq->txb_ba, &txq->txb_va); 1954 if (rc == 0) 1955 txq->txb_avail = txq->txb_size; 1956 else 1957 txq->txb_avail = txq->txb_size = 0; 1958 1959 /* 1960 * TODO: is this too low? Worst case would need around 4 times qsize 1961 * (all tx descriptors filled to the brim with SGLs, with each entry in 1962 * the SGL coming from a distinct DMA handle). Increase tx_dhdl_total 1963 * if you see too many dma_hdl_failed. 1964 */ 1965 txq->tx_dhdl_total = eq->qsize * 2; 1966 txq->tx_dhdl = kmem_zalloc(sizeof (ddi_dma_handle_t) * 1967 txq->tx_dhdl_total, KM_SLEEP); 1968 for (i = 0; i < txq->tx_dhdl_total; i++) { 1969 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx, 1970 DDI_DMA_SLEEP, 0, &txq->tx_dhdl[i]); 1971 if (rc != DDI_SUCCESS) { 1972 cxgb_printf(sc->dip, CE_WARN, 1973 "%s: failed to allocate DMA handle (%d)", 1974 __func__, rc); 1975 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 1976 } 1977 txq->tx_dhdl_avail++; 1978 } 1979 1980 txq->ksp = setup_txq_kstats(pi, txq, idx); 1981 1982 return (rc); 1983 } 1984 1985 static int 1986 free_txq(struct port_info *pi, struct sge_txq *txq) 1987 { 1988 int i; 1989 struct adapter *sc = pi->adapter; 1990 struct sge_eq *eq = &txq->eq; 1991 1992 if (txq->ksp != NULL) { 1993 kstat_delete(txq->ksp); 1994 txq->ksp = NULL; 1995 } 1996 1997 if (txq->txb_va != NULL) { 1998 (void) free_desc_ring(&txq->txb_dhdl, &txq->txb_ahdl); 1999 txq->txb_va = NULL; 2000 } 2001 2002 if (txq->sdesc != NULL) { 2003 struct tx_sdesc *sd; 2004 ddi_dma_handle_t hdl; 2005 2006 TXQ_LOCK(txq); 2007 while (eq->cidx != eq->pidx) { 2008 sd = &txq->sdesc[eq->cidx]; 2009 2010 for (i = sd->hdls_used; i; i--) { 2011 hdl = txq->tx_dhdl[txq->tx_dhdl_cidx]; 2012 (void) ddi_dma_unbind_handle(hdl); 2013 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total) 2014 txq->tx_dhdl_cidx = 0; 2015 } 2016 2017 ASSERT(sd->m); 2018 freemsgchain(sd->m); 2019 2020 eq->cidx += sd->desc_used; 2021 if (eq->cidx >= eq->cap) 2022 eq->cidx -= eq->cap; 2023 2024 txq->txb_avail += txq->txb_used; 2025 } 2026 ASSERT(txq->tx_dhdl_cidx == txq->tx_dhdl_pidx); 2027 ASSERT(txq->txb_avail == txq->txb_size); 2028 TXQ_UNLOCK(txq); 2029 2030 kmem_free(txq->sdesc, sizeof (struct tx_sdesc) * eq->cap); 2031 txq->sdesc = NULL; 2032 } 2033 2034 if (txq->tx_dhdl != NULL) { 2035 for (i = 0; i < txq->tx_dhdl_total; i++) { 2036 if (txq->tx_dhdl[i] != NULL) 2037 ddi_dma_free_handle(&txq->tx_dhdl[i]); 2038 } 2039 } 2040 2041 (void) free_eq(sc, &txq->eq); 2042 2043 bzero(txq, sizeof (*txq)); 2044 return (0); 2045 } 2046 2047 /* 2048 * Allocates a block of contiguous memory for DMA. Can be used to allocate 2049 * memory for descriptor rings or for tx/rx copy buffers. 2050 * 2051 * Caller does not have to clean up anything if this function fails, it cleans 2052 * up after itself. 2053 * 2054 * Caller provides the following: 2055 * len length of the block of memory to allocate. 2056 * flags DDI_DMA_* flags to use (CONSISTENT/STREAMING, READ/WRITE/RDWR) 2057 * acc_attr device access attributes for the allocation. 2058 * dma_attr DMA attributes for the allocation 2059 * 2060 * If the function is successful it fills up this information: 2061 * dma_hdl DMA handle for the allocated memory 2062 * acc_hdl access handle for the allocated memory 2063 * ba bus address of the allocated memory 2064 * va KVA of the allocated memory. 2065 */ 2066 static int 2067 alloc_dma_memory(struct adapter *sc, size_t len, int flags, 2068 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr, 2069 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 2070 uint64_t *pba, caddr_t *pva) 2071 { 2072 int rc; 2073 ddi_dma_handle_t dhdl; 2074 ddi_acc_handle_t ahdl; 2075 ddi_dma_cookie_t cookie; 2076 uint_t ccount; 2077 caddr_t va; 2078 size_t real_len; 2079 2080 *pva = NULL; 2081 2082 /* 2083 * DMA handle. 2084 */ 2085 rc = ddi_dma_alloc_handle(sc->dip, dma_attr, DDI_DMA_SLEEP, 0, &dhdl); 2086 if (rc != DDI_SUCCESS) { 2087 cxgb_printf(sc->dip, CE_WARN, 2088 "failed to allocate DMA handle: %d", rc); 2089 2090 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 2091 } 2092 2093 /* 2094 * Memory suitable for DMA. 2095 */ 2096 rc = ddi_dma_mem_alloc(dhdl, len, acc_attr, 2097 flags & DDI_DMA_CONSISTENT ? DDI_DMA_CONSISTENT : DDI_DMA_STREAMING, 2098 DDI_DMA_SLEEP, 0, &va, &real_len, &ahdl); 2099 if (rc != DDI_SUCCESS) { 2100 cxgb_printf(sc->dip, CE_WARN, 2101 "failed to allocate DMA memory: %d", rc); 2102 2103 ddi_dma_free_handle(&dhdl); 2104 return (ENOMEM); 2105 } 2106 2107 if (len != real_len) { 2108 cxgb_printf(sc->dip, CE_WARN, 2109 "%s: len (%u) != real_len (%u)\n", len, real_len); 2110 } 2111 2112 /* 2113 * DMA bindings. 2114 */ 2115 rc = ddi_dma_addr_bind_handle(dhdl, NULL, va, real_len, flags, NULL, 2116 NULL, &cookie, &ccount); 2117 if (rc != DDI_DMA_MAPPED) { 2118 cxgb_printf(sc->dip, CE_WARN, 2119 "failed to map DMA memory: %d", rc); 2120 2121 ddi_dma_mem_free(&ahdl); 2122 ddi_dma_free_handle(&dhdl); 2123 return (ENOMEM); 2124 } 2125 if (ccount != 1) { 2126 cxgb_printf(sc->dip, CE_WARN, 2127 "unusable DMA mapping (%d segments)", ccount); 2128 (void) free_desc_ring(&dhdl, &ahdl); 2129 } 2130 2131 bzero(va, real_len); 2132 *dma_hdl = dhdl; 2133 *acc_hdl = ahdl; 2134 *pba = cookie.dmac_laddress; 2135 *pva = va; 2136 2137 return (0); 2138 } 2139 2140 static int 2141 free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl) 2142 { 2143 (void) ddi_dma_unbind_handle(*dhdl); 2144 ddi_dma_mem_free(ahdl); 2145 ddi_dma_free_handle(dhdl); 2146 2147 return (0); 2148 } 2149 2150 static int 2151 alloc_desc_ring(struct adapter *sc, size_t len, int rw, 2152 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 2153 uint64_t *pba, caddr_t *pva) 2154 { 2155 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_desc; 2156 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; 2157 2158 return (alloc_dma_memory(sc, len, DDI_DMA_CONSISTENT | rw, acc_attr, 2159 dma_attr, dma_hdl, acc_hdl, pba, pva)); 2160 } 2161 2162 static int 2163 free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl) 2164 { 2165 return (free_dma_memory(dhdl, ahdl)); 2166 } 2167 2168 static int 2169 alloc_tx_copybuffer(struct adapter *sc, size_t len, 2170 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 2171 uint64_t *pba, caddr_t *pva) 2172 { 2173 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_tx; 2174 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; /* NOT dma_attr_tx */ 2175 2176 return (alloc_dma_memory(sc, len, DDI_DMA_STREAMING | DDI_DMA_WRITE, 2177 acc_attr, dma_attr, dma_hdl, acc_hdl, pba, pva)); 2178 } 2179 2180 static inline bool 2181 is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl) 2182 { 2183 (void) ddi_dma_sync(iq->dhdl, (uintptr_t)iq->cdesc - 2184 (uintptr_t)iq->desc, iq->esize, DDI_DMA_SYNC_FORKERNEL); 2185 2186 *ctrl = (void *)((uintptr_t)iq->cdesc + 2187 (iq->esize - sizeof (struct rsp_ctrl))); 2188 2189 return ((((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen)); 2190 } 2191 2192 static inline void 2193 iq_next(struct sge_iq *iq) 2194 { 2195 iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize); 2196 if (++iq->cidx == iq->qsize - 1) { 2197 iq->cidx = 0; 2198 iq->gen ^= 1; 2199 iq->cdesc = iq->desc; 2200 } 2201 } 2202 2203 /* 2204 * Fill up the freelist by upto nbufs and maybe ring its doorbell. 2205 * 2206 * Returns non-zero to indicate that it should be added to the list of starving 2207 * freelists. 2208 */ 2209 static int 2210 refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs) 2211 { 2212 uint64_t *d = &fl->desc[fl->pidx]; 2213 struct fl_sdesc *sd = &fl->sdesc[fl->pidx]; 2214 2215 FL_LOCK_ASSERT_OWNED(fl); 2216 ASSERT(nbufs >= 0); 2217 2218 if (nbufs > fl->needed) 2219 nbufs = fl->needed; 2220 2221 while (nbufs--) { 2222 if (sd->rxb != NULL) { 2223 if (sd->rxb->ref_cnt == 1) { 2224 /* 2225 * Buffer is available for recycling. Two ways 2226 * this can happen: 2227 * 2228 * a) All the packets DMA'd into it last time 2229 * around were within the rx_copy_threshold 2230 * and no part of the buffer was ever passed 2231 * up (ref_cnt never went over 1). 2232 * 2233 * b) Packets DMA'd into the buffer were passed 2234 * up but have all been freed by the upper 2235 * layers by now (ref_cnt went over 1 but is 2236 * now back to 1). 2237 * 2238 * Either way the bus address in the descriptor 2239 * ring is already valid. 2240 */ 2241 ASSERT(*d == cpu_to_be64(sd->rxb->ba)); 2242 d++; 2243 goto recycled; 2244 } else { 2245 /* 2246 * Buffer still in use and we need a 2247 * replacement. But first release our reference 2248 * on the existing buffer. 2249 */ 2250 rxbuf_free(sd->rxb); 2251 } 2252 } 2253 2254 sd->rxb = rxbuf_alloc(sc->sge.rxbuf_cache, KM_NOSLEEP, 1); 2255 if (sd->rxb == NULL) 2256 break; 2257 *d++ = cpu_to_be64(sd->rxb->ba); 2258 2259 recycled: fl->pending++; 2260 sd++; 2261 fl->needed--; 2262 if (++fl->pidx == fl->cap) { 2263 fl->pidx = 0; 2264 sd = fl->sdesc; 2265 d = fl->desc; 2266 } 2267 } 2268 2269 if (fl->pending >= 8) 2270 ring_fl_db(sc, fl); 2271 2272 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING)); 2273 } 2274 2275 #ifndef TAILQ_FOREACH_SAFE 2276 #define TAILQ_FOREACH_SAFE(var, head, field, tvar) \ 2277 for ((var) = TAILQ_FIRST((head)); \ 2278 (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \ 2279 (var) = (tvar)) 2280 #endif 2281 2282 /* 2283 * Attempt to refill all starving freelists. 2284 */ 2285 static void 2286 refill_sfl(void *arg) 2287 { 2288 struct adapter *sc = arg; 2289 struct sge_fl *fl, *fl_temp; 2290 2291 mutex_enter(&sc->sfl_lock); 2292 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) { 2293 FL_LOCK(fl); 2294 (void) refill_fl(sc, fl, 64); 2295 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) { 2296 TAILQ_REMOVE(&sc->sfl, fl, link); 2297 fl->flags &= ~FL_STARVING; 2298 } 2299 FL_UNLOCK(fl); 2300 } 2301 2302 if (!TAILQ_EMPTY(&sc->sfl) != 0) 2303 sc->sfl_timer = timeout(refill_sfl, sc, drv_usectohz(100000)); 2304 mutex_exit(&sc->sfl_lock); 2305 } 2306 2307 static void 2308 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl) 2309 { 2310 mutex_enter(&sc->sfl_lock); 2311 FL_LOCK(fl); 2312 if ((fl->flags & FL_DOOMED) == 0) { 2313 if (TAILQ_EMPTY(&sc->sfl) != 0) { 2314 sc->sfl_timer = timeout(refill_sfl, sc, 2315 drv_usectohz(100000)); 2316 } 2317 fl->flags |= FL_STARVING; 2318 TAILQ_INSERT_TAIL(&sc->sfl, fl, link); 2319 } 2320 FL_UNLOCK(fl); 2321 mutex_exit(&sc->sfl_lock); 2322 } 2323 2324 static void 2325 free_fl_bufs(struct sge_fl *fl) 2326 { 2327 struct fl_sdesc *sd; 2328 unsigned int i; 2329 2330 FL_LOCK_ASSERT_OWNED(fl); 2331 2332 for (i = 0; i < fl->cap; i++) { 2333 sd = &fl->sdesc[i]; 2334 2335 if (sd->rxb != NULL) { 2336 rxbuf_free(sd->rxb); 2337 sd->rxb = NULL; 2338 } 2339 } 2340 } 2341 2342 /* 2343 * Note that fl->cidx and fl->offset are left unchanged in case of failure. 2344 */ 2345 static mblk_t * 2346 get_fl_payload(struct adapter *sc, struct sge_fl *fl, 2347 uint32_t len_newbuf, int *fl_bufs_used) 2348 { 2349 struct mblk_pair frame = {0}; 2350 struct rxbuf *rxb; 2351 mblk_t *m = NULL; 2352 uint_t nbuf = 0, len, copy, n; 2353 uint32_t cidx, offset, rcidx, roffset; 2354 2355 /* 2356 * The SGE won't pack a new frame into the current buffer if the entire 2357 * payload doesn't fit in the remaining space. Move on to the next buf 2358 * in that case. 2359 */ 2360 rcidx = fl->cidx; 2361 roffset = fl->offset; 2362 if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) { 2363 fl->offset = 0; 2364 if (++fl->cidx == fl->cap) 2365 fl->cidx = 0; 2366 nbuf++; 2367 } 2368 cidx = fl->cidx; 2369 offset = fl->offset; 2370 2371 len = G_RSPD_LEN(len_newbuf); /* pktshift + payload length */ 2372 copy = (len <= fl->copy_threshold); 2373 if (copy != 0) { 2374 frame.head = m = allocb(len, BPRI_HI); 2375 if (m == NULL) { 2376 fl->allocb_fail++; 2377 cmn_err(CE_WARN,"%s: mbuf allocation failure " 2378 "count = %llu", __func__, 2379 (unsigned long long)fl->allocb_fail); 2380 fl->cidx = rcidx; 2381 fl->offset = roffset; 2382 return (NULL); 2383 } 2384 } 2385 2386 while (len) { 2387 rxb = fl->sdesc[cidx].rxb; 2388 n = min(len, rxb->buf_size - offset); 2389 2390 (void) ddi_dma_sync(rxb->dhdl, offset, n, 2391 DDI_DMA_SYNC_FORKERNEL); 2392 2393 if (copy != 0) 2394 bcopy(rxb->va + offset, m->b_wptr, n); 2395 else { 2396 m = desballoc((unsigned char *)rxb->va + offset, n, 2397 BPRI_HI, &rxb->freefunc); 2398 if (m == NULL) { 2399 fl->allocb_fail++; 2400 cmn_err(CE_WARN, 2401 "%s: mbuf allocation failure " 2402 "count = %llu", __func__, 2403 (unsigned long long)fl->allocb_fail); 2404 if (frame.head) 2405 freemsgchain(frame.head); 2406 fl->cidx = rcidx; 2407 fl->offset = roffset; 2408 return (NULL); 2409 } 2410 atomic_inc_uint(&rxb->ref_cnt); 2411 if (frame.head != NULL) 2412 frame.tail->b_cont = m; 2413 else 2414 frame.head = m; 2415 frame.tail = m; 2416 } 2417 m->b_wptr += n; 2418 len -= n; 2419 offset += roundup(n, sc->sge.fl_align); 2420 ASSERT(offset <= rxb->buf_size); 2421 if (offset == rxb->buf_size) { 2422 offset = 0; 2423 if (++cidx == fl->cap) 2424 cidx = 0; 2425 nbuf++; 2426 } 2427 } 2428 2429 fl->cidx = cidx; 2430 fl->offset = offset; 2431 (*fl_bufs_used) += nbuf; 2432 2433 ASSERT(frame.head != NULL); 2434 return (frame.head); 2435 } 2436 2437 /* 2438 * We'll do immediate data tx for non-LSO, but only when not coalescing. We're 2439 * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes 2440 * of immediate data. 2441 */ 2442 #define IMM_LEN ( \ 2443 2 * EQ_ESIZE \ 2444 - sizeof (struct fw_eth_tx_pkt_wr) \ 2445 - sizeof (struct cpl_tx_pkt_core)) 2446 2447 /* 2448 * Returns non-zero on failure, no need to cleanup anything in that case. 2449 * 2450 * Note 1: We always try to pull up the mblk if required and return E2BIG only 2451 * if this fails. 2452 * 2453 * Note 2: We'll also pullup incoming mblk if HW_LSO is set and the first mblk 2454 * does not have the TCP header in it. 2455 */ 2456 static int 2457 get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, struct txinfo *txinfo, 2458 int sgl_only) 2459 { 2460 uint32_t flags = 0, len, n; 2461 mblk_t *m = *fp; 2462 int rc; 2463 2464 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb and dma_hdls */ 2465 2466 mac_hcksum_get(m, NULL, NULL, NULL, NULL, &flags); 2467 txinfo->flags = flags; 2468 2469 mac_lso_get(m, &txinfo->mss, &flags); 2470 txinfo->flags |= flags; 2471 2472 if (flags & HW_LSO) 2473 sgl_only = 1; /* Do not allow immediate data with LSO */ 2474 2475 start: txinfo->nsegs = 0; 2476 txinfo->hdls_used = 0; 2477 txinfo->txb_used = 0; 2478 txinfo->len = 0; 2479 2480 /* total length and a rough estimate of # of segments */ 2481 n = 0; 2482 for (; m; m = m->b_cont) { 2483 len = MBLKL(m); 2484 n += (len / PAGE_SIZE) + 1; 2485 txinfo->len += len; 2486 } 2487 m = *fp; 2488 2489 if (n >= TX_SGL_SEGS || (flags & HW_LSO && MBLKL(m) < 50)) { 2490 txq->pullup_early++; 2491 m = msgpullup(*fp, -1); 2492 if (m == NULL) { 2493 txq->pullup_failed++; 2494 return (E2BIG); /* (*fp) left as it was */ 2495 } 2496 freemsg(*fp); 2497 *fp = m; 2498 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags); 2499 } 2500 2501 if (txinfo->len <= IMM_LEN && !sgl_only) 2502 return (0); /* nsegs = 0 tells caller to use imm. tx */ 2503 2504 if (txinfo->len <= txq->copy_threshold && 2505 copy_into_txb(txq, m, txinfo->len, txinfo) == 0) 2506 goto done; 2507 2508 for (; m; m = m->b_cont) { 2509 2510 len = MBLKL(m); 2511 2512 /* Use tx copy buffer if this mblk is small enough */ 2513 if (len <= txq->copy_threshold && 2514 copy_into_txb(txq, m, len, txinfo) == 0) 2515 continue; 2516 2517 /* Add DMA bindings for this mblk to the SGL */ 2518 rc = add_mblk(txq, txinfo, m, len); 2519 2520 if (rc == E2BIG || 2521 (txinfo->nsegs == TX_SGL_SEGS && m->b_cont)) { 2522 2523 txq->pullup_late++; 2524 m = msgpullup(*fp, -1); 2525 if (m != NULL) { 2526 free_txinfo_resources(txq, txinfo); 2527 freemsg(*fp); 2528 *fp = m; 2529 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags); 2530 goto start; 2531 } 2532 2533 txq->pullup_failed++; 2534 rc = E2BIG; 2535 } 2536 2537 if (rc != 0) { 2538 free_txinfo_resources(txq, txinfo); 2539 return (rc); 2540 } 2541 } 2542 2543 ASSERT(txinfo->nsegs > 0 && txinfo->nsegs <= TX_SGL_SEGS); 2544 2545 done: 2546 2547 /* 2548 * Store the # of flits required to hold this frame's SGL in nflits. An 2549 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by 2550 * multiple (len0 + len1, addr0, addr1) tuples. If addr1 is not used 2551 * then len1 must be set to 0. 2552 */ 2553 n = txinfo->nsegs - 1; 2554 txinfo->nflits = (3 * n) / 2 + (n & 1) + 2; 2555 if (n & 1) 2556 txinfo->sgl.sge[n / 2].len[1] = cpu_to_be32(0); 2557 2558 txinfo->sgl.cmd_nsge = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_DSGL) | 2559 V_ULPTX_NSGE(txinfo->nsegs)); 2560 2561 return (0); 2562 } 2563 2564 static inline int 2565 fits_in_txb(struct sge_txq *txq, int len, int *waste) 2566 { 2567 if (txq->txb_avail < len) 2568 return (0); 2569 2570 if (txq->txb_next + len <= txq->txb_size) { 2571 *waste = 0; 2572 return (1); 2573 } 2574 2575 *waste = txq->txb_size - txq->txb_next; 2576 2577 return (txq->txb_avail - *waste < len ? 0 : 1); 2578 } 2579 2580 #define TXB_CHUNK 64 2581 2582 /* 2583 * Copies the specified # of bytes into txq's tx copy buffer and updates txinfo 2584 * and txq to indicate resources used. Caller has to make sure that those many 2585 * bytes are available in the mblk chain (b_cont linked). 2586 */ 2587 static inline int 2588 copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, struct txinfo *txinfo) 2589 { 2590 int waste, n; 2591 2592 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb */ 2593 2594 if (!fits_in_txb(txq, len, &waste)) { 2595 txq->txb_full++; 2596 return (ENOMEM); 2597 } 2598 2599 if (waste != 0) { 2600 ASSERT((waste & (TXB_CHUNK - 1)) == 0); 2601 txinfo->txb_used += waste; 2602 txq->txb_avail -= waste; 2603 txq->txb_next = 0; 2604 } 2605 2606 for (n = 0; n < len; m = m->b_cont) { 2607 bcopy(m->b_rptr, txq->txb_va + txq->txb_next + n, MBLKL(m)); 2608 n += MBLKL(m); 2609 } 2610 2611 add_seg(txinfo, txq->txb_ba + txq->txb_next, len); 2612 2613 n = roundup(len, TXB_CHUNK); 2614 txinfo->txb_used += n; 2615 txq->txb_avail -= n; 2616 txq->txb_next += n; 2617 ASSERT(txq->txb_next <= txq->txb_size); 2618 if (txq->txb_next == txq->txb_size) 2619 txq->txb_next = 0; 2620 2621 return (0); 2622 } 2623 2624 static inline void 2625 add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len) 2626 { 2627 ASSERT(txinfo->nsegs < TX_SGL_SEGS); /* must have room */ 2628 2629 if (txinfo->nsegs != 0) { 2630 int idx = txinfo->nsegs - 1; 2631 txinfo->sgl.sge[idx / 2].len[idx & 1] = cpu_to_be32(len); 2632 txinfo->sgl.sge[idx / 2].addr[idx & 1] = cpu_to_be64(ba); 2633 } else { 2634 txinfo->sgl.len0 = cpu_to_be32(len); 2635 txinfo->sgl.addr0 = cpu_to_be64(ba); 2636 } 2637 txinfo->nsegs++; 2638 } 2639 2640 /* 2641 * This function cleans up any partially allocated resources when it fails so 2642 * there's nothing for the caller to clean up in that case. 2643 * 2644 * EIO indicates permanent failure. Caller should drop the frame containing 2645 * this mblk and continue. 2646 * 2647 * E2BIG indicates that the SGL length for this mblk exceeds the hardware 2648 * limit. Caller should pull up the frame before trying to send it out. 2649 * (This error means our pullup_early heuristic did not work for this frame) 2650 * 2651 * ENOMEM indicates a temporary shortage of resources (DMA handles, other DMA 2652 * resources, etc.). Caller should suspend the tx queue and wait for reclaim to 2653 * free up resources. 2654 */ 2655 static inline int 2656 add_mblk(struct sge_txq *txq, struct txinfo *txinfo, mblk_t *m, int len) 2657 { 2658 ddi_dma_handle_t dhdl; 2659 ddi_dma_cookie_t cookie; 2660 uint_t ccount = 0; 2661 int rc; 2662 2663 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate dhdls */ 2664 2665 if (txq->tx_dhdl_avail == 0) { 2666 txq->dma_hdl_failed++; 2667 return (ENOMEM); 2668 } 2669 2670 dhdl = txq->tx_dhdl[txq->tx_dhdl_pidx]; 2671 rc = ddi_dma_addr_bind_handle(dhdl, NULL, (caddr_t)m->b_rptr, len, 2672 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie, 2673 &ccount); 2674 if (rc != DDI_DMA_MAPPED) { 2675 txq->dma_map_failed++; 2676 2677 ASSERT(rc != DDI_DMA_INUSE && rc != DDI_DMA_PARTIAL_MAP); 2678 2679 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EIO); 2680 } 2681 2682 if (ccount + txinfo->nsegs > TX_SGL_SEGS) { 2683 (void) ddi_dma_unbind_handle(dhdl); 2684 return (E2BIG); 2685 } 2686 2687 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size); 2688 while (--ccount) { 2689 ddi_dma_nextcookie(dhdl, &cookie); 2690 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size); 2691 } 2692 2693 if (++txq->tx_dhdl_pidx == txq->tx_dhdl_total) 2694 txq->tx_dhdl_pidx = 0; 2695 txq->tx_dhdl_avail--; 2696 txinfo->hdls_used++; 2697 2698 return (0); 2699 } 2700 2701 /* 2702 * Releases all the txq resources used up in the specified txinfo. 2703 */ 2704 static void 2705 free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo) 2706 { 2707 int n; 2708 2709 TXQ_LOCK_ASSERT_OWNED(txq); /* dhdls, txb */ 2710 2711 n = txinfo->txb_used; 2712 if (n > 0) { 2713 txq->txb_avail += n; 2714 if (n <= txq->txb_next) 2715 txq->txb_next -= n; 2716 else { 2717 n -= txq->txb_next; 2718 txq->txb_next = txq->txb_size - n; 2719 } 2720 } 2721 2722 for (n = txinfo->hdls_used; n > 0; n--) { 2723 if (txq->tx_dhdl_pidx > 0) 2724 txq->tx_dhdl_pidx--; 2725 else 2726 txq->tx_dhdl_pidx = txq->tx_dhdl_total - 1; 2727 txq->tx_dhdl_avail++; 2728 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_pidx]); 2729 } 2730 } 2731 2732 /* 2733 * Returns 0 to indicate that m has been accepted into a coalesced tx work 2734 * request. It has either been folded into txpkts or txpkts was flushed and m 2735 * has started a new coalesced work request (as the first frame in a fresh 2736 * txpkts). 2737 * 2738 * Returns non-zero to indicate a failure - caller is responsible for 2739 * transmitting m, if there was anything in txpkts it has been flushed. 2740 */ 2741 static int 2742 add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m, 2743 struct txinfo *txinfo) 2744 { 2745 struct sge_eq *eq = &txq->eq; 2746 int can_coalesce; 2747 struct tx_sdesc *txsd; 2748 uint8_t flits; 2749 2750 TXQ_LOCK_ASSERT_OWNED(txq); 2751 2752 if (txpkts->npkt > 0) { 2753 flits = TXPKTS_PKT_HDR + txinfo->nflits; 2754 can_coalesce = (txinfo->flags & HW_LSO) == 0 && 2755 txpkts->nflits + flits <= TX_WR_FLITS && 2756 txpkts->nflits + flits <= eq->avail * 8 && 2757 txpkts->plen + txinfo->len < 65536; 2758 2759 if (can_coalesce != 0) { 2760 txpkts->tail->b_next = m; 2761 txpkts->tail = m; 2762 txpkts->npkt++; 2763 txpkts->nflits += flits; 2764 txpkts->plen += txinfo->len; 2765 2766 txsd = &txq->sdesc[eq->pidx]; 2767 txsd->txb_used += txinfo->txb_used; 2768 txsd->hdls_used += txinfo->hdls_used; 2769 2770 return (0); 2771 } 2772 2773 /* 2774 * Couldn't coalesce m into txpkts. The first order of business 2775 * is to send txpkts on its way. Then we'll revisit m. 2776 */ 2777 write_txpkts_wr(txq, txpkts); 2778 } 2779 2780 /* 2781 * Check if we can start a new coalesced tx work request with m as 2782 * the first packet in it. 2783 */ 2784 2785 ASSERT(txpkts->npkt == 0); 2786 ASSERT(txinfo->len < 65536); 2787 2788 flits = TXPKTS_WR_HDR + txinfo->nflits; 2789 can_coalesce = (txinfo->flags & HW_LSO) == 0 && 2790 flits <= eq->avail * 8 && flits <= TX_WR_FLITS; 2791 2792 if (can_coalesce == 0) 2793 return (EINVAL); 2794 2795 /* 2796 * Start a fresh coalesced tx WR with m as the first frame in it. 2797 */ 2798 txpkts->tail = m; 2799 txpkts->npkt = 1; 2800 txpkts->nflits = flits; 2801 txpkts->flitp = &eq->desc[eq->pidx].flit[2]; 2802 txpkts->plen = txinfo->len; 2803 2804 txsd = &txq->sdesc[eq->pidx]; 2805 txsd->m = m; 2806 txsd->txb_used = txinfo->txb_used; 2807 txsd->hdls_used = txinfo->hdls_used; 2808 2809 return (0); 2810 } 2811 2812 /* 2813 * Note that write_txpkts_wr can never run out of hardware descriptors (but 2814 * write_txpkt_wr can). add_to_txpkts ensures that a frame is accepted for 2815 * coalescing only if sufficient hardware descriptors are available. 2816 */ 2817 static void 2818 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts) 2819 { 2820 struct sge_eq *eq = &txq->eq; 2821 struct fw_eth_tx_pkts_wr *wr; 2822 struct tx_sdesc *txsd; 2823 uint32_t ctrl; 2824 uint16_t ndesc; 2825 2826 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */ 2827 2828 ndesc = howmany(txpkts->nflits, 8); 2829 2830 wr = (void *)&eq->desc[eq->pidx]; 2831 wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR) | 2832 V_FW_WR_IMMDLEN(0)); /* immdlen does not matter in this WR */ 2833 ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2)); 2834 if (eq->avail == ndesc) 2835 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ; 2836 wr->equiq_to_len16 = cpu_to_be32(ctrl); 2837 wr->plen = cpu_to_be16(txpkts->plen); 2838 wr->npkt = txpkts->npkt; 2839 wr->r3 = wr->type = 0; 2840 2841 /* Everything else already written */ 2842 2843 txsd = &txq->sdesc[eq->pidx]; 2844 txsd->desc_used = ndesc; 2845 2846 txq->txb_used += txsd->txb_used / TXB_CHUNK; 2847 txq->hdl_used += txsd->hdls_used; 2848 2849 ASSERT(eq->avail >= ndesc); 2850 2851 eq->pending += ndesc; 2852 eq->avail -= ndesc; 2853 eq->pidx += ndesc; 2854 if (eq->pidx >= eq->cap) 2855 eq->pidx -= eq->cap; 2856 2857 txq->txpkts_pkts += txpkts->npkt; 2858 txq->txpkts_wrs++; 2859 txpkts->npkt = 0; /* emptied */ 2860 } 2861 2862 static int 2863 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m, 2864 struct txinfo *txinfo) 2865 { 2866 struct sge_eq *eq = &txq->eq; 2867 struct fw_eth_tx_pkt_wr *wr; 2868 struct cpl_tx_pkt_core *cpl; 2869 uint32_t ctrl; /* used in many unrelated places */ 2870 uint64_t ctrl1; 2871 int nflits, ndesc; 2872 struct tx_sdesc *txsd; 2873 caddr_t dst; 2874 2875 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */ 2876 2877 /* 2878 * Do we have enough flits to send this frame out? 2879 */ 2880 ctrl = sizeof (struct cpl_tx_pkt_core); 2881 if (txinfo->flags & HW_LSO) { 2882 nflits = TXPKT_LSO_WR_HDR; 2883 ctrl += sizeof(struct cpl_tx_pkt_lso_core); 2884 } else 2885 nflits = TXPKT_WR_HDR; 2886 if (txinfo->nsegs > 0) 2887 nflits += txinfo->nflits; 2888 else { 2889 nflits += howmany(txinfo->len, 8); 2890 ctrl += txinfo->len; 2891 } 2892 ndesc = howmany(nflits, 8); 2893 if (ndesc > eq->avail) 2894 return (ENOMEM); 2895 2896 /* Firmware work request header */ 2897 wr = (void *)&eq->desc[eq->pidx]; 2898 wr->op_immdlen = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | 2899 V_FW_WR_IMMDLEN(ctrl)); 2900 ctrl = V_FW_WR_LEN16(howmany(nflits, 2)); 2901 if (eq->avail == ndesc) 2902 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ; 2903 wr->equiq_to_len16 = cpu_to_be32(ctrl); 2904 wr->r3 = 0; 2905 2906 if (txinfo->flags & HW_LSO) { 2907 uint16_t etype; 2908 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); 2909 char *p = (void *)m->b_rptr; 2910 ctrl = V_LSO_OPCODE((u32)CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | 2911 F_LSO_LAST_SLICE; 2912 2913 etype = ntohs(((struct ether_header *)p)->ether_type); 2914 if (etype == ETHERTYPE_VLAN) { 2915 ctrl |= V_LSO_ETHHDR_LEN(1); 2916 etype = ntohs(((struct ether_vlan_header *)p)->ether_type); 2917 p += sizeof (struct ether_vlan_header); 2918 } else { 2919 p += sizeof (struct ether_header); 2920 } 2921 2922 switch (etype) { 2923 case ETHERTYPE_IP: 2924 ctrl |= V_LSO_IPHDR_LEN(IPH_HDR_LENGTH(p) / 4); 2925 p += IPH_HDR_LENGTH(p); 2926 break; 2927 case ETHERTYPE_IPV6: 2928 ctrl |= F_LSO_IPV6; 2929 ctrl |= V_LSO_IPHDR_LEN(sizeof (ip6_t) / 4); 2930 p += sizeof (ip6_t); 2931 default: 2932 break; 2933 } 2934 2935 ctrl |= V_LSO_TCPHDR_LEN(TCP_HDR_LENGTH((tcph_t *)p) / 4); 2936 2937 lso->lso_ctrl = cpu_to_be32(ctrl); 2938 lso->ipid_ofst = cpu_to_be16(0); 2939 lso->mss = cpu_to_be16(txinfo->mss); 2940 lso->seqno_offset = cpu_to_be32(0); 2941 if (is_t4(pi->adapter->params.chip)) 2942 lso->len = cpu_to_be32(txinfo->len); 2943 else 2944 lso->len = cpu_to_be32(V_LSO_T5_XFER_SIZE(txinfo->len)); 2945 2946 cpl = (void *)(lso + 1); 2947 2948 txq->tso_wrs++; 2949 } else 2950 cpl = (void *)(wr + 1); 2951 2952 /* Checksum offload */ 2953 ctrl1 = 0; 2954 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM)) 2955 ctrl1 |= F_TXPKT_IPCSUM_DIS; 2956 if (!(txinfo->flags & HCK_FULLCKSUM)) 2957 ctrl1 |= F_TXPKT_L4CSUM_DIS; 2958 if (ctrl1 == 0) 2959 txq->txcsum++; /* some hardware assistance provided */ 2960 2961 /* CPL header */ 2962 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) | 2963 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf)); 2964 cpl->pack = 0; 2965 cpl->len = cpu_to_be16(txinfo->len); 2966 cpl->ctrl1 = cpu_to_be64(ctrl1); 2967 2968 /* Software descriptor */ 2969 txsd = &txq->sdesc[eq->pidx]; 2970 txsd->m = m; 2971 txsd->txb_used = txinfo->txb_used; 2972 txsd->hdls_used = txinfo->hdls_used; 2973 /* LINTED: E_ASSIGN_NARROW_CONV */ 2974 txsd->desc_used = ndesc; 2975 2976 txq->txb_used += txinfo->txb_used / TXB_CHUNK; 2977 txq->hdl_used += txinfo->hdls_used; 2978 2979 eq->pending += ndesc; 2980 eq->avail -= ndesc; 2981 eq->pidx += ndesc; 2982 if (eq->pidx >= eq->cap) 2983 eq->pidx -= eq->cap; 2984 2985 /* SGL */ 2986 dst = (void *)(cpl + 1); 2987 if (txinfo->nsegs > 0) { 2988 txq->sgl_wrs++; 2989 copy_to_txd(eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8); 2990 2991 /* Need to zero-pad to a 16 byte boundary if not on one */ 2992 if ((uintptr_t)dst & 0xf) 2993 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 2994 *(uint64_t *)dst = 0; 2995 2996 } else { 2997 txq->imm_wrs++; 2998 #ifdef DEBUG 2999 ctrl = txinfo->len; 3000 #endif 3001 for (; m; m = m->b_cont) { 3002 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m)); 3003 #ifdef DEBUG 3004 ctrl -= MBLKL(m); 3005 #endif 3006 } 3007 ASSERT(ctrl == 0); 3008 } 3009 3010 txq->txpkt_wrs++; 3011 return (0); 3012 } 3013 3014 static inline void 3015 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq, 3016 struct txpkts *txpkts, struct txinfo *txinfo) 3017 { 3018 struct ulp_txpkt *ulpmc; 3019 struct ulptx_idata *ulpsc; 3020 struct cpl_tx_pkt_core *cpl; 3021 uintptr_t flitp, start, end; 3022 uint64_t ctrl; 3023 caddr_t dst; 3024 3025 ASSERT(txpkts->npkt > 0); 3026 3027 start = (uintptr_t)txq->eq.desc; 3028 end = (uintptr_t)txq->eq.spg; 3029 3030 /* Checksum offload */ 3031 ctrl = 0; 3032 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM)) 3033 ctrl |= F_TXPKT_IPCSUM_DIS; 3034 if (!(txinfo->flags & HCK_FULLCKSUM)) 3035 ctrl |= F_TXPKT_L4CSUM_DIS; 3036 if (ctrl == 0) 3037 txq->txcsum++; /* some hardware assistance provided */ 3038 3039 /* 3040 * The previous packet's SGL must have ended at a 16 byte boundary (this 3041 * is required by the firmware/hardware). It follows that flitp cannot 3042 * wrap around between the ULPTX master command and ULPTX subcommand (8 3043 * bytes each), and that it can not wrap around in the middle of the 3044 * cpl_tx_pkt_core either. 3045 */ 3046 flitp = (uintptr_t)txpkts->flitp; 3047 ASSERT((flitp & 0xf) == 0); 3048 3049 /* ULP master command */ 3050 ulpmc = (void *)flitp; 3051 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); 3052 ulpmc->len = htonl(howmany(sizeof (*ulpmc) + sizeof (*ulpsc) + 3053 sizeof (*cpl) + 8 * txinfo->nflits, 16)); 3054 3055 /* ULP subcommand */ 3056 ulpsc = (void *)(ulpmc + 1); 3057 ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) | 3058 F_ULP_TX_SC_MORE); 3059 ulpsc->len = cpu_to_be32(sizeof (struct cpl_tx_pkt_core)); 3060 3061 flitp += sizeof (*ulpmc) + sizeof (*ulpsc); 3062 if (flitp == end) 3063 flitp = start; 3064 3065 /* CPL_TX_PKT */ 3066 cpl = (void *)flitp; 3067 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) | 3068 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf)); 3069 cpl->pack = 0; 3070 cpl->len = cpu_to_be16(txinfo->len); 3071 cpl->ctrl1 = cpu_to_be64(ctrl); 3072 3073 flitp += sizeof (*cpl); 3074 if (flitp == end) 3075 flitp = start; 3076 3077 /* SGL for this frame */ 3078 dst = (caddr_t)flitp; 3079 copy_to_txd(&txq->eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8); 3080 flitp = (uintptr_t)dst; 3081 3082 /* Zero pad and advance to a 16 byte boundary if not already at one. */ 3083 if (flitp & 0xf) { 3084 3085 /* no matter what, flitp should be on an 8 byte boundary */ 3086 ASSERT((flitp & 0x7) == 0); 3087 3088 *(uint64_t *)flitp = 0; 3089 flitp += sizeof (uint64_t); 3090 txpkts->nflits++; 3091 } 3092 3093 if (flitp == end) 3094 flitp = start; 3095 3096 txpkts->flitp = (void *)flitp; 3097 } 3098 3099 static inline void 3100 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) 3101 { 3102 if ((uintptr_t)(*to) + len <= (uintptr_t)eq->spg) { 3103 bcopy(from, *to, len); 3104 (*to) += len; 3105 } else { 3106 int portion = (uintptr_t)eq->spg - (uintptr_t)(*to); 3107 3108 bcopy(from, *to, portion); 3109 from += portion; 3110 portion = len - portion; /* remaining */ 3111 bcopy(from, (void *)eq->desc, portion); 3112 (*to) = (caddr_t)eq->desc + portion; 3113 } 3114 } 3115 3116 static inline void 3117 ring_tx_db(struct adapter *sc, struct sge_eq *eq) 3118 { 3119 int val, db_mode; 3120 u_int db = eq->doorbells; 3121 3122 if (eq->pending > 1) 3123 db &= ~DOORBELL_WCWR; 3124 3125 if (eq->pending > eq->pidx) { 3126 int offset = eq->cap - (eq->pending - eq->pidx); 3127 3128 /* pidx has wrapped around since last doorbell */ 3129 3130 (void) ddi_dma_sync(eq->desc_dhdl, 3131 offset * sizeof (struct tx_desc), 0, 3132 DDI_DMA_SYNC_FORDEV); 3133 (void) ddi_dma_sync(eq->desc_dhdl, 3134 0, eq->pidx * sizeof (struct tx_desc), 3135 DDI_DMA_SYNC_FORDEV); 3136 } else if (eq->pending > 0) { 3137 (void) ddi_dma_sync(eq->desc_dhdl, 3138 (eq->pidx - eq->pending) * sizeof (struct tx_desc), 3139 eq->pending * sizeof (struct tx_desc), 3140 DDI_DMA_SYNC_FORDEV); 3141 } 3142 3143 membar_producer(); 3144 3145 if (is_t4(sc->params.chip)) 3146 val = V_PIDX(eq->pending); 3147 else 3148 val = V_PIDX_T5(eq->pending); 3149 3150 db_mode = (1 << (ffs(db) - 1)); 3151 switch (db_mode) { 3152 case DOORBELL_UDB: 3153 *eq->udb = LE_32(V_QID(eq->udb_qid) | val); 3154 break; 3155 3156 case DOORBELL_WCWR: 3157 { 3158 volatile uint64_t *dst, *src; 3159 int i; 3160 /* 3161 * Queues whose 128B doorbell segment fits in 3162 * the page do not use relative qid 3163 * (udb_qid is always 0). Only queues with 3164 * doorbell segments can do WCWR. 3165 */ 3166 ASSERT(eq->udb_qid == 0 && eq->pending == 1); 3167 3168 dst = (volatile void *)((uintptr_t)eq->udb + 3169 UDBS_WR_OFFSET - UDBS_DB_OFFSET); 3170 i = eq->pidx ? eq->pidx - 1 : eq->cap - 1; 3171 src = (void *)&eq->desc[i]; 3172 while (src != (void *)&eq->desc[i + 1]) 3173 *dst++ = *src++; 3174 membar_producer(); 3175 break; 3176 } 3177 3178 case DOORBELL_UDBWC: 3179 *eq->udb = LE_32(V_QID(eq->udb_qid) | val); 3180 membar_producer(); 3181 break; 3182 3183 case DOORBELL_KDB: 3184 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), 3185 V_QID(eq->cntxt_id) | val); 3186 break; 3187 } 3188 3189 eq->pending = 0; 3190 } 3191 3192 static int 3193 reclaim_tx_descs(struct sge_txq *txq, int howmany) 3194 { 3195 struct tx_sdesc *txsd; 3196 uint_t cidx, can_reclaim, reclaimed, txb_freed, hdls_freed; 3197 struct sge_eq *eq = &txq->eq; 3198 3199 EQ_LOCK_ASSERT_OWNED(eq); 3200 3201 cidx = eq->spg->cidx; /* stable snapshot */ 3202 cidx = be16_to_cpu(cidx); 3203 3204 if (cidx >= eq->cidx) 3205 can_reclaim = cidx - eq->cidx; 3206 else 3207 can_reclaim = cidx + eq->cap - eq->cidx; 3208 3209 if (can_reclaim == 0) 3210 return (0); 3211 3212 txb_freed = hdls_freed = reclaimed = 0; 3213 do { 3214 int ndesc; 3215 3216 txsd = &txq->sdesc[eq->cidx]; 3217 ndesc = txsd->desc_used; 3218 3219 /* Firmware doesn't return "partial" credits. */ 3220 ASSERT(can_reclaim >= ndesc); 3221 3222 /* 3223 * We always keep mblk around, even for immediate data. If mblk 3224 * is NULL, this has to be the software descriptor for a credit 3225 * flush work request. 3226 */ 3227 if (txsd->m != NULL) 3228 freemsgchain(txsd->m); 3229 #ifdef DEBUG 3230 else { 3231 ASSERT(txsd->txb_used == 0); 3232 ASSERT(txsd->hdls_used == 0); 3233 ASSERT(ndesc == 1); 3234 } 3235 #endif 3236 3237 txb_freed += txsd->txb_used; 3238 hdls_freed += txsd->hdls_used; 3239 reclaimed += ndesc; 3240 3241 eq->cidx += ndesc; 3242 if (eq->cidx >= eq->cap) 3243 eq->cidx -= eq->cap; 3244 3245 can_reclaim -= ndesc; 3246 3247 } while (can_reclaim && reclaimed < howmany); 3248 3249 eq->avail += reclaimed; 3250 ASSERT(eq->avail < eq->cap); /* avail tops out at (cap - 1) */ 3251 3252 txq->txb_avail += txb_freed; 3253 3254 txq->tx_dhdl_avail += hdls_freed; 3255 ASSERT(txq->tx_dhdl_avail <= txq->tx_dhdl_total); 3256 for (; hdls_freed; hdls_freed--) { 3257 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_cidx]); 3258 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total) 3259 txq->tx_dhdl_cidx = 0; 3260 } 3261 3262 return (reclaimed); 3263 } 3264 3265 static void 3266 write_txqflush_wr(struct sge_txq *txq) 3267 { 3268 struct sge_eq *eq = &txq->eq; 3269 struct fw_eq_flush_wr *wr; 3270 struct tx_sdesc *txsd; 3271 3272 EQ_LOCK_ASSERT_OWNED(eq); 3273 ASSERT(eq->avail > 0); 3274 3275 wr = (void *)&eq->desc[eq->pidx]; 3276 bzero(wr, sizeof (*wr)); 3277 wr->opcode = FW_EQ_FLUSH_WR; 3278 wr->equiq_to_len16 = cpu_to_be32(V_FW_WR_LEN16(sizeof (*wr) / 16) | 3279 F_FW_WR_EQUEQ | F_FW_WR_EQUIQ); 3280 3281 txsd = &txq->sdesc[eq->pidx]; 3282 txsd->m = NULL; 3283 txsd->txb_used = 0; 3284 txsd->hdls_used = 0; 3285 txsd->desc_used = 1; 3286 3287 eq->pending++; 3288 eq->avail--; 3289 if (++eq->pidx == eq->cap) 3290 eq->pidx = 0; 3291 } 3292 3293 static int 3294 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m) 3295 { 3296 bool csum_ok; 3297 uint16_t err_vec; 3298 struct sge_rxq *rxq = (void *)iq; 3299 struct mblk_pair chain = {0}; 3300 struct adapter *sc = iq->adapter; 3301 const struct cpl_rx_pkt *cpl = (const void *)(rss + 1); 3302 3303 iq->intr_next = iq->intr_params; 3304 3305 m->b_rptr += sc->sge.pktshift; 3306 3307 /* Compressed error vector is enabled for T6 only */ 3308 if (sc->params.tp.rx_pkt_encap) 3309 /* It is enabled only in T6 config file */ 3310 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec)); 3311 else 3312 err_vec = ntohs(cpl->err_vec); 3313 3314 csum_ok = cpl->csum_calc && !err_vec; 3315 /* TODO: what about cpl->ip_frag? */ 3316 if (csum_ok && !cpl->ip_frag) { 3317 mac_hcksum_set(m, 0, 0, 0, 0xffff, 3318 HCK_FULLCKSUM_OK | HCK_FULLCKSUM | 3319 HCK_IPV4_HDRCKSUM_OK); 3320 rxq->rxcsum++; 3321 } 3322 3323 /* Add to the chain that we'll send up */ 3324 if (chain.head != NULL) 3325 chain.tail->b_next = m; 3326 else 3327 chain.head = m; 3328 chain.tail = m; 3329 3330 t4_mac_rx(rxq->port, rxq, chain.head); 3331 3332 rxq->rxpkts++; 3333 rxq->rxbytes += be16_to_cpu(cpl->len); 3334 return (0); 3335 } 3336 3337 #define FL_HW_IDX(idx) ((idx) >> 3) 3338 3339 static inline void 3340 ring_fl_db(struct adapter *sc, struct sge_fl *fl) 3341 { 3342 int desc_start, desc_last, ndesc; 3343 uint32_t v = sc->params.arch.sge_fl_db ; 3344 3345 ndesc = FL_HW_IDX(fl->pending); 3346 3347 /* Hold back one credit if pidx = cidx */ 3348 if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx)) 3349 ndesc--; 3350 3351 /* 3352 * There are chances of ndesc modified above (to avoid pidx = cidx). 3353 * If there is nothing to post, return. 3354 */ 3355 if (ndesc <= 0) 3356 return; 3357 3358 desc_last = FL_HW_IDX(fl->pidx); 3359 3360 if (fl->pidx < fl->pending) { 3361 /* There was a wrap */ 3362 desc_start = FL_HW_IDX(fl->pidx + fl->cap - fl->pending); 3363 3364 /* From desc_start to the end of list */ 3365 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 0, 3366 DDI_DMA_SYNC_FORDEV); 3367 3368 /* From start of list to the desc_last */ 3369 if (desc_last != 0) 3370 (void) ddi_dma_sync(fl->dhdl, 0, desc_last * 3371 RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV); 3372 } else { 3373 /* There was no wrap, sync from start_desc to last_desc */ 3374 desc_start = FL_HW_IDX(fl->pidx - fl->pending); 3375 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 3376 ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV); 3377 } 3378 3379 if (is_t4(sc->params.chip)) 3380 v |= V_PIDX(ndesc); 3381 else 3382 v |= V_PIDX_T5(ndesc); 3383 v |= V_QID(fl->cntxt_id) | V_PIDX(ndesc); 3384 3385 membar_producer(); 3386 3387 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v); 3388 3389 /* 3390 * Update pending count: 3391 * Deduct the number of descriptors posted 3392 */ 3393 fl->pending -= ndesc * 8; 3394 } 3395 3396 static void 3397 tx_reclaim_task(void *arg) 3398 { 3399 struct sge_txq *txq = arg; 3400 3401 TXQ_LOCK(txq); 3402 reclaim_tx_descs(txq, txq->eq.qsize); 3403 TXQ_UNLOCK(txq); 3404 } 3405 3406 /* ARGSUSED */ 3407 static int 3408 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss, 3409 mblk_t *m) 3410 { 3411 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1); 3412 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid)); 3413 struct adapter *sc = iq->adapter; 3414 struct sge *s = &sc->sge; 3415 struct sge_eq *eq; 3416 struct sge_txq *txq; 3417 3418 txq = (void *)s->eqmap[qid - s->eq_start]; 3419 eq = &txq->eq; 3420 txq->qflush++; 3421 t4_mac_tx_update(txq->port, txq); 3422 3423 ddi_taskq_dispatch(sc->tq[eq->tx_chan], tx_reclaim_task, 3424 (void *)txq, DDI_NOSLEEP); 3425 3426 return (0); 3427 } 3428 3429 static int 3430 handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m) 3431 { 3432 struct adapter *sc = iq->adapter; 3433 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1); 3434 3435 ASSERT(m == NULL); 3436 3437 if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) { 3438 const struct rss_header *rss2; 3439 3440 rss2 = (const struct rss_header *)&cpl->data[0]; 3441 return (sc->cpl_handler[rss2->opcode](iq, rss2, m)); 3442 } 3443 return (sc->fw_msg_handler[cpl->type](sc, &cpl->data[0])); 3444 } 3445 3446 int 3447 t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count, 3448 int flags) 3449 { 3450 int i, rc; 3451 3452 txmaps->map_total = count; 3453 txmaps->map_avail = txmaps->map_cidx = txmaps->map_pidx = 0; 3454 3455 txmaps->map = kmem_zalloc(sizeof (ddi_dma_handle_t) * 3456 txmaps->map_total, flags); 3457 3458 for (i = 0; i < count; i++) { 3459 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx, 3460 DDI_DMA_SLEEP, 0, &txmaps->map[i]); 3461 if (rc != DDI_SUCCESS) { 3462 cxgb_printf(sc->dip, CE_WARN, 3463 "%s: failed to allocate DMA handle (%d)", 3464 __func__, rc); 3465 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 3466 } 3467 txmaps->map_avail++; 3468 } 3469 3470 return (0); 3471 } 3472 3473 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG) 3474 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR) 3475 #define KS_U_SET(x, y) kstatp->x.value.ul = (y) 3476 #define KS_U_FROM(x, y) kstatp->x.value.ul = (y)->x 3477 #define KS_C_SET(x, ...) \ 3478 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__) 3479 3480 /* 3481 * cxgbe:X:config 3482 */ 3483 struct cxgbe_port_config_kstats { 3484 kstat_named_t idx; 3485 kstat_named_t nrxq; 3486 kstat_named_t ntxq; 3487 kstat_named_t first_rxq; 3488 kstat_named_t first_txq; 3489 kstat_named_t controller; 3490 kstat_named_t factory_mac_address; 3491 }; 3492 3493 /* 3494 * cxgbe:X:info 3495 */ 3496 struct cxgbe_port_info_kstats { 3497 kstat_named_t transceiver; 3498 kstat_named_t rx_ovflow0; 3499 kstat_named_t rx_ovflow1; 3500 kstat_named_t rx_ovflow2; 3501 kstat_named_t rx_ovflow3; 3502 kstat_named_t rx_trunc0; 3503 kstat_named_t rx_trunc1; 3504 kstat_named_t rx_trunc2; 3505 kstat_named_t rx_trunc3; 3506 kstat_named_t tx_pause; 3507 kstat_named_t rx_pause; 3508 }; 3509 3510 static kstat_t * 3511 setup_port_config_kstats(struct port_info *pi) 3512 { 3513 kstat_t *ksp; 3514 struct cxgbe_port_config_kstats *kstatp; 3515 int ndata; 3516 dev_info_t *pdip = ddi_get_parent(pi->dip); 3517 uint8_t *ma = &pi->hw_addr[0]; 3518 3519 ndata = sizeof (struct cxgbe_port_config_kstats) / 3520 sizeof (kstat_named_t); 3521 3522 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "config", 3523 "net", KSTAT_TYPE_NAMED, ndata, 0); 3524 if (ksp == NULL) { 3525 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats."); 3526 return (NULL); 3527 } 3528 3529 kstatp = (struct cxgbe_port_config_kstats *)ksp->ks_data; 3530 3531 KS_UINIT(idx); 3532 KS_UINIT(nrxq); 3533 KS_UINIT(ntxq); 3534 KS_UINIT(first_rxq); 3535 KS_UINIT(first_txq); 3536 KS_CINIT(controller); 3537 KS_CINIT(factory_mac_address); 3538 3539 KS_U_SET(idx, pi->port_id); 3540 KS_U_SET(nrxq, pi->nrxq); 3541 KS_U_SET(ntxq, pi->ntxq); 3542 KS_U_SET(first_rxq, pi->first_rxq); 3543 KS_U_SET(first_txq, pi->first_txq); 3544 KS_C_SET(controller, "%s%d", ddi_driver_name(pdip), 3545 ddi_get_instance(pdip)); 3546 KS_C_SET(factory_mac_address, "%02X%02X%02X%02X%02X%02X", 3547 ma[0], ma[1], ma[2], ma[3], ma[4], ma[5]); 3548 3549 /* Do NOT set ksp->ks_update. These kstats do not change. */ 3550 3551 /* Install the kstat */ 3552 ksp->ks_private = (void *)pi; 3553 kstat_install(ksp); 3554 3555 return (ksp); 3556 } 3557 3558 static kstat_t * 3559 setup_port_info_kstats(struct port_info *pi) 3560 { 3561 kstat_t *ksp; 3562 struct cxgbe_port_info_kstats *kstatp; 3563 int ndata; 3564 3565 ndata = sizeof (struct cxgbe_port_info_kstats) / sizeof (kstat_named_t); 3566 3567 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "info", 3568 "net", KSTAT_TYPE_NAMED, ndata, 0); 3569 if (ksp == NULL) { 3570 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats."); 3571 return (NULL); 3572 } 3573 3574 kstatp = (struct cxgbe_port_info_kstats *)ksp->ks_data; 3575 3576 KS_CINIT(transceiver); 3577 KS_UINIT(rx_ovflow0); 3578 KS_UINIT(rx_ovflow1); 3579 KS_UINIT(rx_ovflow2); 3580 KS_UINIT(rx_ovflow3); 3581 KS_UINIT(rx_trunc0); 3582 KS_UINIT(rx_trunc1); 3583 KS_UINIT(rx_trunc2); 3584 KS_UINIT(rx_trunc3); 3585 KS_UINIT(tx_pause); 3586 KS_UINIT(rx_pause); 3587 3588 /* Install the kstat */ 3589 ksp->ks_update = update_port_info_kstats; 3590 ksp->ks_private = (void *)pi; 3591 kstat_install(ksp); 3592 3593 return (ksp); 3594 } 3595 3596 static int 3597 update_port_info_kstats(kstat_t *ksp, int rw) 3598 { 3599 struct cxgbe_port_info_kstats *kstatp = 3600 (struct cxgbe_port_info_kstats *)ksp->ks_data; 3601 struct port_info *pi = ksp->ks_private; 3602 static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX", 3603 "active TWINAX", "LRM" }; 3604 uint32_t bgmap; 3605 3606 if (rw == KSTAT_WRITE) 3607 return (0); 3608 3609 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 3610 KS_C_SET(transceiver, "unplugged"); 3611 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 3612 KS_C_SET(transceiver, "unknown"); 3613 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 3614 KS_C_SET(transceiver, "unsupported"); 3615 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str)) 3616 KS_C_SET(transceiver, "%s", mod_str[pi->mod_type]); 3617 else 3618 KS_C_SET(transceiver, "type %d", pi->mod_type); 3619 3620 #define GET_STAT(name) t4_read_reg64(pi->adapter, \ 3621 PORT_REG(pi->port_id, A_MPS_PORT_STAT_##name##_L)) 3622 #define GET_STAT_COM(name) t4_read_reg64(pi->adapter, \ 3623 A_MPS_STAT_##name##_L) 3624 3625 bgmap = G_NUMPORTS(t4_read_reg(pi->adapter, A_MPS_CMN_CTL)); 3626 if (bgmap == 0) 3627 bgmap = (pi->port_id == 0) ? 0xf : 0; 3628 else if (bgmap == 1) 3629 bgmap = (pi->port_id < 2) ? (3 << (2 * pi->port_id)) : 0; 3630 else 3631 bgmap = 1; 3632 3633 KS_U_SET(rx_ovflow0, (bgmap & 1) ? 3634 GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0); 3635 KS_U_SET(rx_ovflow1, (bgmap & 2) ? 3636 GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0); 3637 KS_U_SET(rx_ovflow2, (bgmap & 4) ? 3638 GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0); 3639 KS_U_SET(rx_ovflow3, (bgmap & 8) ? 3640 GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0); 3641 KS_U_SET(rx_trunc0, (bgmap & 1) ? 3642 GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0); 3643 KS_U_SET(rx_trunc1, (bgmap & 2) ? 3644 GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0); 3645 KS_U_SET(rx_trunc2, (bgmap & 4) ? 3646 GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0); 3647 KS_U_SET(rx_trunc3, (bgmap & 8) ? 3648 GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0); 3649 3650 KS_U_SET(tx_pause, GET_STAT(TX_PORT_PAUSE)); 3651 KS_U_SET(rx_pause, GET_STAT(RX_PORT_PAUSE)); 3652 3653 return (0); 3654 3655 } 3656 3657 /* 3658 * cxgbe:X:rxqY 3659 */ 3660 struct rxq_kstats { 3661 kstat_named_t rxcsum; 3662 kstat_named_t rxpkts; 3663 kstat_named_t rxbytes; 3664 kstat_named_t nomem; 3665 }; 3666 3667 static kstat_t * 3668 setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, int idx) 3669 { 3670 struct kstat *ksp; 3671 struct rxq_kstats *kstatp; 3672 int ndata; 3673 char str[16]; 3674 3675 ndata = sizeof (struct rxq_kstats) / sizeof (kstat_named_t); 3676 (void) snprintf(str, sizeof (str), "rxq%u", idx); 3677 3678 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "rxq", 3679 KSTAT_TYPE_NAMED, ndata, 0); 3680 if (ksp == NULL) { 3681 cxgb_printf(pi->dip, CE_WARN, 3682 "%s: failed to initialize rxq kstats for queue %d.", 3683 __func__, idx); 3684 return (NULL); 3685 } 3686 3687 kstatp = (struct rxq_kstats *)ksp->ks_data; 3688 3689 KS_UINIT(rxcsum); 3690 KS_UINIT(rxpkts); 3691 KS_UINIT(rxbytes); 3692 KS_UINIT(nomem); 3693 3694 ksp->ks_update = update_rxq_kstats; 3695 ksp->ks_private = (void *)rxq; 3696 kstat_install(ksp); 3697 3698 return (ksp); 3699 } 3700 3701 static int 3702 update_rxq_kstats(kstat_t *ksp, int rw) 3703 { 3704 struct rxq_kstats *kstatp = (struct rxq_kstats *)ksp->ks_data; 3705 struct sge_rxq *rxq = ksp->ks_private; 3706 3707 if (rw == KSTAT_WRITE) 3708 return (0); 3709 3710 KS_U_FROM(rxcsum, rxq); 3711 KS_U_FROM(rxpkts, rxq); 3712 KS_U_FROM(rxbytes, rxq); 3713 KS_U_FROM(nomem, rxq); 3714 3715 return (0); 3716 } 3717 3718 /* 3719 * cxgbe:X:txqY 3720 */ 3721 struct txq_kstats { 3722 kstat_named_t txcsum; 3723 kstat_named_t tso_wrs; 3724 kstat_named_t imm_wrs; 3725 kstat_named_t sgl_wrs; 3726 kstat_named_t txpkt_wrs; 3727 kstat_named_t txpkts_wrs; 3728 kstat_named_t txpkts_pkts; 3729 kstat_named_t txb_used; 3730 kstat_named_t hdl_used; 3731 kstat_named_t txb_full; 3732 kstat_named_t dma_hdl_failed; 3733 kstat_named_t dma_map_failed; 3734 kstat_named_t qfull; 3735 kstat_named_t qflush; 3736 kstat_named_t pullup_early; 3737 kstat_named_t pullup_late; 3738 kstat_named_t pullup_failed; 3739 }; 3740 3741 static kstat_t * 3742 setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, int idx) 3743 { 3744 struct kstat *ksp; 3745 struct txq_kstats *kstatp; 3746 int ndata; 3747 char str[16]; 3748 3749 ndata = sizeof (struct txq_kstats) / sizeof (kstat_named_t); 3750 (void) snprintf(str, sizeof (str), "txq%u", idx); 3751 3752 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "txq", 3753 KSTAT_TYPE_NAMED, ndata, 0); 3754 if (ksp == NULL) { 3755 cxgb_printf(pi->dip, CE_WARN, 3756 "%s: failed to initialize txq kstats for queue %d.", 3757 __func__, idx); 3758 return (NULL); 3759 } 3760 3761 kstatp = (struct txq_kstats *)ksp->ks_data; 3762 3763 KS_UINIT(txcsum); 3764 KS_UINIT(tso_wrs); 3765 KS_UINIT(imm_wrs); 3766 KS_UINIT(sgl_wrs); 3767 KS_UINIT(txpkt_wrs); 3768 KS_UINIT(txpkts_wrs); 3769 KS_UINIT(txpkts_pkts); 3770 KS_UINIT(txb_used); 3771 KS_UINIT(hdl_used); 3772 KS_UINIT(txb_full); 3773 KS_UINIT(dma_hdl_failed); 3774 KS_UINIT(dma_map_failed); 3775 KS_UINIT(qfull); 3776 KS_UINIT(qflush); 3777 KS_UINIT(pullup_early); 3778 KS_UINIT(pullup_late); 3779 KS_UINIT(pullup_failed); 3780 3781 ksp->ks_update = update_txq_kstats; 3782 ksp->ks_private = (void *)txq; 3783 kstat_install(ksp); 3784 3785 return (ksp); 3786 } 3787 3788 static int 3789 update_txq_kstats(kstat_t *ksp, int rw) 3790 { 3791 struct txq_kstats *kstatp = (struct txq_kstats *)ksp->ks_data; 3792 struct sge_txq *txq = ksp->ks_private; 3793 3794 if (rw == KSTAT_WRITE) 3795 return (0); 3796 3797 KS_U_FROM(txcsum, txq); 3798 KS_U_FROM(tso_wrs, txq); 3799 KS_U_FROM(imm_wrs, txq); 3800 KS_U_FROM(sgl_wrs, txq); 3801 KS_U_FROM(txpkt_wrs, txq); 3802 KS_U_FROM(txpkts_wrs, txq); 3803 KS_U_FROM(txpkts_pkts, txq); 3804 KS_U_FROM(txb_used, txq); 3805 KS_U_FROM(hdl_used, txq); 3806 KS_U_FROM(txb_full, txq); 3807 KS_U_FROM(dma_hdl_failed, txq); 3808 KS_U_FROM(dma_map_failed, txq); 3809 KS_U_FROM(qfull, txq); 3810 KS_U_FROM(qflush, txq); 3811 KS_U_FROM(pullup_early, txq); 3812 KS_U_FROM(pullup_late, txq); 3813 KS_U_FROM(pullup_failed, txq); 3814 3815 return (0); 3816 } 3817