1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * This file is part of the Chelsio T4 support code. 14 * 15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved. 16 * 17 * This program is distributed in the hope that it will be useful, but WITHOUT 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this 20 * release for licensing terms and conditions. 21 */ 22 23 #include <sys/ddi.h> 24 #include <sys/sunddi.h> 25 #include <sys/sunndi.h> 26 #include <sys/atomic.h> 27 #include <sys/dlpi.h> 28 #include <sys/pattr.h> 29 #include <sys/strsubr.h> 30 #include <sys/stream.h> 31 #include <sys/strsun.h> 32 #include <inet/ip.h> 33 #include <inet/tcp.h> 34 35 #include "version.h" 36 #include "common/common.h" 37 #include "common/t4_msg.h" 38 #include "common/t4_regs.h" 39 #include "common/t4_regs_values.h" 40 41 /* TODO: Tune. */ 42 int rx_buf_size = 8192; 43 int tx_copy_threshold = 256; 44 uint16_t rx_copy_threshold = 256; 45 46 enum { 47 SPG_SIZE = 64, /* Size of status page */ 48 FL_ALIGN = CACHE_LINE, /* packet buffer alignment in FL buffers */ 49 FL_PKTSHIFT = 2 /* payload is at this offset in packet buffer */ 50 }; 51 52 /* Used to track coalesced tx work request */ 53 struct txpkts { 54 mblk_t *tail; /* head is in the software descriptor */ 55 uint64_t *flitp; /* ptr to flit where next pkt should start */ 56 uint8_t npkt; /* # of packets in this work request */ 57 uint8_t nflits; /* # of flits used by this work request */ 58 uint16_t plen; /* total payload (sum of all packets) */ 59 }; 60 61 /* All information needed to tx a frame */ 62 struct txinfo { 63 uint32_t len; /* Total length of frame */ 64 uint32_t flags; /* Checksum and LSO flags */ 65 uint32_t mss; /* MSS for LSO */ 66 uint8_t nsegs; /* # of segments in the SGL, 0 means imm. tx */ 67 uint8_t nflits; /* # of flits needed for the SGL */ 68 uint8_t hdls_used; /* # of DMA handles used */ 69 uint32_t txb_used; /* txb_space used */ 70 struct ulptx_sgl sgl __attribute__((aligned(8))); 71 struct ulptx_sge_pair reserved[TX_SGL_SEGS / 2]; 72 }; 73 74 static int service_iq(struct sge_iq *iq, int budget); 75 static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, 76 int8_t pktc_idx, int qsize, uint8_t esize); 77 static inline void init_fl(struct sge_fl *fl, uint16_t qsize); 78 static inline void init_eq(struct adapter *sc, struct sge_eq *eq, 79 uint16_t eqtype, uint16_t qsize,uint8_t tx_chan, uint16_t iqid); 80 static int alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, 81 struct sge_fl *fl, int intr_idx, int cong); 82 static int free_iq_fl(struct port_info *pi, struct sge_iq *iq, 83 struct sge_fl *fl); 84 static int alloc_fwq(struct adapter *sc); 85 static int free_fwq(struct adapter *sc); 86 static int alloc_mgmtq(struct adapter *sc); 87 static int alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, 88 int i); 89 static int free_rxq(struct port_info *pi, struct sge_rxq *rxq); 90 #ifndef TCP_OFFLOAD_DISABLE 91 static int alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq, 92 int intr_idx); 93 static int free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq); 94 #endif 95 static int ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq); 96 static int eth_eq_alloc(struct adapter *sc, struct port_info *pi, 97 struct sge_eq *eq); 98 #ifndef TCP_OFFLOAD_DISABLE 99 static int ofld_eq_alloc(struct adapter *sc, struct port_info *pi, 100 struct sge_eq *eq); 101 #endif 102 static int alloc_eq(struct adapter *sc, struct port_info *pi, 103 struct sge_eq *eq); 104 static int free_eq(struct adapter *sc, struct sge_eq *eq); 105 static int alloc_wrq(struct adapter *sc, struct port_info *pi, 106 struct sge_wrq *wrq, int idx); 107 static int free_wrq(struct adapter *sc, struct sge_wrq *wrq); 108 static int alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx); 109 static int free_txq(struct port_info *pi, struct sge_txq *txq); 110 static int alloc_dma_memory(struct adapter *sc, size_t len, int flags, 111 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr, 112 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 113 caddr_t *pva); 114 static int free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl); 115 static int alloc_desc_ring(struct adapter *sc, size_t len, int rw, 116 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 117 caddr_t *pva); 118 static int free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl); 119 static int alloc_tx_copybuffer(struct adapter *sc, size_t len, 120 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 121 caddr_t *pva); 122 static inline bool is_new_response(const struct sge_iq *iq, 123 struct rsp_ctrl **ctrl); 124 static inline void iq_next(struct sge_iq *iq); 125 static int refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs); 126 static void refill_sfl(void *arg); 127 static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl); 128 static void free_fl_bufs(struct sge_fl *fl); 129 static mblk_t *get_fl_payload(struct sge_fl *fl, uint32_t len_newbuf, 130 int *fl_bufs_used); 131 static int get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, 132 struct txinfo *txinfo, int sgl_only); 133 static inline int fits_in_txb(struct sge_txq *txq, int len, int *waste); 134 static inline int copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, 135 struct txinfo *txinfo); 136 static inline void add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len); 137 static inline int add_mblk(struct sge_txq *txq, struct txinfo *txinfo, 138 mblk_t *m, int len); 139 static void free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo); 140 static int add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m, 141 struct txinfo *txinfo); 142 static void write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts); 143 static int write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m, 144 struct txinfo *txinfo); 145 static inline void write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq, 146 struct txpkts *txpkts, struct txinfo *txinfo); 147 static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, 148 int len); 149 static inline void ring_tx_db(struct adapter *sc, struct sge_eq *eq); 150 static int reclaim_tx_descs(struct sge_txq *txq, int howmany); 151 static void write_txqflush_wr(struct sge_txq *txq); 152 static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, 153 mblk_t *m); 154 static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl); 155 static kstat_t *setup_port_config_kstats(struct port_info *pi); 156 static kstat_t *setup_port_info_kstats(struct port_info *pi); 157 static kstat_t *setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, 158 int idx); 159 static int update_rxq_kstats(kstat_t *ksp, int rw); 160 static int update_port_info_kstats(kstat_t *ksp, int rw); 161 static kstat_t *setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, 162 int idx); 163 static int update_txq_kstats(kstat_t *ksp, int rw); 164 static int handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, 165 mblk_t *m); 166 167 static inline int 168 reclaimable(struct sge_eq *eq) 169 { 170 unsigned int cidx; 171 172 cidx = eq->spg->cidx; /* stable snapshot */ 173 cidx = be16_to_cpu(cidx); 174 175 if (cidx >= eq->cidx) 176 return (cidx - eq->cidx); 177 else 178 return (cidx + eq->cap - eq->cidx); 179 } 180 181 void 182 t4_sge_init(struct adapter *sc) 183 { 184 struct driver_properties *p = &sc->props; 185 ddi_dma_attr_t *dma_attr; 186 ddi_device_acc_attr_t *acc_attr; 187 uint32_t v; 188 189 /* 190 * Device access and DMA attributes for descriptor rings 191 */ 192 acc_attr = &sc->sge.acc_attr_desc; 193 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 194 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 195 acc_attr->devacc_attr_dataorder = DDI_STRICTORDER_ACC; 196 197 dma_attr = &sc->sge.dma_attr_desc; 198 dma_attr->dma_attr_version = DMA_ATTR_V0; 199 dma_attr->dma_attr_addr_lo = 0; 200 dma_attr->dma_attr_addr_hi = UINT64_MAX; 201 dma_attr->dma_attr_count_max = UINT64_MAX; 202 dma_attr->dma_attr_align = 512; 203 dma_attr->dma_attr_burstsizes = 0xfff; 204 dma_attr->dma_attr_minxfer = 1; 205 dma_attr->dma_attr_maxxfer = UINT64_MAX; 206 dma_attr->dma_attr_seg = UINT64_MAX; 207 dma_attr->dma_attr_sgllen = 1; 208 dma_attr->dma_attr_granular = 1; 209 dma_attr->dma_attr_flags = 0; 210 211 /* 212 * Device access and DMA attributes for tx buffers 213 */ 214 acc_attr = &sc->sge.acc_attr_tx; 215 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 216 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 217 218 dma_attr = &sc->sge.dma_attr_tx; 219 dma_attr->dma_attr_version = DMA_ATTR_V0; 220 dma_attr->dma_attr_addr_lo = 0; 221 dma_attr->dma_attr_addr_hi = UINT64_MAX; 222 dma_attr->dma_attr_count_max = UINT64_MAX; 223 dma_attr->dma_attr_align = 1; 224 dma_attr->dma_attr_burstsizes = 0xfff; 225 dma_attr->dma_attr_minxfer = 1; 226 dma_attr->dma_attr_maxxfer = UINT64_MAX; 227 dma_attr->dma_attr_seg = UINT64_MAX; 228 dma_attr->dma_attr_sgllen = TX_SGL_SEGS; 229 dma_attr->dma_attr_granular = 1; 230 dma_attr->dma_attr_flags = 0; 231 232 /* 233 * Device access and DMA attributes for rx buffers 234 */ 235 sc->sge.rxb_params.dip = sc->dip; 236 sc->sge.rxb_params.buf_size = rx_buf_size; 237 238 acc_attr = &sc->sge.rxb_params.acc_attr_rx; 239 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 240 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 241 242 dma_attr = &sc->sge.rxb_params.dma_attr_rx; 243 dma_attr->dma_attr_version = DMA_ATTR_V0; 244 dma_attr->dma_attr_addr_lo = 0; 245 dma_attr->dma_attr_addr_hi = UINT64_MAX; 246 dma_attr->dma_attr_count_max = UINT64_MAX; 247 /* 248 * Low 4 bits of an rx buffer address have a special meaning to the SGE 249 * and an rx buf cannot have an address with any of these bits set. 250 * FL_ALIGN is >= 32 so we're sure things are ok. 251 */ 252 dma_attr->dma_attr_align = FL_ALIGN; 253 dma_attr->dma_attr_burstsizes = 0xfff; 254 dma_attr->dma_attr_minxfer = 1; 255 dma_attr->dma_attr_maxxfer = UINT64_MAX; 256 dma_attr->dma_attr_seg = UINT64_MAX; 257 dma_attr->dma_attr_sgllen = 1; 258 dma_attr->dma_attr_granular = 1; 259 dma_attr->dma_attr_flags = 0; 260 261 sc->sge.rxbuf_cache = rxbuf_cache_create(&sc->sge.rxb_params); 262 263 v = t4_read_reg(sc, A_SGE_CONM_CTRL); 264 sc->sge.fl_starve_threshold = G_EGRTHRESHOLD(v) * 2 + 1; 265 266 t4_set_reg_field(sc, A_SGE_CONTROL, V_PKTSHIFT(M_PKTSHIFT) | 267 V_INGPADBOUNDARY(M_INGPADBOUNDARY) | 268 F_EGRSTATUSPAGESIZE, V_INGPADBOUNDARY(ilog2(FL_ALIGN) - 5) | 269 V_PKTSHIFT(FL_PKTSHIFT) | 270 F_RXPKTCPLMODE | 271 V_EGRSTATUSPAGESIZE(SPG_SIZE == 128)); 272 273 t4_set_reg_field(sc, A_SGE_HOST_PAGE_SIZE, 274 V_HOSTPAGESIZEPF0(M_HOSTPAGESIZEPF0), 275 V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10)); 276 277 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, rx_buf_size); 278 279 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, 280 V_THRESHOLD_0(p->counter_val[0]) | 281 V_THRESHOLD_1(p->counter_val[1]) | 282 V_THRESHOLD_2(p->counter_val[2]) | 283 V_THRESHOLD_3(p->counter_val[3])); 284 285 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, 286 V_TIMERVALUE0(us_to_core_ticks(sc, p->timer_val[0])) | 287 V_TIMERVALUE1(us_to_core_ticks(sc, p->timer_val[1]))); 288 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, 289 V_TIMERVALUE2(us_to_core_ticks(sc, p->timer_val[2])) | 290 V_TIMERVALUE3(us_to_core_ticks(sc, p->timer_val[3]))); 291 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, 292 V_TIMERVALUE4(us_to_core_ticks(sc, p->timer_val[4])) | 293 V_TIMERVALUE5(us_to_core_ticks(sc, p->timer_val[5]))); 294 295 (void) t4_register_cpl_handler(sc, CPL_FW4_MSG, handle_fw_rpl); 296 (void) t4_register_cpl_handler(sc, CPL_FW6_MSG, handle_fw_rpl); 297 (void) t4_register_cpl_handler(sc, CPL_RX_PKT, t4_eth_rx); 298 (void) t4_register_fw_msg_handler(sc, FW6_TYPE_CMD_RPL, 299 t4_handle_fw_rpl); 300 } 301 302 /* 303 * Allocate and initialize the firmware event queue and the forwarded interrupt 304 * queues, if any. The adapter owns all these queues as they are not associated 305 * with any particular port. 306 * 307 * Returns errno on failure. Resources allocated up to that point may still be 308 * allocated. Caller is responsible for cleanup in case this function fails. 309 */ 310 int 311 t4_setup_adapter_queues(struct adapter *sc) 312 { 313 int rc; 314 315 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 316 317 /* 318 * Firmware event queue 319 */ 320 rc = alloc_fwq(sc); 321 if (rc != 0) 322 return (rc); 323 324 /* 325 * Management queue. This is just a control queue that uses the fwq as 326 * its associated iq. 327 */ 328 rc = alloc_mgmtq(sc); 329 330 return (rc); 331 } 332 333 /* 334 * Idempotent 335 */ 336 int 337 t4_teardown_adapter_queues(struct adapter *sc) 338 { 339 340 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 341 342 (void) free_fwq(sc); 343 344 return (0); 345 } 346 347 static inline int 348 first_vector(struct port_info *pi) 349 { 350 struct adapter *sc = pi->adapter; 351 int rc = T4_EXTRA_INTR, i; 352 353 if (sc->intr_count == 1) 354 return (0); 355 356 for_each_port(sc, i) { 357 struct port_info *p = sc->port[i]; 358 359 if (i == pi->port_id) 360 break; 361 362 #ifndef TCP_OFFLOAD_DISABLE 363 if (!(sc->flags & INTR_FWD)) 364 rc += p->nrxq + p->nofldrxq; 365 else 366 rc += max(p->nrxq, p->nofldrxq); 367 #else 368 /* 369 * Not compiled with offload support and intr_count > 1. Only 370 * NIC queues exist and they'd better be taking direct 371 * interrupts. 372 */ 373 ASSERT(!(sc->flags & INTR_FWD)); 374 rc += p->nrxq; 375 #endif 376 } 377 return (rc); 378 } 379 380 /* 381 * Given an arbitrary "index," come up with an iq that can be used by other 382 * queues (of this port) for interrupt forwarding, SGE egress updates, etc. 383 * The iq returned is guaranteed to be something that takes direct interrupts. 384 */ 385 static struct sge_iq * 386 port_intr_iq(struct port_info *pi, int idx) 387 { 388 struct adapter *sc = pi->adapter; 389 struct sge *s = &sc->sge; 390 struct sge_iq *iq = NULL; 391 392 if (sc->intr_count == 1) 393 return (&sc->sge.fwq); 394 395 #ifndef TCP_OFFLOAD_DISABLE 396 if (!(sc->flags & INTR_FWD)) { 397 idx %= pi->nrxq + pi->nofldrxq; 398 399 if (idx >= pi->nrxq) { 400 idx -= pi->nrxq; 401 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq; 402 } else 403 iq = &s->rxq[pi->first_rxq + idx].iq; 404 405 } else { 406 idx %= max(pi->nrxq, pi->nofldrxq); 407 408 if (pi->nrxq >= pi->nofldrxq) 409 iq = &s->rxq[pi->first_rxq + idx].iq; 410 else 411 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq; 412 } 413 #else 414 /* 415 * Not compiled with offload support and intr_count > 1. Only NIC 416 * queues exist and they'd better be taking direct interrupts. 417 */ 418 ASSERT(!(sc->flags & INTR_FWD)); 419 420 idx %= pi->nrxq; 421 iq = &s->rxq[pi->first_rxq + idx].iq; 422 #endif 423 424 return (iq); 425 } 426 427 int 428 t4_setup_port_queues(struct port_info *pi) 429 { 430 int rc = 0, i, intr_idx, j, iqid; 431 struct sge_rxq *rxq; 432 struct sge_txq *txq; 433 #ifndef TCP_OFFLOAD_DISABLE 434 struct sge_wrq *ctrlq; 435 struct sge_ofld_rxq *ofld_rxq; 436 struct sge_wrq *ofld_txq; 437 #endif 438 struct adapter *sc = pi->adapter; 439 struct driver_properties *p = &sc->props; 440 441 pi->ksp_config = setup_port_config_kstats(pi); 442 pi->ksp_info = setup_port_info_kstats(pi); 443 444 /* Interrupt vector to start from (when using multiple vectors) */ 445 intr_idx = first_vector(pi); 446 447 /* 448 * First pass over all rx queues (NIC and TOE): 449 * a) initialize iq and fl 450 * b) allocate queue iff it will take direct interrupts. 451 */ 452 453 for_each_rxq(pi, i, rxq) { 454 455 init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, p->qsize_rxq, 456 RX_IQ_ESIZE); 457 458 init_fl(&rxq->fl, p->qsize_rxq / 8); /* 8 bufs in each entry */ 459 460 #ifndef TCP_OFFLOAD_DISABLE 461 if ((!(sc->flags & INTR_FWD)) || 462 (sc->intr_count > 1 && pi->nrxq >= pi->nofldrxq)) 463 #else 464 if (!(sc->flags & INTR_FWD)) 465 #endif 466 { 467 rxq->iq.flags |= IQ_INTR; 468 rc = alloc_rxq(pi, rxq, intr_idx, i); 469 if (rc != 0) 470 goto done; 471 intr_idx++; 472 } 473 474 } 475 476 #ifndef TCP_OFFLOAD_DISABLE 477 for_each_ofld_rxq(pi, i, ofld_rxq) { 478 479 init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, 480 p->qsize_rxq, RX_IQ_ESIZE); 481 482 init_fl(&ofld_rxq->fl, p->qsize_rxq / 8); 483 484 if (!(sc->flags & INTR_FWD) || 485 (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) { 486 ofld_rxq->iq.flags = IQ_INTR; 487 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx); 488 if (rc != 0) 489 goto done; 490 491 intr_idx++; 492 } 493 } 494 #endif 495 496 /* 497 * Second pass over all rx queues (NIC and TOE). The queues forwarding 498 * their interrupts are allocated now. 499 */ 500 j = 0; 501 for_each_rxq(pi, i, rxq) { 502 if (rxq->iq.flags & IQ_INTR) 503 continue; 504 505 intr_idx = port_intr_iq(pi, j)->abs_id; 506 507 rc = alloc_rxq(pi, rxq, intr_idx, i); 508 if (rc != 0) 509 goto done; 510 j++; 511 } 512 513 #ifndef TCP_OFFLOAD_DISABLE 514 for_each_ofld_rxq(pi, i, ofld_rxq) { 515 if (ofld_rxq->iq.flags & IQ_INTR) 516 continue; 517 518 intr_idx = port_intr_iq(pi, j)->abs_id; 519 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx); 520 if (rc != 0) 521 goto done; 522 j++; 523 } 524 #endif 525 /* 526 * Now the tx queues. Only one pass needed. 527 */ 528 j = 0; 529 for_each_txq(pi, i, txq) { 530 uint16_t iqid; 531 532 iqid = port_intr_iq(pi, j)->cntxt_id; 533 init_eq(sc, &txq->eq, EQ_ETH, p->qsize_txq, pi->tx_chan, iqid); 534 rc = alloc_txq(pi, txq, i); 535 if (rc != 0) 536 goto done; 537 } 538 539 #ifndef TCP_OFFLOAD_DISABLE 540 for_each_ofld_txq(pi, i, ofld_txq) { 541 uint16_t iqid; 542 543 iqid = port_intr_iq(pi, j)->cntxt_id; 544 init_eq(sc, &ofld_txq->eq, EQ_OFLD, p->qsize_txq, pi->tx_chan, 545 iqid); 546 rc = alloc_wrq(sc, pi, ofld_txq, i); 547 if (rc != 0) 548 goto done; 549 } 550 #endif 551 552 /* 553 * Finally, the control queue. 554 */ 555 ctrlq = &sc->sge.ctrlq[pi->port_id]; 556 iqid = port_intr_iq(pi, 0)->cntxt_id; 557 init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, iqid); 558 rc = alloc_wrq(sc, pi, ctrlq, 0); 559 560 done: 561 if (rc != 0) 562 (void) t4_teardown_port_queues(pi); 563 564 return (rc); 565 } 566 567 /* 568 * Idempotent 569 */ 570 int 571 t4_teardown_port_queues(struct port_info *pi) 572 { 573 int i; 574 struct sge_rxq *rxq; 575 struct sge_txq *txq; 576 struct adapter *sc = pi->adapter; 577 #ifndef TCP_OFFLOAD_DISABLE 578 struct sge_ofld_rxq *ofld_rxq; 579 struct sge_wrq *ofld_txq; 580 #endif 581 582 if (pi->ksp_config != NULL) { 583 kstat_delete(pi->ksp_config); 584 pi->ksp_config = NULL; 585 } 586 if (pi->ksp_info != NULL) { 587 kstat_delete(pi->ksp_info); 588 pi->ksp_info = NULL; 589 } 590 591 (void) free_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 592 593 for_each_txq(pi, i, txq) { 594 (void) free_txq(pi, txq); 595 } 596 597 #ifndef TCP_OFFLOAD_DISABLE 598 for_each_ofld_txq(pi, i, ofld_txq) { 599 (void) free_wrq(sc, ofld_txq); 600 } 601 602 for_each_ofld_rxq(pi, i, ofld_rxq) { 603 if ((ofld_rxq->iq.flags & IQ_INTR) == 0) 604 (void) free_ofld_rxq(pi, ofld_rxq); 605 } 606 #endif 607 608 for_each_rxq(pi, i, rxq) { 609 if ((rxq->iq.flags & IQ_INTR) == 0) 610 (void) free_rxq(pi, rxq); 611 } 612 613 /* 614 * Then take down the rx queues that take direct interrupts. 615 */ 616 617 for_each_rxq(pi, i, rxq) { 618 if (rxq->iq.flags & IQ_INTR) 619 (void) free_rxq(pi, rxq); 620 } 621 622 #ifndef TCP_OFFLOAD_DISABLE 623 for_each_ofld_rxq(pi, i, ofld_rxq) { 624 if (ofld_rxq->iq.flags & IQ_INTR) 625 (void) free_ofld_rxq(pi, ofld_rxq); 626 } 627 #endif 628 629 return (0); 630 } 631 632 /* Deals with errors and forwarded interrupts */ 633 uint_t 634 t4_intr_all(caddr_t arg1, caddr_t arg2) 635 { 636 637 (void) t4_intr_err(arg1, arg2); 638 (void) t4_intr(arg1, arg2); 639 640 return (DDI_INTR_CLAIMED); 641 } 642 643 /* Deals with interrupts on the given ingress queue */ 644 /* ARGSUSED */ 645 uint_t 646 t4_intr(caddr_t arg1, caddr_t arg2) 647 { 648 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 649 struct sge_iq *iq = (struct sge_iq *)arg2; 650 651 if (atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY) == IQS_IDLE) { 652 (void) service_iq(iq, 0); 653 (void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE); 654 } 655 return (DDI_INTR_CLAIMED); 656 } 657 658 /* Deals with error interrupts */ 659 /* ARGSUSED */ 660 uint_t 661 t4_intr_err(caddr_t arg1, caddr_t arg2) 662 { 663 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 664 struct adapter *sc = (struct adapter *)arg1; 665 666 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0); 667 (void) t4_slow_intr_handler(sc); 668 669 return (DDI_INTR_CLAIMED); 670 } 671 672 /* 673 * Deals with anything and everything on the given ingress queue. 674 */ 675 static int 676 service_iq(struct sge_iq *iq, int budget) 677 { 678 struct sge_iq *q; 679 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */ 680 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */ 681 struct adapter *sc = iq->adapter; 682 struct rsp_ctrl *ctrl; 683 const struct rss_header *rss; 684 int ndescs = 0, limit, fl_bufs_used = 0; 685 int rsp_type; 686 uint32_t lq; 687 mblk_t *m; 688 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql); 689 690 limit = budget ? budget : iq->qsize / 8; 691 692 /* 693 * We always come back and check the descriptor ring for new indirect 694 * interrupts and other responses after running a single handler. 695 */ 696 for (;;) { 697 while (is_new_response(iq, &ctrl)) { 698 699 membar_consumer(); 700 701 m = NULL; 702 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen); 703 lq = be32_to_cpu(ctrl->pldbuflen_qid); 704 rss = (const void *)iq->cdesc; 705 706 switch (rsp_type) { 707 case X_RSPD_TYPE_FLBUF: 708 709 ASSERT(iq->flags & IQ_HAS_FL); 710 711 m = get_fl_payload(fl, lq, &fl_bufs_used); 712 if (m == NULL) { 713 panic("%s: line %d.", __func__, 714 __LINE__); 715 } 716 717 /* FALLTHRU */ 718 case X_RSPD_TYPE_CPL: 719 720 ASSERT(rss->opcode < NUM_CPL_CMDS); 721 sc->cpl_handler[rss->opcode](iq, rss, m); 722 break; 723 724 case X_RSPD_TYPE_INTR: 725 726 /* 727 * Interrupts should be forwarded only to queues 728 * that are not forwarding their interrupts. 729 * This means service_iq can recurse but only 1 730 * level deep. 731 */ 732 ASSERT(budget == 0); 733 734 q = sc->sge.iqmap[lq - sc->sge.iq_start]; 735 if (atomic_cas_uint(&q->state, IQS_IDLE, 736 IQS_BUSY) == IQS_IDLE) { 737 if (service_iq(q, q->qsize / 8) == 0) { 738 (void) atomic_cas_uint( 739 &q->state, IQS_BUSY, 740 IQS_IDLE); 741 } else { 742 STAILQ_INSERT_TAIL(&iql, q, 743 link); 744 } 745 } 746 break; 747 748 default: 749 break; 750 } 751 752 iq_next(iq); 753 if (++ndescs == limit) { 754 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 755 V_CIDXINC(ndescs) | 756 V_INGRESSQID(iq->cntxt_id) | 757 V_SEINTARM(V_QINTR_TIMER_IDX( 758 X_TIMERREG_UPDATE_CIDX))); 759 ndescs = 0; 760 761 if (fl_bufs_used > 0) { 762 ASSERT(iq->flags & IQ_HAS_FL); 763 FL_LOCK(fl); 764 fl->needed += fl_bufs_used; 765 (void) refill_fl(sc, fl, fl->cap / 8); 766 FL_UNLOCK(fl); 767 fl_bufs_used = 0; 768 } 769 770 if (budget != 0) 771 return (EINPROGRESS); 772 } 773 } 774 775 if (STAILQ_EMPTY(&iql) != 0) 776 break; 777 778 /* 779 * Process the head only, and send it to the back of the list if 780 * it's still not done. 781 */ 782 q = STAILQ_FIRST(&iql); 783 STAILQ_REMOVE_HEAD(&iql, link); 784 if (service_iq(q, q->qsize / 8) == 0) 785 (void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE); 786 else 787 STAILQ_INSERT_TAIL(&iql, q, link); 788 } 789 790 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) | 791 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next)); 792 793 if (iq->flags & IQ_HAS_FL) { 794 int starved; 795 796 FL_LOCK(fl); 797 fl->needed += fl_bufs_used; 798 starved = refill_fl(sc, fl, fl->cap / 4); 799 FL_UNLOCK(fl); 800 if (starved != 0) 801 add_fl_to_sfl(sc, fl); 802 } 803 804 return (0); 805 } 806 807 int 808 t4_mgmt_tx(struct adapter *sc, mblk_t *m) 809 { 810 return (t4_wrq_tx(sc, &sc->sge.mgmtq, m)); 811 } 812 813 /* 814 * Doesn't fail. Holds on to work requests it can't send right away. 815 */ 816 int 817 t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, mblk_t *m0) 818 { 819 struct sge_eq *eq = &wrq->eq; 820 struct mblk_pair *wr_list = &wrq->wr_list; 821 int can_reclaim; 822 caddr_t dst; 823 mblk_t *wr, *next; 824 825 TXQ_LOCK_ASSERT_OWNED(wrq); 826 #ifndef TCP_OFFLOAD_DISABLE 827 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_OFLD || 828 (eq->flags & EQ_TYPEMASK) == EQ_CTRL); 829 #else 830 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_CTRL); 831 #endif 832 833 if (m0 != NULL) { 834 if (wr_list->head != NULL) 835 wr_list->tail->b_next = m0; 836 else 837 wr_list->head = m0; 838 while (m0->b_next) 839 m0 = m0->b_next; 840 wr_list->tail = m0; 841 } 842 843 can_reclaim = reclaimable(eq); 844 eq->cidx += can_reclaim; 845 eq->avail += can_reclaim; 846 if (eq->cidx >= eq->cap) 847 eq->cidx -= eq->cap; 848 849 for (wr = wr_list->head; wr; wr = next) { 850 int ndesc, len = 0; 851 mblk_t *m; 852 853 next = wr->b_next; 854 wr->b_next = NULL; 855 856 for (m = wr; m; m = m->b_cont) 857 len += MBLKL(m); 858 859 ASSERT(len > 0 && (len & 0x7) == 0); 860 ASSERT(len <= SGE_MAX_WR_LEN); 861 862 ndesc = howmany(len, EQ_ESIZE); 863 if (eq->avail < ndesc) { 864 wr->b_next = next; 865 wrq->no_desc++; 866 break; 867 } 868 869 dst = (void *)&eq->desc[eq->pidx]; 870 for (m = wr; m; m = m->b_cont) 871 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m)); 872 873 eq->pidx += ndesc; 874 eq->avail -= ndesc; 875 if (eq->pidx >= eq->cap) 876 eq->pidx -= eq->cap; 877 878 eq->pending += ndesc; 879 if (eq->pending > 16) 880 ring_tx_db(sc, eq); 881 882 wrq->tx_wrs++; 883 freemsg(wr); 884 885 if (eq->avail < 8) { 886 can_reclaim = reclaimable(eq); 887 eq->cidx += can_reclaim; 888 eq->avail += can_reclaim; 889 if (eq->cidx >= eq->cap) 890 eq->cidx -= eq->cap; 891 } 892 } 893 894 if (eq->pending != 0) 895 ring_tx_db(sc, eq); 896 897 if (wr == NULL) 898 wr_list->head = wr_list->tail = NULL; 899 else { 900 wr_list->head = wr; 901 902 ASSERT(wr_list->tail->b_next == NULL); 903 } 904 905 return (0); 906 } 907 908 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */ 909 #define TXPKTS_PKT_HDR ((\ 910 sizeof (struct ulp_txpkt) + \ 911 sizeof (struct ulptx_idata) + \ 912 sizeof (struct cpl_tx_pkt_core)) / 8) 913 914 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */ 915 #define TXPKTS_WR_HDR (\ 916 sizeof (struct fw_eth_tx_pkts_wr) / 8 + \ 917 TXPKTS_PKT_HDR) 918 919 /* Header of a tx WR, before SGL of first packet (in flits) */ 920 #define TXPKT_WR_HDR ((\ 921 sizeof (struct fw_eth_tx_pkt_wr) + \ 922 sizeof (struct cpl_tx_pkt_core)) / 8) 923 924 /* Header of a tx LSO WR, before SGL of first packet (in flits) */ 925 #define TXPKT_LSO_WR_HDR ((\ 926 sizeof (struct fw_eth_tx_pkt_wr) + \ 927 sizeof(struct cpl_tx_pkt_lso_core) + \ 928 sizeof (struct cpl_tx_pkt_core)) / 8) 929 930 mblk_t * 931 t4_eth_tx(struct port_info *pi, struct sge_txq *txq, mblk_t *frame) 932 { 933 struct adapter *sc = pi->adapter; 934 struct sge_eq *eq = &txq->eq; 935 mblk_t *next_frame; 936 int rc, coalescing; 937 struct txpkts txpkts; 938 struct txinfo txinfo; 939 940 txpkts.npkt = 0; /* indicates there's nothing in txpkts */ 941 coalescing = 0; 942 943 TXQ_LOCK(txq); 944 if (eq->avail < 8) 945 (void) reclaim_tx_descs(txq, 8); 946 for (; frame; frame = next_frame) { 947 948 if (eq->avail < 8) 949 break; 950 951 next_frame = frame->b_next; 952 frame->b_next = NULL; 953 954 if (next_frame != NULL) 955 coalescing = 1; 956 957 rc = get_frame_txinfo(txq, &frame, &txinfo, coalescing); 958 if (rc != 0) { 959 if (rc == ENOMEM) { 960 961 /* Short of resources, suspend tx */ 962 963 frame->b_next = next_frame; 964 break; 965 } 966 967 /* 968 * Unrecoverable error for this frame, throw it 969 * away and move on to the next. 970 */ 971 972 freemsg(frame); 973 continue; 974 } 975 976 if (coalescing != 0 && 977 add_to_txpkts(txq, &txpkts, frame, &txinfo) == 0) { 978 979 /* Successfully absorbed into txpkts */ 980 981 write_ulp_cpl_sgl(pi, txq, &txpkts, &txinfo); 982 goto doorbell; 983 } 984 985 /* 986 * We weren't coalescing to begin with, or current frame could 987 * not be coalesced (add_to_txpkts flushes txpkts if a frame 988 * given to it can't be coalesced). Either way there should be 989 * nothing in txpkts. 990 */ 991 ASSERT(txpkts.npkt == 0); 992 993 /* We're sending out individual frames now */ 994 coalescing = 0; 995 996 if (eq->avail < 8) 997 (void) reclaim_tx_descs(txq, 8); 998 rc = write_txpkt_wr(pi, txq, frame, &txinfo); 999 if (rc != 0) { 1000 1001 /* Short of hardware descriptors, suspend tx */ 1002 1003 /* 1004 * This is an unlikely but expensive failure. We've 1005 * done all the hard work (DMA bindings etc.) and now we 1006 * can't send out the frame. What's worse, we have to 1007 * spend even more time freeing up everything in txinfo. 1008 */ 1009 txq->qfull++; 1010 free_txinfo_resources(txq, &txinfo); 1011 1012 frame->b_next = next_frame; 1013 break; 1014 } 1015 1016 doorbell: 1017 /* Fewer and fewer doorbells as the queue fills up */ 1018 if (eq->pending >= (1 << (fls(eq->qsize - eq->avail) / 2))) 1019 ring_tx_db(sc, eq); 1020 (void) reclaim_tx_descs(txq, 32); 1021 } 1022 1023 if (txpkts.npkt > 0) 1024 write_txpkts_wr(txq, &txpkts); 1025 1026 /* 1027 * frame not NULL means there was an error but we haven't thrown it 1028 * away. This can happen when we're short of tx descriptors (qfull) or 1029 * maybe even DMA handles (dma_hdl_failed). Either way, a credit flush 1030 * and reclaim will get things going again. 1031 * 1032 * If eq->avail is already 0 we know a credit flush was requested in the 1033 * WR that reduced it to 0 so we don't need another flush (we don't have 1034 * any descriptor for a flush WR anyway, duh). 1035 */ 1036 if (frame && eq->avail > 0) 1037 write_txqflush_wr(txq); 1038 1039 if (eq->pending != 0) 1040 ring_tx_db(sc, eq); 1041 1042 (void) reclaim_tx_descs(txq, eq->qsize); 1043 TXQ_UNLOCK(txq); 1044 1045 return (frame); 1046 } 1047 1048 static inline void 1049 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int8_t pktc_idx, int 1050 qsize, uint8_t esize) 1051 { 1052 ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS); 1053 ASSERT(pktc_idx < SGE_NCOUNTERS); /* -ve is ok, means don't use */ 1054 1055 iq->flags = 0; 1056 iq->adapter = sc; 1057 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx); 1058 iq->intr_pktc_idx = SGE_NCOUNTERS - 1; 1059 if (pktc_idx >= 0) { 1060 iq->intr_params |= F_QINTR_CNT_EN; 1061 iq->intr_pktc_idx = pktc_idx; 1062 } 1063 iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */ 1064 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */ 1065 } 1066 1067 static inline void 1068 init_fl(struct sge_fl *fl, uint16_t qsize) 1069 { 1070 1071 fl->qsize = qsize; 1072 } 1073 1074 static inline void 1075 init_eq(struct adapter *sc, struct sge_eq *eq, uint16_t eqtype, uint16_t qsize, 1076 uint8_t tx_chan, uint16_t iqid) 1077 { 1078 struct sge *s = &sc->sge; 1079 uint32_t r; 1080 1081 ASSERT(tx_chan < NCHAN); 1082 ASSERT(eqtype <= EQ_TYPEMASK); 1083 1084 if (is_t5(sc->params.chip)) { 1085 r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF); 1086 r >>= S_QUEUESPERPAGEPF0 + 1087 (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf; 1088 s->s_qpp = r & M_QUEUESPERPAGEPF0; 1089 } 1090 1091 eq->flags = eqtype & EQ_TYPEMASK; 1092 eq->tx_chan = tx_chan; 1093 eq->iqid = iqid; 1094 eq->qsize = qsize; 1095 } 1096 1097 /* 1098 * Allocates the ring for an ingress queue and an optional freelist. If the 1099 * freelist is specified it will be allocated and then associated with the 1100 * ingress queue. 1101 * 1102 * Returns errno on failure. Resources allocated up to that point may still be 1103 * allocated. Caller is responsible for cleanup in case this function fails. 1104 * 1105 * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then 1106 * the intr_idx specifies the vector, starting from 0. Otherwise it specifies 1107 * the index of the queue to which its interrupts will be forwarded. 1108 */ 1109 static int 1110 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl, 1111 int intr_idx, int cong) 1112 { 1113 int rc, i, cntxt_id; 1114 size_t len; 1115 struct fw_iq_cmd c; 1116 struct adapter *sc = iq->adapter; 1117 uint32_t v = 0; 1118 1119 len = iq->qsize * iq->esize; 1120 rc = alloc_desc_ring(sc, len, DDI_DMA_READ, &iq->dhdl, &iq->ahdl, 1121 &iq->ba, (caddr_t *)&iq->desc); 1122 if (rc != 0) 1123 return (rc); 1124 1125 bzero(&c, sizeof (c)); 1126 c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | 1127 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) | 1128 V_FW_IQ_CMD_VFN(0)); 1129 1130 c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART | 1131 FW_LEN16(c)); 1132 1133 /* Special handling for firmware event queue */ 1134 if (iq == &sc->sge.fwq) 1135 v |= F_FW_IQ_CMD_IQASYNCH; 1136 1137 if (iq->flags & IQ_INTR) 1138 ASSERT(intr_idx < sc->intr_count); 1139 else 1140 v |= F_FW_IQ_CMD_IQANDST; 1141 v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx); 1142 1143 c.type_to_iqandstindex = cpu_to_be32(v | 1144 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | 1145 V_FW_IQ_CMD_VIID(pi->viid) | 1146 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT)); 1147 c.iqdroprss_to_iqesize = cpu_to_be16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | 1148 F_FW_IQ_CMD_IQGTSMODE | 1149 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) | 1150 V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4)); 1151 c.iqsize = cpu_to_be16(iq->qsize); 1152 c.iqaddr = cpu_to_be64(iq->ba); 1153 if (cong >= 0) 1154 c.iqns_to_fl0congen = BE_32(F_FW_IQ_CMD_IQFLINTCONGEN); 1155 1156 if (fl != NULL) { 1157 mutex_init(&fl->lock, NULL, MUTEX_DRIVER, 1158 DDI_INTR_PRI(sc->intr_pri)); 1159 fl->flags |= FL_MTX; 1160 1161 len = fl->qsize * RX_FL_ESIZE; 1162 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &fl->dhdl, 1163 &fl->ahdl, &fl->ba, (caddr_t *)&fl->desc); 1164 if (rc != 0) 1165 return (rc); 1166 1167 /* Allocate space for one software descriptor per buffer. */ 1168 fl->cap = (fl->qsize - SPG_SIZE / RX_FL_ESIZE) * 8; 1169 fl->sdesc = kmem_zalloc(sizeof (struct fl_sdesc) * fl->cap, 1170 KM_SLEEP); 1171 fl->needed = fl->cap; 1172 fl->lowat = roundup(sc->sge.fl_starve_threshold, 8); 1173 1174 c.iqns_to_fl0congen |= 1175 cpu_to_be32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) | 1176 F_FW_IQ_CMD_FL0PACKEN | F_FW_IQ_CMD_FL0PADEN); 1177 if (cong >= 0) { 1178 c.iqns_to_fl0congen |= 1179 BE_32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) | 1180 F_FW_IQ_CMD_FL0CONGCIF | 1181 F_FW_IQ_CMD_FL0CONGEN); 1182 } 1183 c.fl0dcaen_to_fl0cidxfthresh = 1184 cpu_to_be16(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_64B) | 1185 V_FW_IQ_CMD_FL0FBMAX(X_FETCHBURSTMAX_512B)); 1186 c.fl0size = cpu_to_be16(fl->qsize); 1187 c.fl0addr = cpu_to_be64(fl->ba); 1188 } 1189 1190 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1191 if (rc != 0) { 1192 cxgb_printf(sc->dip, CE_WARN, 1193 "failed to create ingress queue: %d", rc); 1194 return (rc); 1195 } 1196 1197 iq->cdesc = iq->desc; 1198 iq->cidx = 0; 1199 iq->gen = 1; 1200 iq->intr_next = iq->intr_params; 1201 iq->adapter = sc; 1202 iq->cntxt_id = be16_to_cpu(c.iqid); 1203 iq->abs_id = be16_to_cpu(c.physiqid); 1204 iq->flags |= IQ_ALLOCATED; 1205 1206 cntxt_id = iq->cntxt_id - sc->sge.iq_start; 1207 if (cntxt_id >= sc->sge.niq) { 1208 panic("%s: iq->cntxt_id (%d) more than the max (%d)", __func__, 1209 cntxt_id, sc->sge.niq - 1); 1210 } 1211 sc->sge.iqmap[cntxt_id] = iq; 1212 1213 if (fl != NULL) { 1214 fl->cntxt_id = be16_to_cpu(c.fl0id); 1215 fl->pidx = fl->cidx = 0; 1216 fl->copy_threshold = rx_copy_threshold; 1217 1218 cntxt_id = fl->cntxt_id - sc->sge.eq_start; 1219 if (cntxt_id >= sc->sge.neq) { 1220 panic("%s: fl->cntxt_id (%d) more than the max (%d)", 1221 __func__, cntxt_id, sc->sge.neq - 1); 1222 } 1223 sc->sge.eqmap[cntxt_id] = (void *)fl; 1224 1225 FL_LOCK(fl); 1226 (void) refill_fl(sc, fl, fl->lowat); 1227 FL_UNLOCK(fl); 1228 1229 iq->flags |= IQ_HAS_FL; 1230 } 1231 1232 if (is_t5(sc->params.chip) && cong >= 0) { 1233 uint32_t param, val; 1234 1235 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 1236 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) | 1237 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id); 1238 if (cong == 0) 1239 val = 1 << 19; 1240 else { 1241 val = 2 << 19; 1242 for (i = 0; i < 4; i++) { 1243 if (cong & (1 << i)) 1244 val |= 1 << (i << 2); 1245 } 1246 } 1247 1248 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1249 if (rc != 0) { 1250 /* report error but carry on */ 1251 cxgb_printf(sc->dip, CE_WARN, 1252 "failed to set congestion manager context for " 1253 "ingress queue %d: %d", iq->cntxt_id, rc); 1254 } 1255 } 1256 1257 /* Enable IQ interrupts */ 1258 iq->state = IQS_IDLE; 1259 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) | 1260 V_INGRESSQID(iq->cntxt_id)); 1261 1262 return (0); 1263 } 1264 1265 static int 1266 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl) 1267 { 1268 int rc; 1269 struct adapter *sc = iq->adapter; 1270 dev_info_t *dip; 1271 1272 dip = pi ? pi->dip : sc->dip; 1273 1274 if (iq != NULL) { 1275 if (iq->flags & IQ_ALLOCATED) { 1276 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, 1277 FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id, 1278 fl ? fl->cntxt_id : 0xffff, 0xffff); 1279 if (rc != 0) { 1280 cxgb_printf(dip, CE_WARN, 1281 "failed to free queue %p: %d", iq, rc); 1282 return (rc); 1283 } 1284 iq->flags &= ~IQ_ALLOCATED; 1285 } 1286 1287 if (iq->desc != NULL) { 1288 (void) free_desc_ring(&iq->dhdl, &iq->ahdl); 1289 iq->desc = NULL; 1290 } 1291 1292 bzero(iq, sizeof (*iq)); 1293 } 1294 1295 if (fl != NULL) { 1296 if (fl->sdesc != NULL) { 1297 FL_LOCK(fl); 1298 free_fl_bufs(fl); 1299 FL_UNLOCK(fl); 1300 1301 kmem_free(fl->sdesc, sizeof (struct fl_sdesc) * 1302 fl->cap); 1303 fl->sdesc = NULL; 1304 } 1305 1306 if (fl->desc != NULL) { 1307 (void) free_desc_ring(&fl->dhdl, &fl->ahdl); 1308 fl->desc = NULL; 1309 } 1310 1311 if (fl->flags & FL_MTX) { 1312 mutex_destroy(&fl->lock); 1313 fl->flags &= ~FL_MTX; 1314 } 1315 1316 bzero(fl, sizeof (struct sge_fl)); 1317 } 1318 1319 return (0); 1320 } 1321 1322 static int 1323 alloc_fwq(struct adapter *sc) 1324 { 1325 int rc, intr_idx; 1326 struct sge_iq *fwq = &sc->sge.fwq; 1327 1328 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE); 1329 fwq->flags |= IQ_INTR; /* always */ 1330 intr_idx = sc->intr_count > 1 ? 1 : 0; 1331 rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1); 1332 if (rc != 0) { 1333 cxgb_printf(sc->dip, CE_WARN, 1334 "failed to create firmware event queue: %d.", rc); 1335 return (rc); 1336 } 1337 1338 return (0); 1339 } 1340 1341 static int 1342 free_fwq(struct adapter *sc) 1343 { 1344 1345 return (free_iq_fl(NULL, &sc->sge.fwq, NULL)); 1346 } 1347 1348 static int 1349 alloc_mgmtq(struct adapter *sc) 1350 { 1351 int rc; 1352 struct sge_wrq *mgmtq = &sc->sge.mgmtq; 1353 1354 init_eq(sc, &mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan, 1355 sc->sge.fwq.cntxt_id); 1356 rc = alloc_wrq(sc, NULL, mgmtq, 0); 1357 if (rc != 0) { 1358 cxgb_printf(sc->dip, CE_WARN, 1359 "failed to create management queue: %d\n", rc); 1360 return (rc); 1361 } 1362 1363 return (0); 1364 } 1365 1366 static int 1367 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int i) 1368 { 1369 int rc; 1370 1371 rxq->port = pi; 1372 rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx, 1 << pi->tx_chan); 1373 if (rc != 0) 1374 return (rc); 1375 1376 rxq->ksp = setup_rxq_kstats(pi, rxq, i); 1377 1378 return (rc); 1379 } 1380 1381 static int 1382 free_rxq(struct port_info *pi, struct sge_rxq *rxq) 1383 { 1384 int rc; 1385 1386 if (rxq->ksp != NULL) { 1387 kstat_delete(rxq->ksp); 1388 rxq->ksp = NULL; 1389 } 1390 1391 rc = free_iq_fl(pi, &rxq->iq, &rxq->fl); 1392 if (rc == 0) 1393 bzero(&rxq->fl, sizeof (*rxq) - offsetof(struct sge_rxq, fl)); 1394 1395 return (rc); 1396 } 1397 1398 #ifndef TCP_OFFLOAD_DISABLE 1399 static int 1400 alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq, 1401 int intr_idx) 1402 { 1403 int rc; 1404 1405 rc = alloc_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx, 1406 1 << pi->tx_chan); 1407 if (rc != 0) 1408 return (rc); 1409 1410 return (rc); 1411 } 1412 1413 static int 1414 free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq) 1415 { 1416 int rc; 1417 1418 rc = free_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl); 1419 if (rc == 0) 1420 bzero(&ofld_rxq->fl, sizeof (*ofld_rxq) - 1421 offsetof(struct sge_ofld_rxq, fl)); 1422 1423 return (rc); 1424 } 1425 #endif 1426 1427 static int 1428 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) 1429 { 1430 int rc, cntxt_id; 1431 struct fw_eq_ctrl_cmd c; 1432 1433 bzero(&c, sizeof (c)); 1434 1435 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST | 1436 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) | 1437 V_FW_EQ_CTRL_CMD_VFN(0)); 1438 c.alloc_to_len16 = BE_32(F_FW_EQ_CTRL_CMD_ALLOC | 1439 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c)); 1440 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* TODO */ 1441 c.physeqid_pkd = BE_32(0); 1442 c.fetchszm_to_iqid = 1443 BE_32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1444 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) | 1445 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid)); 1446 c.dcaen_to_eqsize = 1447 BE_32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1448 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1449 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1450 V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize)); 1451 c.eqaddr = BE_64(eq->ba); 1452 1453 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1454 if (rc != 0) { 1455 cxgb_printf(sc->dip, CE_WARN, 1456 "failed to create control queue %d: %d", eq->tx_chan, rc); 1457 return (rc); 1458 } 1459 eq->flags |= EQ_ALLOCATED; 1460 1461 eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(BE_32(c.cmpliqid_eqid)); 1462 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1463 if (cntxt_id >= sc->sge.neq) 1464 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1465 cntxt_id, sc->sge.neq - 1); 1466 sc->sge.eqmap[cntxt_id] = eq; 1467 1468 return (rc); 1469 } 1470 1471 static int 1472 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1473 { 1474 int rc, cntxt_id; 1475 struct fw_eq_eth_cmd c; 1476 1477 bzero(&c, sizeof (c)); 1478 1479 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST | 1480 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) | 1481 V_FW_EQ_ETH_CMD_VFN(0)); 1482 c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC | 1483 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c)); 1484 c.viid_pkd = BE_32(V_FW_EQ_ETH_CMD_VIID(pi->viid)); 1485 c.fetchszm_to_iqid = 1486 BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1487 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | 1488 V_FW_EQ_ETH_CMD_IQID(eq->iqid)); 1489 c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1490 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1491 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1492 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize)); 1493 c.eqaddr = BE_64(eq->ba); 1494 1495 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1496 if (rc != 0) { 1497 cxgb_printf(pi->dip, CE_WARN, 1498 "failed to create Ethernet egress queue: %d", rc); 1499 return (rc); 1500 } 1501 eq->flags |= EQ_ALLOCATED; 1502 1503 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd)); 1504 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1505 if (cntxt_id >= sc->sge.neq) 1506 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1507 cntxt_id, sc->sge.neq - 1); 1508 sc->sge.eqmap[cntxt_id] = eq; 1509 1510 return (rc); 1511 } 1512 1513 #ifndef TCP_OFFLOAD_DISABLE 1514 static int 1515 ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1516 { 1517 int rc, cntxt_id; 1518 struct fw_eq_ofld_cmd c; 1519 1520 bzero(&c, sizeof (c)); 1521 1522 c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST | 1523 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) | 1524 V_FW_EQ_OFLD_CMD_VFN(0)); 1525 c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC | 1526 F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c)); 1527 c.fetchszm_to_iqid = 1528 htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1529 V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) | 1530 F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid)); 1531 c.dcaen_to_eqsize = 1532 BE_32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1533 V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1534 V_FW_EQ_OFLD_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1535 V_FW_EQ_OFLD_CMD_EQSIZE(eq->qsize)); 1536 c.eqaddr = BE_64(eq->ba); 1537 1538 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1539 if (rc != 0) { 1540 cxgb_printf(pi->dip, CE_WARN, 1541 "failed to create egress queue for TCP offload: %d", rc); 1542 return (rc); 1543 } 1544 eq->flags |= EQ_ALLOCATED; 1545 1546 eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(BE_32(c.eqid_pkd)); 1547 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1548 if (cntxt_id >= sc->sge.neq) 1549 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1550 cntxt_id, sc->sge.neq - 1); 1551 sc->sge.eqmap[cntxt_id] = eq; 1552 1553 return (rc); 1554 } 1555 #endif 1556 1557 static int 1558 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1559 { 1560 int rc; 1561 size_t len; 1562 1563 mutex_init(&eq->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(sc->intr_pri)); 1564 eq->flags |= EQ_MTX; 1565 1566 len = eq->qsize * EQ_ESIZE; 1567 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &eq->desc_dhdl, 1568 &eq->desc_ahdl, &eq->ba, (caddr_t *)&eq->desc); 1569 if (rc != 0) 1570 return (rc); 1571 1572 eq->cap = eq->qsize - SPG_SIZE / EQ_ESIZE; 1573 eq->spg = (void *)&eq->desc[eq->cap]; 1574 eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */ 1575 eq->pidx = eq->cidx = 0; 1576 eq->doorbells = sc->doorbells; 1577 1578 switch (eq->flags & EQ_TYPEMASK) { 1579 case EQ_CTRL: 1580 rc = ctrl_eq_alloc(sc, eq); 1581 break; 1582 1583 case EQ_ETH: 1584 rc = eth_eq_alloc(sc, pi, eq); 1585 break; 1586 1587 #ifndef TCP_OFFLOAD_DISABLE 1588 case EQ_OFLD: 1589 rc = ofld_eq_alloc(sc, pi, eq); 1590 break; 1591 #endif 1592 1593 default: 1594 panic("%s: invalid eq type %d.", __func__, 1595 eq->flags & EQ_TYPEMASK); 1596 } 1597 1598 if (eq->doorbells & 1599 (DOORBELL_UDB | DOORBELL_UDBWC | DOORBELL_WCWR)) { 1600 uint32_t s_qpp = sc->sge.s_qpp; 1601 uint32_t mask = (1 << s_qpp) - 1; 1602 volatile uint8_t *udb; 1603 1604 udb = (volatile uint8_t *)sc->reg1p + UDBS_DB_OFFSET; 1605 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */ 1606 eq->udb_qid = eq->cntxt_id & mask; /* id in page */ 1607 if (eq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE) 1608 eq->doorbells &= ~DOORBELL_WCWR; 1609 else { 1610 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */ 1611 eq->udb_qid = 0; 1612 } 1613 eq->udb = (volatile void *)udb; 1614 } 1615 1616 if (rc != 0) { 1617 cxgb_printf(sc->dip, CE_WARN, 1618 "failed to allocate egress queue(%d): %d", 1619 eq->flags & EQ_TYPEMASK, rc); 1620 } 1621 1622 return (rc); 1623 } 1624 1625 static int 1626 free_eq(struct adapter *sc, struct sge_eq *eq) 1627 { 1628 int rc; 1629 1630 if (eq->flags & EQ_ALLOCATED) { 1631 switch (eq->flags & EQ_TYPEMASK) { 1632 case EQ_CTRL: 1633 rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0, 1634 eq->cntxt_id); 1635 break; 1636 1637 case EQ_ETH: 1638 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, 1639 eq->cntxt_id); 1640 break; 1641 1642 #ifndef TCP_OFFLOAD_DISABLE 1643 case EQ_OFLD: 1644 rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0, 1645 eq->cntxt_id); 1646 break; 1647 #endif 1648 1649 default: 1650 panic("%s: invalid eq type %d.", __func__, 1651 eq->flags & EQ_TYPEMASK); 1652 } 1653 if (rc != 0) { 1654 cxgb_printf(sc->dip, CE_WARN, 1655 "failed to free egress queue (%d): %d", 1656 eq->flags & EQ_TYPEMASK, rc); 1657 return (rc); 1658 } 1659 eq->flags &= ~EQ_ALLOCATED; 1660 } 1661 1662 if (eq->desc != NULL) { 1663 (void) free_desc_ring(&eq->desc_dhdl, &eq->desc_ahdl); 1664 eq->desc = NULL; 1665 } 1666 1667 if (eq->flags & EQ_MTX) 1668 mutex_destroy(&eq->lock); 1669 1670 bzero(eq, sizeof (*eq)); 1671 return (0); 1672 } 1673 1674 /* ARGSUSED */ 1675 static int 1676 alloc_wrq(struct adapter *sc, struct port_info *pi, struct sge_wrq *wrq, 1677 int idx) 1678 { 1679 int rc; 1680 1681 rc = alloc_eq(sc, pi, &wrq->eq); 1682 if (rc != 0) 1683 return (rc); 1684 1685 wrq->adapter = sc; 1686 wrq->wr_list.head = NULL; 1687 wrq->wr_list.tail = NULL; 1688 1689 /* 1690 * TODO: use idx to figure out what kind of wrq this is and install 1691 * useful kstats for it. 1692 */ 1693 1694 return (rc); 1695 } 1696 1697 static int 1698 free_wrq(struct adapter *sc, struct sge_wrq *wrq) 1699 { 1700 int rc; 1701 1702 rc = free_eq(sc, &wrq->eq); 1703 if (rc != 0) 1704 return (rc); 1705 1706 bzero(wrq, sizeof (*wrq)); 1707 return (0); 1708 } 1709 1710 static int 1711 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx) 1712 { 1713 int rc, i; 1714 struct adapter *sc = pi->adapter; 1715 struct sge_eq *eq = &txq->eq; 1716 1717 rc = alloc_eq(sc, pi, eq); 1718 if (rc != 0) 1719 return (rc); 1720 1721 txq->port = pi; 1722 txq->sdesc = kmem_zalloc(sizeof (struct tx_sdesc) * eq->cap, KM_SLEEP); 1723 txq->txb_size = eq->qsize * tx_copy_threshold; 1724 rc = alloc_tx_copybuffer(sc, txq->txb_size, &txq->txb_dhdl, 1725 &txq->txb_ahdl, &txq->txb_ba, &txq->txb_va); 1726 if (rc == 0) 1727 txq->txb_avail = txq->txb_size; 1728 else 1729 txq->txb_avail = txq->txb_size = 0; 1730 1731 /* 1732 * TODO: is this too low? Worst case would need around 4 times qsize 1733 * (all tx descriptors filled to the brim with SGLs, with each entry in 1734 * the SGL coming from a distinct DMA handle). Increase tx_dhdl_total 1735 * if you see too many dma_hdl_failed. 1736 */ 1737 txq->tx_dhdl_total = eq->qsize * 2; 1738 txq->tx_dhdl = kmem_zalloc(sizeof (ddi_dma_handle_t) * 1739 txq->tx_dhdl_total, KM_SLEEP); 1740 for (i = 0; i < txq->tx_dhdl_total; i++) { 1741 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx, 1742 DDI_DMA_SLEEP, 0, &txq->tx_dhdl[i]); 1743 if (rc != DDI_SUCCESS) { 1744 cxgb_printf(sc->dip, CE_WARN, 1745 "%s: failed to allocate DMA handle (%d)", 1746 __func__, rc); 1747 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 1748 } 1749 txq->tx_dhdl_avail++; 1750 } 1751 1752 txq->ksp = setup_txq_kstats(pi, txq, idx); 1753 1754 return (rc); 1755 } 1756 1757 static int 1758 free_txq(struct port_info *pi, struct sge_txq *txq) 1759 { 1760 int i; 1761 struct adapter *sc = pi->adapter; 1762 struct sge_eq *eq = &txq->eq; 1763 1764 if (txq->ksp != NULL) { 1765 kstat_delete(txq->ksp); 1766 txq->ksp = NULL; 1767 } 1768 1769 if (txq->txb_va != NULL) { 1770 (void) free_desc_ring(&txq->txb_dhdl, &txq->txb_ahdl); 1771 txq->txb_va = NULL; 1772 } 1773 1774 if (txq->sdesc != NULL) { 1775 struct tx_sdesc *sd; 1776 ddi_dma_handle_t hdl; 1777 1778 TXQ_LOCK(txq); 1779 while (eq->cidx != eq->pidx) { 1780 sd = &txq->sdesc[eq->cidx]; 1781 1782 for (i = sd->hdls_used; i; i--) { 1783 hdl = txq->tx_dhdl[txq->tx_dhdl_cidx]; 1784 (void) ddi_dma_unbind_handle(hdl); 1785 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total) 1786 txq->tx_dhdl_cidx = 0; 1787 } 1788 1789 ASSERT(sd->m); 1790 freemsgchain(sd->m); 1791 1792 eq->cidx += sd->desc_used; 1793 if (eq->cidx >= eq->cap) 1794 eq->cidx -= eq->cap; 1795 1796 txq->txb_avail += txq->txb_used; 1797 } 1798 ASSERT(txq->tx_dhdl_cidx == txq->tx_dhdl_pidx); 1799 ASSERT(txq->txb_avail == txq->txb_size); 1800 TXQ_UNLOCK(txq); 1801 1802 kmem_free(txq->sdesc, sizeof (struct tx_sdesc) * eq->cap); 1803 txq->sdesc = NULL; 1804 } 1805 1806 if (txq->tx_dhdl != NULL) { 1807 for (i = 0; i < txq->tx_dhdl_total; i++) { 1808 if (txq->tx_dhdl[i] != NULL) 1809 ddi_dma_free_handle(&txq->tx_dhdl[i]); 1810 } 1811 } 1812 1813 (void) free_eq(sc, &txq->eq); 1814 1815 bzero(txq, sizeof (*txq)); 1816 return (0); 1817 } 1818 1819 /* 1820 * Allocates a block of contiguous memory for DMA. Can be used to allocate 1821 * memory for descriptor rings or for tx/rx copy buffers. 1822 * 1823 * Caller does not have to clean up anything if this function fails, it cleans 1824 * up after itself. 1825 * 1826 * Caller provides the following: 1827 * len length of the block of memory to allocate. 1828 * flags DDI_DMA_* flags to use (CONSISTENT/STREAMING, READ/WRITE/RDWR) 1829 * acc_attr device access attributes for the allocation. 1830 * dma_attr DMA attributes for the allocation 1831 * 1832 * If the function is successful it fills up this information: 1833 * dma_hdl DMA handle for the allocated memory 1834 * acc_hdl access handle for the allocated memory 1835 * ba bus address of the allocated memory 1836 * va KVA of the allocated memory. 1837 */ 1838 static int 1839 alloc_dma_memory(struct adapter *sc, size_t len, int flags, 1840 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr, 1841 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 1842 uint64_t *pba, caddr_t *pva) 1843 { 1844 int rc; 1845 ddi_dma_handle_t dhdl; 1846 ddi_acc_handle_t ahdl; 1847 ddi_dma_cookie_t cookie; 1848 uint_t ccount; 1849 caddr_t va; 1850 size_t real_len; 1851 1852 *pva = NULL; 1853 1854 /* 1855 * DMA handle. 1856 */ 1857 rc = ddi_dma_alloc_handle(sc->dip, dma_attr, DDI_DMA_SLEEP, 0, &dhdl); 1858 if (rc != DDI_SUCCESS) { 1859 cxgb_printf(sc->dip, CE_WARN, 1860 "failed to allocate DMA handle: %d", rc); 1861 1862 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 1863 } 1864 1865 /* 1866 * Memory suitable for DMA. 1867 */ 1868 rc = ddi_dma_mem_alloc(dhdl, len, acc_attr, 1869 flags & DDI_DMA_CONSISTENT ? DDI_DMA_CONSISTENT : DDI_DMA_STREAMING, 1870 DDI_DMA_SLEEP, 0, &va, &real_len, &ahdl); 1871 if (rc != DDI_SUCCESS) { 1872 cxgb_printf(sc->dip, CE_WARN, 1873 "failed to allocate DMA memory: %d", rc); 1874 1875 ddi_dma_free_handle(&dhdl); 1876 return (ENOMEM); 1877 } 1878 1879 if (len != real_len) { 1880 cxgb_printf(sc->dip, CE_WARN, 1881 "%s: len (%u) != real_len (%u)\n", len, real_len); 1882 } 1883 1884 /* 1885 * DMA bindings. 1886 */ 1887 rc = ddi_dma_addr_bind_handle(dhdl, NULL, va, real_len, flags, NULL, 1888 NULL, &cookie, &ccount); 1889 if (rc != DDI_DMA_MAPPED) { 1890 cxgb_printf(sc->dip, CE_WARN, 1891 "failed to map DMA memory: %d", rc); 1892 1893 ddi_dma_mem_free(&ahdl); 1894 ddi_dma_free_handle(&dhdl); 1895 return (ENOMEM); 1896 } 1897 if (ccount != 1) { 1898 cxgb_printf(sc->dip, CE_WARN, 1899 "unusable DMA mapping (%d segments)", ccount); 1900 (void) free_desc_ring(&dhdl, &ahdl); 1901 } 1902 1903 bzero(va, real_len); 1904 *dma_hdl = dhdl; 1905 *acc_hdl = ahdl; 1906 *pba = cookie.dmac_laddress; 1907 *pva = va; 1908 1909 return (0); 1910 } 1911 1912 static int 1913 free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl) 1914 { 1915 (void) ddi_dma_unbind_handle(*dhdl); 1916 ddi_dma_mem_free(ahdl); 1917 ddi_dma_free_handle(dhdl); 1918 1919 return (0); 1920 } 1921 1922 static int 1923 alloc_desc_ring(struct adapter *sc, size_t len, int rw, 1924 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 1925 uint64_t *pba, caddr_t *pva) 1926 { 1927 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_desc; 1928 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; 1929 1930 return (alloc_dma_memory(sc, len, DDI_DMA_CONSISTENT | rw, acc_attr, 1931 dma_attr, dma_hdl, acc_hdl, pba, pva)); 1932 } 1933 1934 static int 1935 free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl) 1936 { 1937 return (free_dma_memory(dhdl, ahdl)); 1938 } 1939 1940 static int 1941 alloc_tx_copybuffer(struct adapter *sc, size_t len, 1942 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 1943 uint64_t *pba, caddr_t *pva) 1944 { 1945 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_tx; 1946 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; /* NOT dma_attr_tx */ 1947 1948 return (alloc_dma_memory(sc, len, DDI_DMA_STREAMING | DDI_DMA_WRITE, 1949 acc_attr, dma_attr, dma_hdl, acc_hdl, pba, pva)); 1950 } 1951 1952 static inline bool 1953 is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl) 1954 { 1955 (void) ddi_dma_sync(iq->dhdl, (uintptr_t)iq->cdesc - 1956 (uintptr_t)iq->desc, iq->esize, DDI_DMA_SYNC_FORKERNEL); 1957 1958 *ctrl = (void *)((uintptr_t)iq->cdesc + 1959 (iq->esize - sizeof (struct rsp_ctrl))); 1960 1961 return ((((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen)); 1962 } 1963 1964 static inline void 1965 iq_next(struct sge_iq *iq) 1966 { 1967 iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize); 1968 if (++iq->cidx == iq->qsize - 1) { 1969 iq->cidx = 0; 1970 iq->gen ^= 1; 1971 iq->cdesc = iq->desc; 1972 } 1973 } 1974 1975 /* 1976 * Fill up the freelist by upto nbufs and maybe ring its doorbell. 1977 * 1978 * Returns non-zero to indicate that it should be added to the list of starving 1979 * freelists. 1980 */ 1981 static int 1982 refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs) 1983 { 1984 uint64_t *d = &fl->desc[fl->pidx]; 1985 struct fl_sdesc *sd = &fl->sdesc[fl->pidx]; 1986 1987 FL_LOCK_ASSERT_OWNED(fl); 1988 ASSERT(nbufs >= 0); 1989 1990 if (nbufs > fl->needed) 1991 nbufs = fl->needed; 1992 1993 while (nbufs--) { 1994 if (sd->rxb != NULL) { 1995 if (sd->rxb->ref_cnt == 1) { 1996 /* 1997 * Buffer is available for recycling. Two ways 1998 * this can happen: 1999 * 2000 * a) All the packets DMA'd into it last time 2001 * around were within the rx_copy_threshold 2002 * and no part of the buffer was ever passed 2003 * up (ref_cnt never went over 1). 2004 * 2005 * b) Packets DMA'd into the buffer were passed 2006 * up but have all been freed by the upper 2007 * layers by now (ref_cnt went over 1 but is 2008 * now back to 1). 2009 * 2010 * Either way the bus address in the descriptor 2011 * ring is already valid. 2012 */ 2013 ASSERT(*d == cpu_to_be64(sd->rxb->ba)); 2014 d++; 2015 goto recycled; 2016 } else { 2017 /* 2018 * Buffer still in use and we need a 2019 * replacement. But first release our reference 2020 * on the existing buffer. 2021 */ 2022 rxbuf_free(sd->rxb); 2023 } 2024 } 2025 2026 sd->rxb = rxbuf_alloc(sc->sge.rxbuf_cache, KM_NOSLEEP, 1); 2027 if (sd->rxb == NULL) 2028 break; 2029 *d++ = cpu_to_be64(sd->rxb->ba); 2030 2031 recycled: fl->pending++; 2032 sd++; 2033 fl->needed--; 2034 if (++fl->pidx == fl->cap) { 2035 fl->pidx = 0; 2036 sd = fl->sdesc; 2037 d = fl->desc; 2038 } 2039 } 2040 2041 if (fl->pending >= 8) 2042 ring_fl_db(sc, fl); 2043 2044 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING)); 2045 } 2046 2047 #ifndef TAILQ_FOREACH_SAFE 2048 #define TAILQ_FOREACH_SAFE(var, head, field, tvar) \ 2049 for ((var) = TAILQ_FIRST((head)); \ 2050 (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \ 2051 (var) = (tvar)) 2052 #endif 2053 2054 /* 2055 * Attempt to refill all starving freelists. 2056 */ 2057 static void 2058 refill_sfl(void *arg) 2059 { 2060 struct adapter *sc = arg; 2061 struct sge_fl *fl, *fl_temp; 2062 2063 mutex_enter(&sc->sfl_lock); 2064 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) { 2065 FL_LOCK(fl); 2066 (void) refill_fl(sc, fl, 64); 2067 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) { 2068 TAILQ_REMOVE(&sc->sfl, fl, link); 2069 fl->flags &= ~FL_STARVING; 2070 } 2071 FL_UNLOCK(fl); 2072 } 2073 2074 if (!TAILQ_EMPTY(&sc->sfl) != 0) 2075 sc->sfl_timer = timeout(refill_sfl, sc, drv_usectohz(100000)); 2076 mutex_exit(&sc->sfl_lock); 2077 } 2078 2079 static void 2080 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl) 2081 { 2082 mutex_enter(&sc->sfl_lock); 2083 FL_LOCK(fl); 2084 if ((fl->flags & FL_DOOMED) == 0) { 2085 if (TAILQ_EMPTY(&sc->sfl) != 0) { 2086 sc->sfl_timer = timeout(refill_sfl, sc, 2087 drv_usectohz(100000)); 2088 } 2089 fl->flags |= FL_STARVING; 2090 TAILQ_INSERT_TAIL(&sc->sfl, fl, link); 2091 } 2092 FL_UNLOCK(fl); 2093 mutex_exit(&sc->sfl_lock); 2094 } 2095 2096 static void 2097 free_fl_bufs(struct sge_fl *fl) 2098 { 2099 struct fl_sdesc *sd; 2100 unsigned int i; 2101 2102 FL_LOCK_ASSERT_OWNED(fl); 2103 2104 for (i = 0; i < fl->cap; i++) { 2105 sd = &fl->sdesc[i]; 2106 2107 if (sd->rxb != NULL) { 2108 rxbuf_free(sd->rxb); 2109 sd->rxb = NULL; 2110 } 2111 } 2112 } 2113 2114 /* 2115 * Note that fl->cidx and fl->offset are left unchanged in case of failure. 2116 */ 2117 static mblk_t * 2118 get_fl_payload(struct sge_fl *fl, uint32_t len_newbuf, int *fl_bufs_used) 2119 { 2120 struct mblk_pair frame = {0}; 2121 struct rxbuf *rxb; 2122 mblk_t *m = NULL; 2123 uint_t nbuf = 0, len, copy, n; 2124 uint32_t cidx, offset; 2125 2126 /* 2127 * The SGE won't pack a new frame into the current buffer if the entire 2128 * payload doesn't fit in the remaining space. Move on to the next buf 2129 * in that case. 2130 */ 2131 if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) { 2132 fl->offset = 0; 2133 if (++fl->cidx == fl->cap) 2134 fl->cidx = 0; 2135 nbuf++; 2136 } 2137 cidx = fl->cidx; 2138 offset = fl->offset; 2139 2140 len = G_RSPD_LEN(len_newbuf); /* pktshift + payload length */ 2141 copy = (len <= fl->copy_threshold); 2142 if (copy != 0) { 2143 frame.head = m = allocb(len, BPRI_HI); 2144 if (m == NULL) 2145 return (NULL); 2146 } 2147 2148 while (len) { 2149 rxb = fl->sdesc[cidx].rxb; 2150 n = min(len, rxb->buf_size - offset); 2151 2152 (void) ddi_dma_sync(rxb->dhdl, offset, n, 2153 DDI_DMA_SYNC_FORKERNEL); 2154 2155 if (copy != 0) 2156 bcopy(rxb->va + offset, m->b_wptr, n); 2157 else { 2158 m = desballoc((unsigned char *)rxb->va + offset, n, 2159 BPRI_HI, &rxb->freefunc); 2160 if (m == NULL) { 2161 freemsg(frame.head); 2162 return (NULL); 2163 } 2164 atomic_inc_uint(&rxb->ref_cnt); 2165 if (frame.head != NULL) 2166 frame.tail->b_cont = m; 2167 else 2168 frame.head = m; 2169 frame.tail = m; 2170 } 2171 m->b_wptr += n; 2172 len -= n; 2173 offset += roundup(n, FL_ALIGN); 2174 ASSERT(offset <= rxb->buf_size); 2175 if (offset == rxb->buf_size) { 2176 offset = 0; 2177 if (++cidx == fl->cap) 2178 cidx = 0; 2179 nbuf++; 2180 } 2181 } 2182 2183 fl->cidx = cidx; 2184 fl->offset = offset; 2185 (*fl_bufs_used) += nbuf; 2186 2187 ASSERT(frame.head != NULL); 2188 return (frame.head); 2189 } 2190 2191 /* 2192 * We'll do immediate data tx for non-LSO, but only when not coalescing. We're 2193 * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes 2194 * of immediate data. 2195 */ 2196 #define IMM_LEN ( \ 2197 2 * EQ_ESIZE \ 2198 - sizeof (struct fw_eth_tx_pkt_wr) \ 2199 - sizeof (struct cpl_tx_pkt_core)) 2200 2201 /* 2202 * Returns non-zero on failure, no need to cleanup anything in that case. 2203 * 2204 * Note 1: We always try to pull up the mblk if required and return E2BIG only 2205 * if this fails. 2206 * 2207 * Note 2: We'll also pullup incoming mblk if HW_LSO is set and the first mblk 2208 * does not have the TCP header in it. 2209 */ 2210 static int 2211 get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, struct txinfo *txinfo, 2212 int sgl_only) 2213 { 2214 uint32_t flags = 0, len, n; 2215 mblk_t *m = *fp; 2216 int rc; 2217 2218 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb and dma_hdls */ 2219 2220 mac_hcksum_get(m, NULL, NULL, NULL, NULL, &flags); 2221 txinfo->flags = flags; 2222 2223 mac_lso_get(m, &txinfo->mss, &flags); 2224 txinfo->flags |= flags; 2225 2226 if (flags & HW_LSO) 2227 sgl_only = 1; /* Do not allow immediate data with LSO */ 2228 2229 start: txinfo->nsegs = 0; 2230 txinfo->hdls_used = 0; 2231 txinfo->txb_used = 0; 2232 txinfo->len = 0; 2233 2234 /* total length and a rough estimate of # of segments */ 2235 n = 0; 2236 for (; m; m = m->b_cont) { 2237 len = MBLKL(m); 2238 n += (len / PAGE_SIZE) + 1; 2239 txinfo->len += len; 2240 } 2241 m = *fp; 2242 2243 if (n >= TX_SGL_SEGS || (flags & HW_LSO && MBLKL(m) < 50)) { 2244 txq->pullup_early++; 2245 m = msgpullup(*fp, -1); 2246 if (m == NULL) { 2247 txq->pullup_failed++; 2248 return (E2BIG); /* (*fp) left as it was */ 2249 } 2250 freemsg(*fp); 2251 *fp = m; 2252 mac_hcksum_set(m, NULL, NULL, NULL, NULL, txinfo->flags); 2253 } 2254 2255 if (txinfo->len <= IMM_LEN && !sgl_only) 2256 return (0); /* nsegs = 0 tells caller to use imm. tx */ 2257 2258 if (txinfo->len <= txq->copy_threshold && 2259 copy_into_txb(txq, m, txinfo->len, txinfo) == 0) 2260 goto done; 2261 2262 for (; m; m = m->b_cont) { 2263 2264 len = MBLKL(m); 2265 2266 /* Use tx copy buffer if this mblk is small enough */ 2267 if (len <= txq->copy_threshold && 2268 copy_into_txb(txq, m, len, txinfo) == 0) 2269 continue; 2270 2271 /* Add DMA bindings for this mblk to the SGL */ 2272 rc = add_mblk(txq, txinfo, m, len); 2273 2274 if (rc == E2BIG || 2275 (txinfo->nsegs == TX_SGL_SEGS && m->b_cont)) { 2276 2277 txq->pullup_late++; 2278 m = msgpullup(*fp, -1); 2279 if (m != NULL) { 2280 free_txinfo_resources(txq, txinfo); 2281 freemsg(*fp); 2282 *fp = m; 2283 mac_hcksum_set(m, NULL, NULL, NULL, NULL, 2284 txinfo->flags); 2285 goto start; 2286 } 2287 2288 txq->pullup_failed++; 2289 rc = E2BIG; 2290 } 2291 2292 if (rc != 0) { 2293 free_txinfo_resources(txq, txinfo); 2294 return (rc); 2295 } 2296 } 2297 2298 ASSERT(txinfo->nsegs > 0 && txinfo->nsegs <= TX_SGL_SEGS); 2299 2300 done: 2301 2302 /* 2303 * Store the # of flits required to hold this frame's SGL in nflits. An 2304 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by 2305 * multiple (len0 + len1, addr0, addr1) tuples. If addr1 is not used 2306 * then len1 must be set to 0. 2307 */ 2308 n = txinfo->nsegs - 1; 2309 txinfo->nflits = (3 * n) / 2 + (n & 1) + 2; 2310 if (n & 1) 2311 txinfo->sgl.sge[n / 2].len[1] = cpu_to_be32(0); 2312 2313 txinfo->sgl.cmd_nsge = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_DSGL) | 2314 V_ULPTX_NSGE(txinfo->nsegs)); 2315 2316 return (0); 2317 } 2318 2319 static inline int 2320 fits_in_txb(struct sge_txq *txq, int len, int *waste) 2321 { 2322 if (txq->txb_avail < len) 2323 return (0); 2324 2325 if (txq->txb_next + len <= txq->txb_size) { 2326 *waste = 0; 2327 return (1); 2328 } 2329 2330 *waste = txq->txb_size - txq->txb_next; 2331 2332 return (txq->txb_avail - *waste < len ? 0 : 1); 2333 } 2334 2335 #define TXB_CHUNK 64 2336 2337 /* 2338 * Copies the specified # of bytes into txq's tx copy buffer and updates txinfo 2339 * and txq to indicate resources used. Caller has to make sure that those many 2340 * bytes are available in the mblk chain (b_cont linked). 2341 */ 2342 static inline int 2343 copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, struct txinfo *txinfo) 2344 { 2345 int waste, n; 2346 2347 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb */ 2348 2349 if (!fits_in_txb(txq, len, &waste)) { 2350 txq->txb_full++; 2351 return (ENOMEM); 2352 } 2353 2354 if (waste != 0) { 2355 ASSERT((waste & (TXB_CHUNK - 1)) == 0); 2356 txinfo->txb_used += waste; 2357 txq->txb_avail -= waste; 2358 txq->txb_next = 0; 2359 } 2360 2361 for (n = 0; n < len; m = m->b_cont) { 2362 bcopy(m->b_rptr, txq->txb_va + txq->txb_next + n, MBLKL(m)); 2363 n += MBLKL(m); 2364 } 2365 2366 add_seg(txinfo, txq->txb_ba + txq->txb_next, len); 2367 2368 n = roundup(len, TXB_CHUNK); 2369 txinfo->txb_used += n; 2370 txq->txb_avail -= n; 2371 txq->txb_next += n; 2372 ASSERT(txq->txb_next <= txq->txb_size); 2373 if (txq->txb_next == txq->txb_size) 2374 txq->txb_next = 0; 2375 2376 return (0); 2377 } 2378 2379 static inline void 2380 add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len) 2381 { 2382 ASSERT(txinfo->nsegs < TX_SGL_SEGS); /* must have room */ 2383 2384 if (txinfo->nsegs != 0) { 2385 int idx = txinfo->nsegs - 1; 2386 txinfo->sgl.sge[idx / 2].len[idx & 1] = cpu_to_be32(len); 2387 txinfo->sgl.sge[idx / 2].addr[idx & 1] = cpu_to_be64(ba); 2388 } else { 2389 txinfo->sgl.len0 = cpu_to_be32(len); 2390 txinfo->sgl.addr0 = cpu_to_be64(ba); 2391 } 2392 txinfo->nsegs++; 2393 } 2394 2395 /* 2396 * This function cleans up any partially allocated resources when it fails so 2397 * there's nothing for the caller to clean up in that case. 2398 * 2399 * EIO indicates permanent failure. Caller should drop the frame containing 2400 * this mblk and continue. 2401 * 2402 * E2BIG indicates that the SGL length for this mblk exceeds the hardware 2403 * limit. Caller should pull up the frame before trying to send it out. 2404 * (This error means our pullup_early heuristic did not work for this frame) 2405 * 2406 * ENOMEM indicates a temporary shortage of resources (DMA handles, other DMA 2407 * resources, etc.). Caller should suspend the tx queue and wait for reclaim to 2408 * free up resources. 2409 */ 2410 static inline int 2411 add_mblk(struct sge_txq *txq, struct txinfo *txinfo, mblk_t *m, int len) 2412 { 2413 ddi_dma_handle_t dhdl; 2414 ddi_dma_cookie_t cookie; 2415 uint_t ccount = 0; 2416 int rc; 2417 2418 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate dhdls */ 2419 2420 if (txq->tx_dhdl_avail == 0) { 2421 txq->dma_hdl_failed++; 2422 return (ENOMEM); 2423 } 2424 2425 dhdl = txq->tx_dhdl[txq->tx_dhdl_pidx]; 2426 rc = ddi_dma_addr_bind_handle(dhdl, NULL, (caddr_t)m->b_rptr, len, 2427 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie, 2428 &ccount); 2429 if (rc != DDI_DMA_MAPPED) { 2430 txq->dma_map_failed++; 2431 2432 ASSERT(rc != DDI_DMA_INUSE && rc != DDI_DMA_PARTIAL_MAP); 2433 2434 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EIO); 2435 } 2436 2437 if (ccount + txinfo->nsegs > TX_SGL_SEGS) { 2438 (void) ddi_dma_unbind_handle(dhdl); 2439 return (E2BIG); 2440 } 2441 2442 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size); 2443 while (--ccount) { 2444 ddi_dma_nextcookie(dhdl, &cookie); 2445 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size); 2446 } 2447 2448 if (++txq->tx_dhdl_pidx == txq->tx_dhdl_total) 2449 txq->tx_dhdl_pidx = 0; 2450 txq->tx_dhdl_avail--; 2451 txinfo->hdls_used++; 2452 2453 return (0); 2454 } 2455 2456 /* 2457 * Releases all the txq resources used up in the specified txinfo. 2458 */ 2459 static void 2460 free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo) 2461 { 2462 int n; 2463 2464 TXQ_LOCK_ASSERT_OWNED(txq); /* dhdls, txb */ 2465 2466 n = txinfo->txb_used; 2467 if (n > 0) { 2468 txq->txb_avail += n; 2469 if (n <= txq->txb_next) 2470 txq->txb_next -= n; 2471 else { 2472 n -= txq->txb_next; 2473 txq->txb_next = txq->txb_size - n; 2474 } 2475 } 2476 2477 for (n = txinfo->hdls_used; n > 0; n--) { 2478 if (txq->tx_dhdl_pidx > 0) 2479 txq->tx_dhdl_pidx--; 2480 else 2481 txq->tx_dhdl_pidx = txq->tx_dhdl_total - 1; 2482 txq->tx_dhdl_avail++; 2483 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_pidx]); 2484 } 2485 } 2486 2487 /* 2488 * Returns 0 to indicate that m has been accepted into a coalesced tx work 2489 * request. It has either been folded into txpkts or txpkts was flushed and m 2490 * has started a new coalesced work request (as the first frame in a fresh 2491 * txpkts). 2492 * 2493 * Returns non-zero to indicate a failure - caller is responsible for 2494 * transmitting m, if there was anything in txpkts it has been flushed. 2495 */ 2496 static int 2497 add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m, 2498 struct txinfo *txinfo) 2499 { 2500 struct sge_eq *eq = &txq->eq; 2501 int can_coalesce; 2502 struct tx_sdesc *txsd; 2503 uint8_t flits; 2504 2505 TXQ_LOCK_ASSERT_OWNED(txq); 2506 2507 if (txpkts->npkt > 0) { 2508 flits = TXPKTS_PKT_HDR + txinfo->nflits; 2509 can_coalesce = (txinfo->flags & HW_LSO) == 0 && 2510 txpkts->nflits + flits <= TX_WR_FLITS && 2511 txpkts->nflits + flits <= eq->avail * 8 && 2512 txpkts->plen + txinfo->len < 65536; 2513 2514 if (can_coalesce != 0) { 2515 txpkts->tail->b_next = m; 2516 txpkts->tail = m; 2517 txpkts->npkt++; 2518 txpkts->nflits += flits; 2519 txpkts->plen += txinfo->len; 2520 2521 txsd = &txq->sdesc[eq->pidx]; 2522 txsd->txb_used += txinfo->txb_used; 2523 txsd->hdls_used += txinfo->hdls_used; 2524 2525 return (0); 2526 } 2527 2528 /* 2529 * Couldn't coalesce m into txpkts. The first order of business 2530 * is to send txpkts on its way. Then we'll revisit m. 2531 */ 2532 write_txpkts_wr(txq, txpkts); 2533 } 2534 2535 /* 2536 * Check if we can start a new coalesced tx work request with m as 2537 * the first packet in it. 2538 */ 2539 2540 ASSERT(txpkts->npkt == 0); 2541 ASSERT(txinfo->len < 65536); 2542 2543 flits = TXPKTS_WR_HDR + txinfo->nflits; 2544 can_coalesce = (txinfo->flags & HW_LSO) == 0 && 2545 flits <= eq->avail * 8 && flits <= TX_WR_FLITS; 2546 2547 if (can_coalesce == 0) 2548 return (EINVAL); 2549 2550 /* 2551 * Start a fresh coalesced tx WR with m as the first frame in it. 2552 */ 2553 txpkts->tail = m; 2554 txpkts->npkt = 1; 2555 txpkts->nflits = flits; 2556 txpkts->flitp = &eq->desc[eq->pidx].flit[2]; 2557 txpkts->plen = txinfo->len; 2558 2559 txsd = &txq->sdesc[eq->pidx]; 2560 txsd->m = m; 2561 txsd->txb_used = txinfo->txb_used; 2562 txsd->hdls_used = txinfo->hdls_used; 2563 2564 return (0); 2565 } 2566 2567 /* 2568 * Note that write_txpkts_wr can never run out of hardware descriptors (but 2569 * write_txpkt_wr can). add_to_txpkts ensures that a frame is accepted for 2570 * coalescing only if sufficient hardware descriptors are available. 2571 */ 2572 static void 2573 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts) 2574 { 2575 struct sge_eq *eq = &txq->eq; 2576 struct fw_eth_tx_pkts_wr *wr; 2577 struct tx_sdesc *txsd; 2578 uint32_t ctrl; 2579 uint16_t ndesc; 2580 2581 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */ 2582 2583 ndesc = howmany(txpkts->nflits, 8); 2584 2585 wr = (void *)&eq->desc[eq->pidx]; 2586 wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR) | 2587 V_FW_WR_IMMDLEN(0)); /* immdlen does not matter in this WR */ 2588 ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2)); 2589 if (eq->avail == ndesc) 2590 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ; 2591 wr->equiq_to_len16 = cpu_to_be32(ctrl); 2592 wr->plen = cpu_to_be16(txpkts->plen); 2593 wr->npkt = txpkts->npkt; 2594 wr->r3 = wr->type = 0; 2595 2596 /* Everything else already written */ 2597 2598 txsd = &txq->sdesc[eq->pidx]; 2599 txsd->desc_used = ndesc; 2600 2601 txq->txb_used += txsd->txb_used / TXB_CHUNK; 2602 txq->hdl_used += txsd->hdls_used; 2603 2604 ASSERT(eq->avail >= ndesc); 2605 2606 eq->pending += ndesc; 2607 eq->avail -= ndesc; 2608 eq->pidx += ndesc; 2609 if (eq->pidx >= eq->cap) 2610 eq->pidx -= eq->cap; 2611 2612 txq->txpkts_pkts += txpkts->npkt; 2613 txq->txpkts_wrs++; 2614 txpkts->npkt = 0; /* emptied */ 2615 } 2616 2617 static int 2618 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m, 2619 struct txinfo *txinfo) 2620 { 2621 struct sge_eq *eq = &txq->eq; 2622 struct fw_eth_tx_pkt_wr *wr; 2623 struct cpl_tx_pkt_core *cpl; 2624 uint32_t ctrl; /* used in many unrelated places */ 2625 uint64_t ctrl1; 2626 int nflits, ndesc; 2627 struct tx_sdesc *txsd; 2628 caddr_t dst; 2629 2630 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */ 2631 2632 /* 2633 * Do we have enough flits to send this frame out? 2634 */ 2635 ctrl = sizeof (struct cpl_tx_pkt_core); 2636 if (txinfo->flags & HW_LSO) { 2637 nflits = TXPKT_LSO_WR_HDR; 2638 ctrl += sizeof(struct cpl_tx_pkt_lso_core); 2639 } else 2640 nflits = TXPKT_WR_HDR; 2641 if (txinfo->nsegs > 0) 2642 nflits += txinfo->nflits; 2643 else { 2644 nflits += howmany(txinfo->len, 8); 2645 ctrl += txinfo->len; 2646 } 2647 ndesc = howmany(nflits, 8); 2648 if (ndesc > eq->avail) 2649 return (ENOMEM); 2650 2651 /* Firmware work request header */ 2652 wr = (void *)&eq->desc[eq->pidx]; 2653 wr->op_immdlen = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | 2654 V_FW_WR_IMMDLEN(ctrl)); 2655 ctrl = V_FW_WR_LEN16(howmany(nflits, 2)); 2656 if (eq->avail == ndesc) 2657 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ; 2658 wr->equiq_to_len16 = cpu_to_be32(ctrl); 2659 wr->r3 = 0; 2660 2661 if (txinfo->flags & HW_LSO) { 2662 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); 2663 char *p = (void *)m->b_rptr; 2664 ctrl = V_LSO_OPCODE((u32)CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | 2665 F_LSO_LAST_SLICE; 2666 2667 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 2668 if (((struct ether_header *)p)->ether_type == 2669 htons(ETHERTYPE_VLAN)) { 2670 ctrl |= V_LSO_ETHHDR_LEN(1); 2671 p += sizeof (struct ether_vlan_header); 2672 } else 2673 p += sizeof (struct ether_header); 2674 2675 /* LINTED: E_BAD_PTR_CAST_ALIGN for IPH_HDR_LENGTH() */ 2676 ctrl |= V_LSO_IPHDR_LEN(IPH_HDR_LENGTH(p) / 4); 2677 /* LINTED: E_BAD_PTR_CAST_ALIGN for IPH_HDR_LENGTH() */ 2678 p += IPH_HDR_LENGTH(p); 2679 ctrl |= V_LSO_TCPHDR_LEN(TCP_HDR_LENGTH((tcph_t *)p) / 4); 2680 2681 lso->lso_ctrl = cpu_to_be32(ctrl); 2682 lso->ipid_ofst = cpu_to_be16(0); 2683 lso->mss = cpu_to_be16(txinfo->mss); 2684 lso->seqno_offset = cpu_to_be32(0); 2685 if (is_t4(pi->adapter->params.chip)) 2686 lso->len = cpu_to_be32(txinfo->len); 2687 else 2688 lso->len = cpu_to_be32(V_LSO_T5_XFER_SIZE(txinfo->len)); 2689 2690 cpl = (void *)(lso + 1); 2691 2692 txq->tso_wrs++; 2693 } else 2694 cpl = (void *)(wr + 1); 2695 2696 /* Checksum offload */ 2697 ctrl1 = 0; 2698 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM)) 2699 ctrl1 |= F_TXPKT_IPCSUM_DIS; 2700 if (!(txinfo->flags & HCK_FULLCKSUM)) 2701 ctrl1 |= F_TXPKT_L4CSUM_DIS; 2702 if (ctrl1 == 0) 2703 txq->txcsum++; /* some hardware assistance provided */ 2704 2705 /* CPL header */ 2706 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) | 2707 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf)); 2708 cpl->pack = 0; 2709 cpl->len = cpu_to_be16(txinfo->len); 2710 cpl->ctrl1 = cpu_to_be64(ctrl1); 2711 2712 /* Software descriptor */ 2713 txsd = &txq->sdesc[eq->pidx]; 2714 txsd->m = m; 2715 txsd->txb_used = txinfo->txb_used; 2716 txsd->hdls_used = txinfo->hdls_used; 2717 /* LINTED: E_ASSIGN_NARROW_CONV */ 2718 txsd->desc_used = ndesc; 2719 2720 txq->txb_used += txinfo->txb_used / TXB_CHUNK; 2721 txq->hdl_used += txinfo->hdls_used; 2722 2723 eq->pending += ndesc; 2724 eq->avail -= ndesc; 2725 eq->pidx += ndesc; 2726 if (eq->pidx >= eq->cap) 2727 eq->pidx -= eq->cap; 2728 2729 /* SGL */ 2730 dst = (void *)(cpl + 1); 2731 if (txinfo->nsegs > 0) { 2732 txq->sgl_wrs++; 2733 copy_to_txd(eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8); 2734 2735 /* Need to zero-pad to a 16 byte boundary if not on one */ 2736 if ((uintptr_t)dst & 0xf) 2737 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 2738 *(uint64_t *)dst = 0; 2739 2740 } else { 2741 txq->imm_wrs++; 2742 #ifdef DEBUG 2743 ctrl = txinfo->len; 2744 #endif 2745 for (; m; m = m->b_cont) { 2746 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m)); 2747 #ifdef DEBUG 2748 ctrl -= MBLKL(m); 2749 #endif 2750 } 2751 ASSERT(ctrl == 0); 2752 } 2753 2754 txq->txpkt_wrs++; 2755 return (0); 2756 } 2757 2758 static inline void 2759 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq, 2760 struct txpkts *txpkts, struct txinfo *txinfo) 2761 { 2762 struct ulp_txpkt *ulpmc; 2763 struct ulptx_idata *ulpsc; 2764 struct cpl_tx_pkt_core *cpl; 2765 uintptr_t flitp, start, end; 2766 uint64_t ctrl; 2767 caddr_t dst; 2768 2769 ASSERT(txpkts->npkt > 0); 2770 2771 start = (uintptr_t)txq->eq.desc; 2772 end = (uintptr_t)txq->eq.spg; 2773 2774 /* Checksum offload */ 2775 ctrl = 0; 2776 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM)) 2777 ctrl |= F_TXPKT_IPCSUM_DIS; 2778 if (!(txinfo->flags & HCK_FULLCKSUM)) 2779 ctrl |= F_TXPKT_L4CSUM_DIS; 2780 if (ctrl == 0) 2781 txq->txcsum++; /* some hardware assistance provided */ 2782 2783 /* 2784 * The previous packet's SGL must have ended at a 16 byte boundary (this 2785 * is required by the firmware/hardware). It follows that flitp cannot 2786 * wrap around between the ULPTX master command and ULPTX subcommand (8 2787 * bytes each), and that it can not wrap around in the middle of the 2788 * cpl_tx_pkt_core either. 2789 */ 2790 flitp = (uintptr_t)txpkts->flitp; 2791 ASSERT((flitp & 0xf) == 0); 2792 2793 /* ULP master command */ 2794 ulpmc = (void *)flitp; 2795 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); 2796 ulpmc->len = htonl(howmany(sizeof (*ulpmc) + sizeof (*ulpsc) + 2797 sizeof (*cpl) + 8 * txinfo->nflits, 16)); 2798 2799 /* ULP subcommand */ 2800 ulpsc = (void *)(ulpmc + 1); 2801 ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) | 2802 F_ULP_TX_SC_MORE); 2803 ulpsc->len = cpu_to_be32(sizeof (struct cpl_tx_pkt_core)); 2804 2805 flitp += sizeof (*ulpmc) + sizeof (*ulpsc); 2806 if (flitp == end) 2807 flitp = start; 2808 2809 /* CPL_TX_PKT */ 2810 cpl = (void *)flitp; 2811 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) | 2812 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf)); 2813 cpl->pack = 0; 2814 cpl->len = cpu_to_be16(txinfo->len); 2815 cpl->ctrl1 = cpu_to_be64(ctrl); 2816 2817 flitp += sizeof (*cpl); 2818 if (flitp == end) 2819 flitp = start; 2820 2821 /* SGL for this frame */ 2822 dst = (caddr_t)flitp; 2823 copy_to_txd(&txq->eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8); 2824 flitp = (uintptr_t)dst; 2825 2826 /* Zero pad and advance to a 16 byte boundary if not already at one. */ 2827 if (flitp & 0xf) { 2828 2829 /* no matter what, flitp should be on an 8 byte boundary */ 2830 ASSERT((flitp & 0x7) == 0); 2831 2832 *(uint64_t *)flitp = 0; 2833 flitp += sizeof (uint64_t); 2834 txpkts->nflits++; 2835 } 2836 2837 if (flitp == end) 2838 flitp = start; 2839 2840 txpkts->flitp = (void *)flitp; 2841 } 2842 2843 static inline void 2844 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) 2845 { 2846 if ((uintptr_t)(*to) + len <= (uintptr_t)eq->spg) { 2847 bcopy(from, *to, len); 2848 (*to) += len; 2849 } else { 2850 int portion = (uintptr_t)eq->spg - (uintptr_t)(*to); 2851 2852 bcopy(from, *to, portion); 2853 from += portion; 2854 portion = len - portion; /* remaining */ 2855 bcopy(from, (void *)eq->desc, portion); 2856 (*to) = (caddr_t)eq->desc + portion; 2857 } 2858 } 2859 2860 static inline void 2861 ring_tx_db(struct adapter *sc, struct sge_eq *eq) 2862 { 2863 int val, db_mode; 2864 u_int db = eq->doorbells; 2865 2866 if (eq->pending > 1) 2867 db &= ~DOORBELL_WCWR; 2868 2869 if (eq->pending > eq->pidx) { 2870 int offset = eq->cap - (eq->pending - eq->pidx); 2871 2872 /* pidx has wrapped around since last doorbell */ 2873 2874 (void) ddi_dma_sync(eq->desc_dhdl, 2875 offset * sizeof (struct tx_desc), 0, 2876 DDI_DMA_SYNC_FORDEV); 2877 (void) ddi_dma_sync(eq->desc_dhdl, 2878 0, eq->pidx * sizeof (struct tx_desc), 2879 DDI_DMA_SYNC_FORDEV); 2880 } else if (eq->pending > 0) { 2881 (void) ddi_dma_sync(eq->desc_dhdl, 2882 (eq->pidx - eq->pending) * sizeof (struct tx_desc), 2883 eq->pending * sizeof (struct tx_desc), 2884 DDI_DMA_SYNC_FORDEV); 2885 } 2886 2887 membar_producer(); 2888 2889 if (is_t4(sc->params.chip)) 2890 val = V_PIDX(eq->pending); 2891 else 2892 val = V_PIDX_T5(eq->pending); 2893 2894 db_mode = (1 << (ffs(db) - 1)); 2895 switch (db_mode) { 2896 case DOORBELL_UDB: 2897 *eq->udb = LE_32(V_QID(eq->udb_qid) | val); 2898 break; 2899 2900 case DOORBELL_WCWR: 2901 { 2902 volatile uint64_t *dst, *src; 2903 int i; 2904 /* 2905 * Queues whose 128B doorbell segment fits in 2906 * the page do not use relative qid 2907 * (udb_qid is always 0). Only queues with 2908 * doorbell segments can do WCWR. 2909 */ 2910 ASSERT(eq->udb_qid == 0 && eq->pending == 1); 2911 2912 dst = (volatile void *)((uintptr_t)eq->udb + 2913 UDBS_WR_OFFSET - UDBS_DB_OFFSET); 2914 i = eq->pidx ? eq->pidx - 1 : eq->cap - 1; 2915 src = (void *)&eq->desc[i]; 2916 while (src != (void *)&eq->desc[i + 1]) 2917 *dst++ = *src++; 2918 membar_producer(); 2919 break; 2920 } 2921 2922 case DOORBELL_UDBWC: 2923 *eq->udb = LE_32(V_QID(eq->udb_qid) | val); 2924 membar_producer(); 2925 break; 2926 2927 case DOORBELL_KDB: 2928 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), 2929 V_QID(eq->cntxt_id) | val); 2930 break; 2931 } 2932 2933 eq->pending = 0; 2934 } 2935 2936 static int 2937 reclaim_tx_descs(struct sge_txq *txq, int howmany) 2938 { 2939 struct tx_sdesc *txsd; 2940 uint_t cidx, can_reclaim, reclaimed, txb_freed, hdls_freed; 2941 struct sge_eq *eq = &txq->eq; 2942 2943 EQ_LOCK_ASSERT_OWNED(eq); 2944 2945 cidx = eq->spg->cidx; /* stable snapshot */ 2946 cidx = be16_to_cpu(cidx); 2947 2948 if (cidx >= eq->cidx) 2949 can_reclaim = cidx - eq->cidx; 2950 else 2951 can_reclaim = cidx + eq->cap - eq->cidx; 2952 2953 if (can_reclaim == 0) 2954 return (0); 2955 2956 txb_freed = hdls_freed = reclaimed = 0; 2957 do { 2958 int ndesc; 2959 2960 txsd = &txq->sdesc[eq->cidx]; 2961 ndesc = txsd->desc_used; 2962 2963 /* Firmware doesn't return "partial" credits. */ 2964 ASSERT(can_reclaim >= ndesc); 2965 2966 /* 2967 * We always keep mblk around, even for immediate data. If mblk 2968 * is NULL, this has to be the software descriptor for a credit 2969 * flush work request. 2970 */ 2971 if (txsd->m != NULL) 2972 freemsgchain(txsd->m); 2973 #ifdef DEBUG 2974 else { 2975 ASSERT(txsd->txb_used == 0); 2976 ASSERT(txsd->hdls_used == 0); 2977 ASSERT(ndesc == 1); 2978 } 2979 #endif 2980 2981 txb_freed += txsd->txb_used; 2982 hdls_freed += txsd->hdls_used; 2983 reclaimed += ndesc; 2984 2985 eq->cidx += ndesc; 2986 if (eq->cidx >= eq->cap) 2987 eq->cidx -= eq->cap; 2988 2989 can_reclaim -= ndesc; 2990 2991 } while (can_reclaim && reclaimed < howmany); 2992 2993 eq->avail += reclaimed; 2994 ASSERT(eq->avail < eq->cap); /* avail tops out at (cap - 1) */ 2995 2996 txq->txb_avail += txb_freed; 2997 2998 txq->tx_dhdl_avail += hdls_freed; 2999 ASSERT(txq->tx_dhdl_avail <= txq->tx_dhdl_total); 3000 for (; hdls_freed; hdls_freed--) { 3001 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_cidx]); 3002 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total) 3003 txq->tx_dhdl_cidx = 0; 3004 } 3005 3006 return (reclaimed); 3007 } 3008 3009 static void 3010 write_txqflush_wr(struct sge_txq *txq) 3011 { 3012 struct sge_eq *eq = &txq->eq; 3013 struct fw_eq_flush_wr *wr; 3014 struct tx_sdesc *txsd; 3015 3016 EQ_LOCK_ASSERT_OWNED(eq); 3017 ASSERT(eq->avail > 0); 3018 3019 wr = (void *)&eq->desc[eq->pidx]; 3020 bzero(wr, sizeof (*wr)); 3021 wr->opcode = FW_EQ_FLUSH_WR; 3022 wr->equiq_to_len16 = cpu_to_be32(V_FW_WR_LEN16(sizeof (*wr) / 16) | 3023 F_FW_WR_EQUEQ | F_FW_WR_EQUIQ); 3024 3025 txsd = &txq->sdesc[eq->pidx]; 3026 txsd->m = NULL; 3027 txsd->txb_used = 0; 3028 txsd->hdls_used = 0; 3029 txsd->desc_used = 1; 3030 3031 eq->pending++; 3032 eq->avail--; 3033 if (++eq->pidx == eq->cap) 3034 eq->pidx = 0; 3035 } 3036 3037 static int 3038 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m) 3039 { 3040 struct sge_rxq *rxq = (void *)iq; 3041 struct mblk_pair chain = {0}; 3042 const struct cpl_rx_pkt *cpl = (const void *)(rss + 1); 3043 3044 iq->intr_next = iq->intr_params; 3045 3046 m->b_rptr += FL_PKTSHIFT; 3047 3048 /* TODO: what about cpl->ip_frag? */ 3049 if (cpl->csum_calc && !cpl->err_vec && !cpl->ip_frag) { 3050 mac_hcksum_set(m, 0, 0, 0, 0xffff, 3051 HCK_FULLCKSUM_OK | HCK_FULLCKSUM | 3052 HCK_IPV4_HDRCKSUM_OK); 3053 rxq->rxcsum++; 3054 } 3055 3056 /* Add to the chain that we'll send up */ 3057 if (chain.head != NULL) 3058 chain.tail->b_next = m; 3059 else 3060 chain.head = m; 3061 chain.tail = m; 3062 3063 t4_mac_rx(rxq->port, rxq, chain.head); 3064 3065 return (0); 3066 } 3067 3068 #define FL_HW_IDX(idx) ((idx) >> 3) 3069 3070 static inline void 3071 ring_fl_db(struct adapter *sc, struct sge_fl *fl) 3072 { 3073 int desc_start, desc_last, ndesc; 3074 uint32_t v; 3075 3076 ndesc = FL_HW_IDX(fl->pending); 3077 3078 /* Hold back one credit if pidx = cidx */ 3079 if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx)) 3080 ndesc--; 3081 3082 /* 3083 * There are chances of ndesc modified above (to avoid pidx = cidx). 3084 * If there is nothing to post, return. 3085 */ 3086 if (ndesc <= 0) 3087 return; 3088 3089 desc_last = FL_HW_IDX(fl->pidx); 3090 3091 if (fl->pidx < fl->pending) { 3092 /* There was a wrap */ 3093 desc_start = FL_HW_IDX(fl->pidx + fl->cap - fl->pending); 3094 3095 /* From desc_start to the end of list */ 3096 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 0, 3097 DDI_DMA_SYNC_FORDEV); 3098 3099 /* From start of list to the desc_last */ 3100 if (desc_last != 0) 3101 (void) ddi_dma_sync(fl->dhdl, 0, desc_last * 3102 RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV); 3103 } else { 3104 /* There was no wrap, sync from start_desc to last_desc */ 3105 desc_start = FL_HW_IDX(fl->pidx - fl->pending); 3106 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 3107 ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV); 3108 } 3109 3110 if (is_t4(sc->params.chip)) 3111 v = V_PIDX(ndesc); 3112 else 3113 v = V_PIDX_T5(ndesc) | F_DBTYPE; 3114 v |= F_DBPRIO | V_QID(fl->cntxt_id) | V_PIDX(ndesc); 3115 3116 membar_producer(); 3117 3118 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v); 3119 3120 /* 3121 * Update pending count: 3122 * Deduct the number of descriptors posted 3123 */ 3124 fl->pending -= ndesc * 8; 3125 } 3126 3127 /* ARGSUSED */ 3128 static int 3129 handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m) 3130 { 3131 struct adapter *sc = iq->adapter; 3132 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1); 3133 3134 ASSERT(m == NULL); 3135 3136 if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) { 3137 const struct rss_header *rss2; 3138 3139 rss2 = (const struct rss_header *)&cpl->data[0]; 3140 return (sc->cpl_handler[rss2->opcode](iq, rss2, m)); 3141 } 3142 return (sc->fw_msg_handler[cpl->type](sc, &cpl->data[0])); 3143 } 3144 3145 int 3146 t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count, 3147 int flags) 3148 { 3149 int i, rc; 3150 3151 txmaps->map_total = count; 3152 txmaps->map_avail = txmaps->map_cidx = txmaps->map_pidx = 0; 3153 3154 txmaps->map = kmem_zalloc(sizeof (ddi_dma_handle_t) * 3155 txmaps->map_total, flags); 3156 3157 for (i = 0; i < count; i++) { 3158 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx, 3159 DDI_DMA_SLEEP, 0, &txmaps->map[i]); 3160 if (rc != DDI_SUCCESS) { 3161 cxgb_printf(sc->dip, CE_WARN, 3162 "%s: failed to allocate DMA handle (%d)", 3163 __func__, rc); 3164 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 3165 } 3166 txmaps->map_avail++; 3167 } 3168 3169 return (0); 3170 } 3171 3172 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG) 3173 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR) 3174 #define KS_U_SET(x, y) kstatp->x.value.ul = (y) 3175 #define KS_U_FROM(x, y) kstatp->x.value.ul = (y)->x 3176 #define KS_C_SET(x, ...) \ 3177 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__) 3178 3179 /* 3180 * cxgbe:X:config 3181 */ 3182 struct cxgbe_port_config_kstats { 3183 kstat_named_t idx; 3184 kstat_named_t nrxq; 3185 kstat_named_t ntxq; 3186 kstat_named_t first_rxq; 3187 kstat_named_t first_txq; 3188 kstat_named_t controller; 3189 kstat_named_t factory_mac_address; 3190 }; 3191 3192 /* 3193 * cxgbe:X:info 3194 */ 3195 struct cxgbe_port_info_kstats { 3196 kstat_named_t transceiver; 3197 kstat_named_t rx_ovflow0; 3198 kstat_named_t rx_ovflow1; 3199 kstat_named_t rx_ovflow2; 3200 kstat_named_t rx_ovflow3; 3201 kstat_named_t rx_trunc0; 3202 kstat_named_t rx_trunc1; 3203 kstat_named_t rx_trunc2; 3204 kstat_named_t rx_trunc3; 3205 kstat_named_t tx_pause; 3206 kstat_named_t rx_pause; 3207 }; 3208 3209 static kstat_t * 3210 setup_port_config_kstats(struct port_info *pi) 3211 { 3212 kstat_t *ksp; 3213 struct cxgbe_port_config_kstats *kstatp; 3214 int ndata; 3215 dev_info_t *pdip = ddi_get_parent(pi->dip); 3216 uint8_t *ma = &pi->hw_addr[0]; 3217 3218 ndata = sizeof (struct cxgbe_port_config_kstats) / 3219 sizeof (kstat_named_t); 3220 3221 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "config", 3222 "net", KSTAT_TYPE_NAMED, ndata, 0); 3223 if (ksp == NULL) { 3224 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats."); 3225 return (NULL); 3226 } 3227 3228 kstatp = (struct cxgbe_port_config_kstats *)ksp->ks_data; 3229 3230 KS_UINIT(idx); 3231 KS_UINIT(nrxq); 3232 KS_UINIT(ntxq); 3233 KS_UINIT(first_rxq); 3234 KS_UINIT(first_txq); 3235 KS_CINIT(controller); 3236 KS_CINIT(factory_mac_address); 3237 3238 KS_U_SET(idx, pi->port_id); 3239 KS_U_SET(nrxq, pi->nrxq); 3240 KS_U_SET(ntxq, pi->ntxq); 3241 KS_U_SET(first_rxq, pi->first_rxq); 3242 KS_U_SET(first_txq, pi->first_txq); 3243 KS_C_SET(controller, "%s%d", ddi_driver_name(pdip), 3244 ddi_get_instance(pdip)); 3245 KS_C_SET(factory_mac_address, "%02X%02X%02X%02X%02X%02X", 3246 ma[0], ma[1], ma[2], ma[3], ma[4], ma[5]); 3247 3248 /* Do NOT set ksp->ks_update. These kstats do not change. */ 3249 3250 /* Install the kstat */ 3251 ksp->ks_private = (void *)pi; 3252 kstat_install(ksp); 3253 3254 return (ksp); 3255 } 3256 3257 static kstat_t * 3258 setup_port_info_kstats(struct port_info *pi) 3259 { 3260 kstat_t *ksp; 3261 struct cxgbe_port_info_kstats *kstatp; 3262 int ndata; 3263 3264 ndata = sizeof (struct cxgbe_port_info_kstats) / sizeof (kstat_named_t); 3265 3266 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "info", 3267 "net", KSTAT_TYPE_NAMED, ndata, 0); 3268 if (ksp == NULL) { 3269 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats."); 3270 return (NULL); 3271 } 3272 3273 kstatp = (struct cxgbe_port_info_kstats *)ksp->ks_data; 3274 3275 KS_CINIT(transceiver); 3276 KS_UINIT(rx_ovflow0); 3277 KS_UINIT(rx_ovflow1); 3278 KS_UINIT(rx_ovflow2); 3279 KS_UINIT(rx_ovflow3); 3280 KS_UINIT(rx_trunc0); 3281 KS_UINIT(rx_trunc1); 3282 KS_UINIT(rx_trunc2); 3283 KS_UINIT(rx_trunc3); 3284 KS_UINIT(tx_pause); 3285 KS_UINIT(rx_pause); 3286 3287 /* Install the kstat */ 3288 ksp->ks_update = update_port_info_kstats; 3289 ksp->ks_private = (void *)pi; 3290 kstat_install(ksp); 3291 3292 return (ksp); 3293 } 3294 3295 static int 3296 update_port_info_kstats(kstat_t *ksp, int rw) 3297 { 3298 struct cxgbe_port_info_kstats *kstatp = 3299 (struct cxgbe_port_info_kstats *)ksp->ks_data; 3300 struct port_info *pi = ksp->ks_private; 3301 static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX", 3302 "active TWINAX", "LRM" }; 3303 uint32_t bgmap; 3304 3305 if (rw == KSTAT_WRITE) 3306 return (0); 3307 3308 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 3309 KS_C_SET(transceiver, "unplugged"); 3310 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 3311 KS_C_SET(transceiver, "unknown"); 3312 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 3313 KS_C_SET(transceiver, "unsupported"); 3314 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str)) 3315 KS_C_SET(transceiver, "%s", mod_str[pi->mod_type]); 3316 else 3317 KS_C_SET(transceiver, "type %d", pi->mod_type); 3318 3319 #define GET_STAT(name) t4_read_reg64(pi->adapter, \ 3320 PORT_REG(pi->port_id, A_MPS_PORT_STAT_##name##_L)) 3321 #define GET_STAT_COM(name) t4_read_reg64(pi->adapter, \ 3322 A_MPS_STAT_##name##_L) 3323 3324 bgmap = G_NUMPORTS(t4_read_reg(pi->adapter, A_MPS_CMN_CTL)); 3325 if (bgmap == 0) 3326 bgmap = (pi->port_id == 0) ? 0xf : 0; 3327 else if (bgmap == 1) 3328 bgmap = (pi->port_id < 2) ? (3 << (2 * pi->port_id)) : 0; 3329 else 3330 bgmap = 1; 3331 3332 KS_U_SET(rx_ovflow0, (bgmap & 1) ? 3333 GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0); 3334 KS_U_SET(rx_ovflow1, (bgmap & 2) ? 3335 GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0); 3336 KS_U_SET(rx_ovflow2, (bgmap & 4) ? 3337 GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0); 3338 KS_U_SET(rx_ovflow3, (bgmap & 8) ? 3339 GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0); 3340 KS_U_SET(rx_trunc0, (bgmap & 1) ? 3341 GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0); 3342 KS_U_SET(rx_trunc1, (bgmap & 2) ? 3343 GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0); 3344 KS_U_SET(rx_trunc2, (bgmap & 4) ? 3345 GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0); 3346 KS_U_SET(rx_trunc3, (bgmap & 8) ? 3347 GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0); 3348 3349 KS_U_SET(tx_pause, GET_STAT(TX_PORT_PAUSE)); 3350 KS_U_SET(rx_pause, GET_STAT(RX_PORT_PAUSE)); 3351 3352 return (0); 3353 3354 } 3355 3356 /* 3357 * cxgbe:X:rxqY 3358 */ 3359 struct rxq_kstats { 3360 kstat_named_t rxcsum; 3361 kstat_named_t nomem; 3362 }; 3363 3364 static kstat_t * 3365 setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, int idx) 3366 { 3367 struct kstat *ksp; 3368 struct rxq_kstats *kstatp; 3369 int ndata; 3370 char str[16]; 3371 3372 ndata = sizeof (struct rxq_kstats) / sizeof (kstat_named_t); 3373 (void) snprintf(str, sizeof (str), "rxq%u", idx); 3374 3375 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "rxq", 3376 KSTAT_TYPE_NAMED, ndata, 0); 3377 if (ksp == NULL) { 3378 cxgb_printf(pi->dip, CE_WARN, 3379 "%s: failed to initialize rxq kstats for queue %d.", 3380 __func__, idx); 3381 return (NULL); 3382 } 3383 3384 kstatp = (struct rxq_kstats *)ksp->ks_data; 3385 3386 KS_UINIT(rxcsum); 3387 KS_UINIT(nomem); 3388 3389 ksp->ks_update = update_rxq_kstats; 3390 ksp->ks_private = (void *)rxq; 3391 kstat_install(ksp); 3392 3393 return (ksp); 3394 } 3395 3396 static int 3397 update_rxq_kstats(kstat_t *ksp, int rw) 3398 { 3399 struct rxq_kstats *kstatp = (struct rxq_kstats *)ksp->ks_data; 3400 struct sge_rxq *rxq = ksp->ks_private; 3401 3402 if (rw == KSTAT_WRITE) 3403 return (0); 3404 3405 KS_U_FROM(rxcsum, rxq); 3406 KS_U_FROM(nomem, rxq); 3407 3408 return (0); 3409 } 3410 3411 /* 3412 * cxgbe:X:txqY 3413 */ 3414 struct txq_kstats { 3415 kstat_named_t txcsum; 3416 kstat_named_t tso_wrs; 3417 kstat_named_t imm_wrs; 3418 kstat_named_t sgl_wrs; 3419 kstat_named_t txpkt_wrs; 3420 kstat_named_t txpkts_wrs; 3421 kstat_named_t txpkts_pkts; 3422 kstat_named_t txb_used; 3423 kstat_named_t hdl_used; 3424 kstat_named_t txb_full; 3425 kstat_named_t dma_hdl_failed; 3426 kstat_named_t dma_map_failed; 3427 kstat_named_t qfull; 3428 kstat_named_t qflush; 3429 kstat_named_t pullup_early; 3430 kstat_named_t pullup_late; 3431 kstat_named_t pullup_failed; 3432 }; 3433 3434 static kstat_t * 3435 setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, int idx) 3436 { 3437 struct kstat *ksp; 3438 struct txq_kstats *kstatp; 3439 int ndata; 3440 char str[16]; 3441 3442 ndata = sizeof (struct txq_kstats) / sizeof (kstat_named_t); 3443 (void) snprintf(str, sizeof (str), "txq%u", idx); 3444 3445 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "txq", 3446 KSTAT_TYPE_NAMED, ndata, 0); 3447 if (ksp == NULL) { 3448 cxgb_printf(pi->dip, CE_WARN, 3449 "%s: failed to initialize txq kstats for queue %d.", 3450 __func__, idx); 3451 return (NULL); 3452 } 3453 3454 kstatp = (struct txq_kstats *)ksp->ks_data; 3455 3456 KS_UINIT(txcsum); 3457 KS_UINIT(tso_wrs); 3458 KS_UINIT(imm_wrs); 3459 KS_UINIT(sgl_wrs); 3460 KS_UINIT(txpkt_wrs); 3461 KS_UINIT(txpkts_wrs); 3462 KS_UINIT(txpkts_pkts); 3463 KS_UINIT(txb_used); 3464 KS_UINIT(hdl_used); 3465 KS_UINIT(txb_full); 3466 KS_UINIT(dma_hdl_failed); 3467 KS_UINIT(dma_map_failed); 3468 KS_UINIT(qfull); 3469 KS_UINIT(qflush); 3470 KS_UINIT(pullup_early); 3471 KS_UINIT(pullup_late); 3472 KS_UINIT(pullup_failed); 3473 3474 ksp->ks_update = update_txq_kstats; 3475 ksp->ks_private = (void *)txq; 3476 kstat_install(ksp); 3477 3478 return (ksp); 3479 } 3480 3481 static int 3482 update_txq_kstats(kstat_t *ksp, int rw) 3483 { 3484 struct txq_kstats *kstatp = (struct txq_kstats *)ksp->ks_data; 3485 struct sge_txq *txq = ksp->ks_private; 3486 3487 if (rw == KSTAT_WRITE) 3488 return (0); 3489 3490 KS_U_FROM(txcsum, txq); 3491 KS_U_FROM(tso_wrs, txq); 3492 KS_U_FROM(imm_wrs, txq); 3493 KS_U_FROM(sgl_wrs, txq); 3494 KS_U_FROM(txpkt_wrs, txq); 3495 KS_U_FROM(txpkts_wrs, txq); 3496 KS_U_FROM(txpkts_pkts, txq); 3497 KS_U_FROM(txb_used, txq); 3498 KS_U_FROM(hdl_used, txq); 3499 KS_U_FROM(txb_full, txq); 3500 KS_U_FROM(dma_hdl_failed, txq); 3501 KS_U_FROM(dma_map_failed, txq); 3502 KS_U_FROM(qfull, txq); 3503 KS_U_FROM(qflush, txq); 3504 KS_U_FROM(pullup_early, txq); 3505 KS_U_FROM(pullup_late, txq); 3506 KS_U_FROM(pullup_failed, txq); 3507 3508 return (0); 3509 } 3510