1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * This file is part of the Chelsio T4 support code. 14 * 15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved. 16 * 17 * This program is distributed in the hope that it will be useful, but WITHOUT 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this 20 * release for licensing terms and conditions. 21 */ 22 23 #include <sys/ddi.h> 24 #include <sys/sunddi.h> 25 #include <sys/sunndi.h> 26 #include <sys/atomic.h> 27 #include <sys/dlpi.h> 28 #include <sys/pattr.h> 29 #include <sys/strsubr.h> 30 #include <sys/stream.h> 31 #include <sys/strsun.h> 32 #include <inet/ip.h> 33 #include <inet/tcp.h> 34 35 #include "version.h" 36 #include "common/common.h" 37 #include "common/t4_msg.h" 38 #include "common/t4_regs.h" 39 #include "common/t4_regs_values.h" 40 41 /* TODO: Tune. */ 42 int rx_buf_size = 8192; 43 int tx_copy_threshold = 256; 44 uint16_t rx_copy_threshold = 256; 45 46 enum { 47 SPG_SIZE = 64, /* Size of status page */ 48 FL_ALIGN = CACHE_LINE, /* packet buffer alignment in FL buffers */ 49 FL_PKTSHIFT = 2 /* payload is at this offset in packet buffer */ 50 }; 51 52 /* Used to track coalesced tx work request */ 53 struct txpkts { 54 mblk_t *tail; /* head is in the software descriptor */ 55 uint64_t *flitp; /* ptr to flit where next pkt should start */ 56 uint8_t npkt; /* # of packets in this work request */ 57 uint8_t nflits; /* # of flits used by this work request */ 58 uint16_t plen; /* total payload (sum of all packets) */ 59 }; 60 61 /* All information needed to tx a frame */ 62 struct txinfo { 63 uint32_t len; /* Total length of frame */ 64 uint32_t flags; /* Checksum and LSO flags */ 65 uint32_t mss; /* MSS for LSO */ 66 uint8_t nsegs; /* # of segments in the SGL, 0 means imm. tx */ 67 uint8_t nflits; /* # of flits needed for the SGL */ 68 uint8_t hdls_used; /* # of DMA handles used */ 69 uint32_t txb_used; /* txb_space used */ 70 struct ulptx_sgl sgl __attribute__((aligned(8))); 71 struct ulptx_sge_pair reserved[TX_SGL_SEGS / 2]; 72 }; 73 74 static int service_iq(struct sge_iq *iq, int budget); 75 static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, 76 int8_t pktc_idx, int qsize, uint8_t esize); 77 static inline void init_fl(struct sge_fl *fl, uint16_t qsize); 78 static inline void init_eq(struct sge_eq *eq, uint16_t eqtype, uint16_t qsize, 79 uint8_t tx_chan, uint16_t iqid); 80 static int alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, 81 struct sge_fl *fl, int intr_idx, int cong); 82 static int free_iq_fl(struct port_info *pi, struct sge_iq *iq, 83 struct sge_fl *fl); 84 static int alloc_fwq(struct adapter *sc); 85 static int free_fwq(struct adapter *sc); 86 static int alloc_mgmtq(struct adapter *sc); 87 static int alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, 88 int i); 89 static int free_rxq(struct port_info *pi, struct sge_rxq *rxq); 90 #ifndef TCP_OFFLOAD_DISABLE 91 static int alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq, 92 int intr_idx); 93 static int free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq); 94 #endif 95 static int ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq); 96 static int eth_eq_alloc(struct adapter *sc, struct port_info *pi, 97 struct sge_eq *eq); 98 #ifndef TCP_OFFLOAD_DISABLE 99 static int ofld_eq_alloc(struct adapter *sc, struct port_info *pi, 100 struct sge_eq *eq); 101 #endif 102 static int alloc_eq(struct adapter *sc, struct port_info *pi, 103 struct sge_eq *eq); 104 static int free_eq(struct adapter *sc, struct sge_eq *eq); 105 static int alloc_wrq(struct adapter *sc, struct port_info *pi, 106 struct sge_wrq *wrq, int idx); 107 static int free_wrq(struct adapter *sc, struct sge_wrq *wrq); 108 static int alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx); 109 static int free_txq(struct port_info *pi, struct sge_txq *txq); 110 static int alloc_dma_memory(struct adapter *sc, size_t len, int flags, 111 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr, 112 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 113 caddr_t *pva); 114 static int free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl); 115 static int alloc_desc_ring(struct adapter *sc, size_t len, int rw, 116 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 117 caddr_t *pva); 118 static int free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl); 119 static int alloc_tx_copybuffer(struct adapter *sc, size_t len, 120 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 121 caddr_t *pva); 122 static inline bool is_new_response(const struct sge_iq *iq, 123 struct rsp_ctrl **ctrl); 124 static inline void iq_next(struct sge_iq *iq); 125 static int refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs); 126 static void refill_sfl(void *arg); 127 static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl); 128 static void free_fl_bufs(struct sge_fl *fl); 129 static mblk_t *get_fl_payload(struct sge_fl *fl, uint32_t len_newbuf, 130 int *fl_bufs_used); 131 static int get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, 132 struct txinfo *txinfo, int sgl_only); 133 static inline int fits_in_txb(struct sge_txq *txq, int len, int *waste); 134 static inline int copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, 135 struct txinfo *txinfo); 136 static inline void add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len); 137 static inline int add_mblk(struct sge_txq *txq, struct txinfo *txinfo, 138 mblk_t *m, int len); 139 static void free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo); 140 static int add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m, 141 struct txinfo *txinfo); 142 static void write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts); 143 static int write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m, 144 struct txinfo *txinfo); 145 static inline void write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq, 146 struct txpkts *txpkts, struct txinfo *txinfo); 147 static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, 148 int len); 149 static inline void ring_tx_db(struct adapter *sc, struct sge_eq *eq); 150 static int reclaim_tx_descs(struct sge_txq *txq, int howmany); 151 static void write_txqflush_wr(struct sge_txq *txq); 152 static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, 153 mblk_t *m); 154 static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl); 155 static kstat_t *setup_port_config_kstats(struct port_info *pi); 156 static kstat_t *setup_port_info_kstats(struct port_info *pi); 157 static kstat_t *setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, 158 int idx); 159 static int update_rxq_kstats(kstat_t *ksp, int rw); 160 static int update_port_info_kstats(kstat_t *ksp, int rw); 161 static kstat_t *setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, 162 int idx); 163 static int update_txq_kstats(kstat_t *ksp, int rw); 164 static int handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, 165 mblk_t *m); 166 167 static inline int 168 reclaimable(struct sge_eq *eq) 169 { 170 unsigned int cidx; 171 172 cidx = eq->spg->cidx; /* stable snapshot */ 173 cidx = be16_to_cpu(cidx); 174 175 if (cidx >= eq->cidx) 176 return (cidx - eq->cidx); 177 else 178 return (cidx + eq->cap - eq->cidx); 179 } 180 181 void 182 t4_sge_init(struct adapter *sc) 183 { 184 struct driver_properties *p = &sc->props; 185 ddi_dma_attr_t *dma_attr; 186 ddi_device_acc_attr_t *acc_attr; 187 uint32_t v; 188 189 /* 190 * Device access and DMA attributes for descriptor rings 191 */ 192 acc_attr = &sc->sge.acc_attr_desc; 193 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 194 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 195 acc_attr->devacc_attr_dataorder = DDI_STRICTORDER_ACC; 196 197 dma_attr = &sc->sge.dma_attr_desc; 198 dma_attr->dma_attr_version = DMA_ATTR_V0; 199 dma_attr->dma_attr_addr_lo = 0; 200 dma_attr->dma_attr_addr_hi = UINT64_MAX; 201 dma_attr->dma_attr_count_max = UINT64_MAX; 202 dma_attr->dma_attr_align = 512; 203 dma_attr->dma_attr_burstsizes = 0xfff; 204 dma_attr->dma_attr_minxfer = 1; 205 dma_attr->dma_attr_maxxfer = UINT64_MAX; 206 dma_attr->dma_attr_seg = UINT64_MAX; 207 dma_attr->dma_attr_sgllen = 1; 208 dma_attr->dma_attr_granular = 1; 209 dma_attr->dma_attr_flags = 0; 210 211 /* 212 * Device access and DMA attributes for tx buffers 213 */ 214 acc_attr = &sc->sge.acc_attr_tx; 215 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 216 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 217 218 dma_attr = &sc->sge.dma_attr_tx; 219 dma_attr->dma_attr_version = DMA_ATTR_V0; 220 dma_attr->dma_attr_addr_lo = 0; 221 dma_attr->dma_attr_addr_hi = UINT64_MAX; 222 dma_attr->dma_attr_count_max = UINT64_MAX; 223 dma_attr->dma_attr_align = 1; 224 dma_attr->dma_attr_burstsizes = 0xfff; 225 dma_attr->dma_attr_minxfer = 1; 226 dma_attr->dma_attr_maxxfer = UINT64_MAX; 227 dma_attr->dma_attr_seg = UINT64_MAX; 228 dma_attr->dma_attr_sgllen = TX_SGL_SEGS; 229 dma_attr->dma_attr_granular = 1; 230 dma_attr->dma_attr_flags = 0; 231 232 /* 233 * Device access and DMA attributes for rx buffers 234 */ 235 sc->sge.rxb_params.dip = sc->dip; 236 sc->sge.rxb_params.buf_size = rx_buf_size; 237 238 acc_attr = &sc->sge.rxb_params.acc_attr_rx; 239 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 240 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 241 242 dma_attr = &sc->sge.rxb_params.dma_attr_rx; 243 dma_attr->dma_attr_version = DMA_ATTR_V0; 244 dma_attr->dma_attr_addr_lo = 0; 245 dma_attr->dma_attr_addr_hi = UINT64_MAX; 246 dma_attr->dma_attr_count_max = UINT64_MAX; 247 /* 248 * Low 4 bits of an rx buffer address have a special meaning to the SGE 249 * and an rx buf cannot have an address with any of these bits set. 250 * FL_ALIGN is >= 32 so we're sure things are ok. 251 */ 252 dma_attr->dma_attr_align = FL_ALIGN; 253 dma_attr->dma_attr_burstsizes = 0xfff; 254 dma_attr->dma_attr_minxfer = 1; 255 dma_attr->dma_attr_maxxfer = UINT64_MAX; 256 dma_attr->dma_attr_seg = UINT64_MAX; 257 dma_attr->dma_attr_sgllen = 1; 258 dma_attr->dma_attr_granular = 1; 259 dma_attr->dma_attr_flags = 0; 260 261 sc->sge.rxbuf_cache = rxbuf_cache_create(&sc->sge.rxb_params); 262 263 v = t4_read_reg(sc, A_SGE_CONM_CTRL); 264 sc->sge.fl_starve_threshold = G_EGRTHRESHOLD(v) * 2 + 1; 265 266 t4_set_reg_field(sc, A_SGE_CONTROL, V_PKTSHIFT(M_PKTSHIFT) | 267 V_INGPADBOUNDARY(M_INGPADBOUNDARY) | 268 F_EGRSTATUSPAGESIZE, V_INGPADBOUNDARY(ilog2(FL_ALIGN) - 5) | 269 V_PKTSHIFT(FL_PKTSHIFT) | 270 F_RXPKTCPLMODE | 271 V_EGRSTATUSPAGESIZE(SPG_SIZE == 128)); 272 273 t4_set_reg_field(sc, A_SGE_HOST_PAGE_SIZE, 274 V_HOSTPAGESIZEPF0(M_HOSTPAGESIZEPF0), 275 V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10)); 276 277 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, rx_buf_size); 278 279 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, 280 V_THRESHOLD_0(p->counter_val[0]) | 281 V_THRESHOLD_1(p->counter_val[1]) | 282 V_THRESHOLD_2(p->counter_val[2]) | 283 V_THRESHOLD_3(p->counter_val[3])); 284 285 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, 286 V_TIMERVALUE0(us_to_core_ticks(sc, p->timer_val[0])) | 287 V_TIMERVALUE1(us_to_core_ticks(sc, p->timer_val[1]))); 288 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, 289 V_TIMERVALUE2(us_to_core_ticks(sc, p->timer_val[2])) | 290 V_TIMERVALUE3(us_to_core_ticks(sc, p->timer_val[3]))); 291 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, 292 V_TIMERVALUE4(us_to_core_ticks(sc, p->timer_val[4])) | 293 V_TIMERVALUE5(us_to_core_ticks(sc, p->timer_val[5]))); 294 295 (void) t4_register_cpl_handler(sc, CPL_FW4_MSG, handle_fw_rpl); 296 (void) t4_register_cpl_handler(sc, CPL_FW6_MSG, handle_fw_rpl); 297 (void) t4_register_cpl_handler(sc, CPL_RX_PKT, t4_eth_rx); 298 } 299 300 /* 301 * Allocate and initialize the firmware event queue and the forwarded interrupt 302 * queues, if any. The adapter owns all these queues as they are not associated 303 * with any particular port. 304 * 305 * Returns errno on failure. Resources allocated up to that point may still be 306 * allocated. Caller is responsible for cleanup in case this function fails. 307 */ 308 int 309 t4_setup_adapter_queues(struct adapter *sc) 310 { 311 int rc; 312 313 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 314 315 /* 316 * Firmware event queue 317 */ 318 rc = alloc_fwq(sc); 319 if (rc != 0) 320 return (rc); 321 322 /* 323 * Management queue. This is just a control queue that uses the fwq as 324 * its associated iq. 325 */ 326 rc = alloc_mgmtq(sc); 327 328 return (rc); 329 } 330 331 /* 332 * Idempotent 333 */ 334 int 335 t4_teardown_adapter_queues(struct adapter *sc) 336 { 337 338 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 339 340 (void) free_fwq(sc); 341 342 return (0); 343 } 344 345 static inline int 346 first_vector(struct port_info *pi) 347 { 348 struct adapter *sc = pi->adapter; 349 int rc = T4_EXTRA_INTR, i; 350 351 if (sc->intr_count == 1) 352 return (0); 353 354 for_each_port(sc, i) { 355 struct port_info *p = sc->port[i]; 356 357 if (i == pi->port_id) 358 break; 359 360 #ifndef TCP_OFFLOAD_DISABLE 361 if (!(sc->flags & INTR_FWD)) 362 rc += p->nrxq + p->nofldrxq; 363 else 364 rc += max(p->nrxq, p->nofldrxq); 365 #else 366 /* 367 * Not compiled with offload support and intr_count > 1. Only 368 * NIC queues exist and they'd better be taking direct 369 * interrupts. 370 */ 371 KASSERT(!(sc->flags & INTR_FWD)); 372 rc += p->nrxq; 373 #endif 374 } 375 return (rc); 376 } 377 378 /* 379 * Given an arbitrary "index," come up with an iq that can be used by other 380 * queues (of this port) for interrupt forwarding, SGE egress updates, etc. 381 * The iq returned is guaranteed to be something that takes direct interrupts. 382 */ 383 static struct sge_iq * 384 port_intr_iq(struct port_info *pi, int idx) 385 { 386 struct adapter *sc = pi->adapter; 387 struct sge *s = &sc->sge; 388 struct sge_iq *iq = NULL; 389 390 if (sc->intr_count == 1) 391 return (&sc->sge.fwq); 392 393 #ifndef TCP_OFFLOAD_DISABLE 394 if (!(sc->flags & INTR_FWD)) { 395 idx %= pi->nrxq + pi->nofldrxq; 396 397 if (idx >= pi->nrxq) { 398 idx -= pi->nrxq; 399 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq; 400 } else 401 iq = &s->rxq[pi->first_rxq + idx].iq; 402 403 } else { 404 idx %= max(pi->nrxq, pi->nofldrxq); 405 406 if (pi->nrxq >= pi->nofldrxq) 407 iq = &s->rxq[pi->first_rxq + idx].iq; 408 else 409 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq; 410 } 411 #else 412 /* 413 * Not compiled with offload support and intr_count > 1. Only NIC 414 * queues exist and they'd better be taking direct interrupts. 415 */ 416 ASSERT(!(sc->flags & INTR_FWD)); 417 418 idx %= pi->nrxq; 419 iq = &s->rxq[pi->first_rxq + idx].iq; 420 #endif 421 422 return (iq); 423 } 424 425 int 426 t4_setup_port_queues(struct port_info *pi) 427 { 428 int rc = 0, i, intr_idx, j, iqid; 429 struct sge_rxq *rxq; 430 struct sge_txq *txq; 431 #ifndef TCP_OFFLOAD_DISABLE 432 struct sge_wrq *ctrlq; 433 struct sge_ofld_rxq *ofld_rxq; 434 struct sge_wrq *ofld_txq; 435 #endif 436 struct adapter *sc = pi->adapter; 437 struct driver_properties *p = &sc->props; 438 439 pi->ksp_config = setup_port_config_kstats(pi); 440 pi->ksp_info = setup_port_info_kstats(pi); 441 442 /* Interrupt vector to start from (when using multiple vectors) */ 443 intr_idx = first_vector(pi); 444 445 /* 446 * First pass over all rx queues (NIC and TOE): 447 * a) initialize iq and fl 448 * b) allocate queue iff it will take direct interrupts. 449 */ 450 451 for_each_rxq(pi, i, rxq) { 452 453 init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, p->qsize_rxq, 454 RX_IQ_ESIZE); 455 456 init_fl(&rxq->fl, p->qsize_rxq / 8); /* 8 bufs in each entry */ 457 458 #ifndef TCP_OFFLOAD_DISABLE 459 if ((!(sc->flags & INTR_FWD)) || 460 (sc->intr_count > 1 && pi->nrxq >= pi->nofldrxq)) 461 #else 462 if (!(sc->flags & INTR_FWD)) 463 #endif 464 { 465 rxq->iq.flags |= IQ_INTR; 466 rc = alloc_rxq(pi, rxq, intr_idx, i); 467 if (rc != 0) 468 goto done; 469 intr_idx++; 470 } 471 472 } 473 474 #ifndef TCP_OFFLOAD_DISABLE 475 for_each_ofld_rxq(pi, i, ofld_rxq) { 476 477 init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, 478 p->qsize_rxq, RX_IQ_ESIZE); 479 480 init_fl(&ofld_rxq->fl, p->qsize_rxq / 8); 481 482 if (!(sc->flags & INTR_FWD) || 483 (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) { 484 ofld_rxq->iq.flags = IQ_INTR; 485 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx); 486 if (rc != 0) 487 goto done; 488 489 intr_idx++; 490 } 491 } 492 #endif 493 494 /* 495 * Second pass over all rx queues (NIC and TOE). The queues forwarding 496 * their interrupts are allocated now. 497 */ 498 j = 0; 499 for_each_rxq(pi, i, rxq) { 500 if (rxq->iq.flags & IQ_INTR) 501 continue; 502 503 intr_idx = port_intr_iq(pi, j)->abs_id; 504 505 rc = alloc_rxq(pi, rxq, intr_idx, i); 506 if (rc != 0) 507 goto done; 508 j++; 509 } 510 511 #ifndef TCP_OFFLOAD_DISABLE 512 for_each_ofld_rxq(pi, i, ofld_rxq) { 513 if (ofld_rxq->iq.flags & IQ_INTR) 514 continue; 515 516 intr_idx = port_intr_iq(pi, j)->abs_id; 517 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx); 518 if (rc != 0) 519 goto done; 520 j++; 521 } 522 #endif 523 /* 524 * Now the tx queues. Only one pass needed. 525 */ 526 j = 0; 527 for_each_txq(pi, i, txq) { 528 uint16_t iqid; 529 530 iqid = port_intr_iq(pi, j)->cntxt_id; 531 init_eq(&txq->eq, EQ_ETH, p->qsize_txq, pi->tx_chan, iqid); 532 rc = alloc_txq(pi, txq, i); 533 if (rc != 0) 534 goto done; 535 } 536 537 #ifndef TCP_OFFLOAD_DISABLE 538 for_each_ofld_txq(pi, i, ofld_txq) { 539 uint16_t iqid; 540 541 iqid = port_intr_iq(pi, j)->cntxt_id; 542 init_eq(&ofld_txq->eq, EQ_OFLD, p->qsize_txq, pi->tx_chan, 543 iqid); 544 rc = alloc_wrq(sc, pi, ofld_txq, i); 545 if (rc != 0) 546 goto done; 547 } 548 #endif 549 550 /* 551 * Finally, the control queue. 552 */ 553 ctrlq = &sc->sge.ctrlq[pi->port_id]; 554 iqid = port_intr_iq(pi, 0)->cntxt_id; 555 init_eq(&ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, iqid); 556 rc = alloc_wrq(sc, pi, ctrlq, 0); 557 558 done: 559 if (rc != 0) 560 (void) t4_teardown_port_queues(pi); 561 562 return (rc); 563 } 564 565 /* 566 * Idempotent 567 */ 568 int 569 t4_teardown_port_queues(struct port_info *pi) 570 { 571 int i; 572 struct sge_rxq *rxq; 573 struct sge_txq *txq; 574 struct adapter *sc = pi->adapter; 575 #ifndef TCP_OFFLOAD_DISABLE 576 struct sge_ofld_rxq *ofld_rxq; 577 struct sge_wrq *ofld_txq; 578 #endif 579 580 if (pi->ksp_config != NULL) { 581 kstat_delete(pi->ksp_config); 582 pi->ksp_config = NULL; 583 } 584 if (pi->ksp_info != NULL) { 585 kstat_delete(pi->ksp_info); 586 pi->ksp_info = NULL; 587 } 588 589 (void) free_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 590 591 for_each_txq(pi, i, txq) { 592 (void) free_txq(pi, txq); 593 } 594 595 #ifndef TCP_OFFLOAD_DISABLE 596 for_each_ofld_txq(pi, i, ofld_txq) { 597 (void) free_wrq(sc, ofld_txq); 598 } 599 600 for_each_ofld_rxq(pi, i, ofld_rxq) { 601 if ((ofld_rxq->iq.flags & IQ_INTR) == 0) 602 (void) free_ofld_rxq(pi, ofld_rxq); 603 } 604 #endif 605 606 for_each_rxq(pi, i, rxq) { 607 if ((rxq->iq.flags & IQ_INTR) == 0) 608 (void) free_rxq(pi, rxq); 609 } 610 611 /* 612 * Then take down the rx queues that take direct interrupts. 613 */ 614 615 for_each_rxq(pi, i, rxq) { 616 if (rxq->iq.flags & IQ_INTR) 617 (void) free_rxq(pi, rxq); 618 } 619 620 #ifndef TCP_OFFLOAD_DISABLE 621 for_each_ofld_rxq(pi, i, ofld_rxq) { 622 if (ofld_rxq->iq.flags & IQ_INTR) 623 (void) free_ofld_rxq(pi, ofld_rxq); 624 } 625 #endif 626 627 return (0); 628 } 629 630 /* Deals with errors and forwarded interrupts */ 631 uint_t 632 t4_intr_all(caddr_t arg1, caddr_t arg2) 633 { 634 635 (void) t4_intr_err(arg1, arg2); 636 (void) t4_intr(arg1, arg2); 637 638 return (DDI_INTR_CLAIMED); 639 } 640 641 /* Deals with interrupts on the given ingress queue */ 642 /* ARGSUSED */ 643 uint_t 644 t4_intr(caddr_t arg1, caddr_t arg2) 645 { 646 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 647 struct sge_iq *iq = (struct sge_iq *)arg2; 648 649 if (atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY) == IQS_IDLE) { 650 (void) service_iq(iq, 0); 651 (void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE); 652 } 653 return (DDI_INTR_CLAIMED); 654 } 655 656 /* Deals with error interrupts */ 657 /* ARGSUSED */ 658 uint_t 659 t4_intr_err(caddr_t arg1, caddr_t arg2) 660 { 661 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 662 struct adapter *sc = (struct adapter *)arg1; 663 664 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0); 665 (void) t4_slow_intr_handler(sc); 666 667 return (DDI_INTR_CLAIMED); 668 } 669 670 /* 671 * Deals with anything and everything on the given ingress queue. 672 */ 673 static int 674 service_iq(struct sge_iq *iq, int budget) 675 { 676 struct sge_iq *q; 677 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */ 678 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */ 679 struct adapter *sc = iq->adapter; 680 struct rsp_ctrl *ctrl; 681 const struct rss_header *rss; 682 int ndescs = 0, limit, fl_bufs_used = 0; 683 int rsp_type; 684 uint32_t lq; 685 mblk_t *m; 686 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql); 687 688 limit = budget ? budget : iq->qsize / 8; 689 690 /* 691 * We always come back and check the descriptor ring for new indirect 692 * interrupts and other responses after running a single handler. 693 */ 694 for (;;) { 695 while (is_new_response(iq, &ctrl)) { 696 697 membar_consumer(); 698 699 m = NULL; 700 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen); 701 lq = be32_to_cpu(ctrl->pldbuflen_qid); 702 rss = (const void *)iq->cdesc; 703 704 switch (rsp_type) { 705 case X_RSPD_TYPE_FLBUF: 706 707 ASSERT(iq->flags & IQ_HAS_FL); 708 709 m = get_fl_payload(fl, lq, &fl_bufs_used); 710 if (m == NULL) { 711 panic("%s: line %d.", __func__, 712 __LINE__); 713 } 714 715 /* FALLTHRU */ 716 case X_RSPD_TYPE_CPL: 717 718 ASSERT(rss->opcode < NUM_CPL_CMDS); 719 sc->cpl_handler[rss->opcode](iq, rss, m); 720 break; 721 722 case X_RSPD_TYPE_INTR: 723 724 /* 725 * Interrupts should be forwarded only to queues 726 * that are not forwarding their interrupts. 727 * This means service_iq can recurse but only 1 728 * level deep. 729 */ 730 ASSERT(budget == 0); 731 732 q = sc->sge.iqmap[lq - sc->sge.iq_start]; 733 if (atomic_cas_uint(&q->state, IQS_IDLE, 734 IQS_BUSY) == IQS_IDLE) { 735 if (service_iq(q, q->qsize / 8) == 0) { 736 (void) atomic_cas_uint( 737 &q->state, IQS_BUSY, 738 IQS_IDLE); 739 } else { 740 STAILQ_INSERT_TAIL(&iql, q, 741 link); 742 } 743 } 744 break; 745 746 default: 747 break; 748 } 749 750 iq_next(iq); 751 if (++ndescs == limit) { 752 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 753 V_CIDXINC(ndescs) | 754 V_INGRESSQID(iq->cntxt_id) | 755 V_SEINTARM(V_QINTR_TIMER_IDX( 756 X_TIMERREG_UPDATE_CIDX))); 757 ndescs = 0; 758 759 if (fl_bufs_used > 0) { 760 ASSERT(iq->flags & IQ_HAS_FL); 761 FL_LOCK(fl); 762 fl->needed += fl_bufs_used; 763 (void) refill_fl(sc, fl, fl->cap / 8); 764 FL_UNLOCK(fl); 765 fl_bufs_used = 0; 766 } 767 768 if (budget != 0) 769 return (EINPROGRESS); 770 } 771 } 772 773 if (STAILQ_EMPTY(&iql) != 0) 774 break; 775 776 /* 777 * Process the head only, and send it to the back of the list if 778 * it's still not done. 779 */ 780 q = STAILQ_FIRST(&iql); 781 STAILQ_REMOVE_HEAD(&iql, link); 782 if (service_iq(q, q->qsize / 8) == 0) 783 (void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE); 784 else 785 STAILQ_INSERT_TAIL(&iql, q, link); 786 } 787 788 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) | 789 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next)); 790 791 if (iq->flags & IQ_HAS_FL) { 792 int starved; 793 794 FL_LOCK(fl); 795 fl->needed += fl_bufs_used; 796 starved = refill_fl(sc, fl, fl->cap / 4); 797 FL_UNLOCK(fl); 798 if (starved != 0) 799 add_fl_to_sfl(sc, fl); 800 } 801 802 return (0); 803 } 804 805 int 806 t4_mgmt_tx(struct adapter *sc, mblk_t *m) 807 { 808 return (t4_wrq_tx(sc, &sc->sge.mgmtq, m)); 809 } 810 811 /* 812 * Doesn't fail. Holds on to work requests it can't send right away. 813 */ 814 int 815 t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, mblk_t *m0) 816 { 817 struct sge_eq *eq = &wrq->eq; 818 struct mblk_pair *wr_list = &wrq->wr_list; 819 int can_reclaim; 820 caddr_t dst; 821 mblk_t *wr, *next; 822 823 TXQ_LOCK_ASSERT_OWNED(wrq); 824 #ifndef TCP_OFFLOAD_DISABLE 825 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_OFLD || 826 (eq->flags & EQ_TYPEMASK) == EQ_CTRL); 827 #else 828 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_CTRL); 829 #endif 830 831 if (m0 != NULL) { 832 if (wr_list->head != NULL) 833 wr_list->tail->b_next = m0; 834 else 835 wr_list->head = m0; 836 while (m0->b_next) 837 m0 = m0->b_next; 838 wr_list->tail = m0; 839 } 840 841 can_reclaim = reclaimable(eq); 842 eq->cidx += can_reclaim; 843 eq->avail += can_reclaim; 844 if (eq->cidx >= eq->cap) 845 eq->cidx -= eq->cap; 846 847 for (wr = wr_list->head; wr; wr = next) { 848 int ndesc, len = 0; 849 mblk_t *m; 850 851 next = wr->b_next; 852 wr->b_next = NULL; 853 854 for (m = wr; m; m = m->b_cont) 855 len += MBLKL(m); 856 857 ASSERT(len > 0 && (len & 0x7) == 0); 858 ASSERT(len <= SGE_MAX_WR_LEN); 859 860 ndesc = howmany(len, EQ_ESIZE); 861 if (eq->avail < ndesc) { 862 wr->b_next = next; 863 wrq->no_desc++; 864 break; 865 } 866 867 dst = (void *)&eq->desc[eq->pidx]; 868 for (m = wr; m; m = m->b_cont) 869 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m)); 870 871 eq->pidx += ndesc; 872 eq->avail -= ndesc; 873 if (eq->pidx >= eq->cap) 874 eq->pidx -= eq->cap; 875 876 eq->pending += ndesc; 877 if (eq->pending > 16) 878 ring_tx_db(sc, eq); 879 880 wrq->tx_wrs++; 881 freemsg(wr); 882 883 if (eq->avail < 8) { 884 can_reclaim = reclaimable(eq); 885 eq->cidx += can_reclaim; 886 eq->avail += can_reclaim; 887 if (eq->cidx >= eq->cap) 888 eq->cidx -= eq->cap; 889 } 890 } 891 892 if (eq->pending != 0) 893 ring_tx_db(sc, eq); 894 895 if (wr == NULL) 896 wr_list->head = wr_list->tail = NULL; 897 else { 898 wr_list->head = wr; 899 900 ASSERT(wr_list->tail->b_next == NULL); 901 } 902 903 return (0); 904 } 905 906 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */ 907 #define TXPKTS_PKT_HDR ((\ 908 sizeof (struct ulp_txpkt) + \ 909 sizeof (struct ulptx_idata) + \ 910 sizeof (struct cpl_tx_pkt_core)) / 8) 911 912 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */ 913 #define TXPKTS_WR_HDR (\ 914 sizeof (struct fw_eth_tx_pkts_wr) / 8 + \ 915 TXPKTS_PKT_HDR) 916 917 /* Header of a tx WR, before SGL of first packet (in flits) */ 918 #define TXPKT_WR_HDR ((\ 919 sizeof (struct fw_eth_tx_pkt_wr) + \ 920 sizeof (struct cpl_tx_pkt_core)) / 8) 921 922 /* Header of a tx LSO WR, before SGL of first packet (in flits) */ 923 #define TXPKT_LSO_WR_HDR ((\ 924 sizeof (struct fw_eth_tx_pkt_wr) + \ 925 sizeof (struct cpl_tx_pkt_lso) + \ 926 sizeof (struct cpl_tx_pkt_core)) / 8) 927 928 mblk_t * 929 t4_eth_tx(struct port_info *pi, struct sge_txq *txq, mblk_t *frame) 930 { 931 struct adapter *sc = pi->adapter; 932 struct sge_eq *eq = &txq->eq; 933 mblk_t *next_frame; 934 int rc, coalescing; 935 struct txpkts txpkts; 936 struct txinfo txinfo; 937 938 txpkts.npkt = 0; /* indicates there's nothing in txpkts */ 939 coalescing = 0; 940 941 TXQ_LOCK(txq); 942 if (eq->avail < 8) 943 (void) reclaim_tx_descs(txq, 8); 944 for (; frame; frame = next_frame) { 945 946 if (eq->avail < 8) 947 break; 948 949 next_frame = frame->b_next; 950 frame->b_next = NULL; 951 952 if (next_frame != NULL) 953 coalescing = 1; 954 955 rc = get_frame_txinfo(txq, &frame, &txinfo, coalescing); 956 if (rc != 0) { 957 if (rc == ENOMEM) { 958 959 /* Short of resources, suspend tx */ 960 961 frame->b_next = next_frame; 962 break; 963 } 964 965 /* 966 * Unrecoverable error for this frame, throw it 967 * away and move on to the next. 968 */ 969 970 freemsg(frame); 971 continue; 972 } 973 974 if (coalescing != 0 && 975 add_to_txpkts(txq, &txpkts, frame, &txinfo) == 0) { 976 977 /* Successfully absorbed into txpkts */ 978 979 write_ulp_cpl_sgl(pi, txq, &txpkts, &txinfo); 980 goto doorbell; 981 } 982 983 /* 984 * We weren't coalescing to begin with, or current frame could 985 * not be coalesced (add_to_txpkts flushes txpkts if a frame 986 * given to it can't be coalesced). Either way there should be 987 * nothing in txpkts. 988 */ 989 ASSERT(txpkts.npkt == 0); 990 991 /* We're sending out individual frames now */ 992 coalescing = 0; 993 994 if (eq->avail < 8) 995 (void) reclaim_tx_descs(txq, 8); 996 rc = write_txpkt_wr(pi, txq, frame, &txinfo); 997 if (rc != 0) { 998 999 /* Short of hardware descriptors, suspend tx */ 1000 1001 /* 1002 * This is an unlikely but expensive failure. We've 1003 * done all the hard work (DMA bindings etc.) and now we 1004 * can't send out the frame. What's worse, we have to 1005 * spend even more time freeing up everything in txinfo. 1006 */ 1007 txq->qfull++; 1008 free_txinfo_resources(txq, &txinfo); 1009 1010 frame->b_next = next_frame; 1011 break; 1012 } 1013 1014 doorbell: 1015 /* Fewer and fewer doorbells as the queue fills up */ 1016 if (eq->pending >= (1 << (fls(eq->qsize - eq->avail) / 2))) 1017 ring_tx_db(sc, eq); 1018 (void) reclaim_tx_descs(txq, 32); 1019 } 1020 1021 if (txpkts.npkt > 0) 1022 write_txpkts_wr(txq, &txpkts); 1023 1024 /* 1025 * frame not NULL means there was an error but we haven't thrown it 1026 * away. This can happen when we're short of tx descriptors (qfull) or 1027 * maybe even DMA handles (dma_hdl_failed). Either way, a credit flush 1028 * and reclaim will get things going again. 1029 * 1030 * If eq->avail is already 0 we know a credit flush was requested in the 1031 * WR that reduced it to 0 so we don't need another flush (we don't have 1032 * any descriptor for a flush WR anyway, duh). 1033 */ 1034 if (frame && eq->avail > 0) 1035 write_txqflush_wr(txq); 1036 1037 if (eq->pending != 0) 1038 ring_tx_db(sc, eq); 1039 1040 (void) reclaim_tx_descs(txq, eq->qsize); 1041 TXQ_UNLOCK(txq); 1042 1043 return (frame); 1044 } 1045 1046 static inline void 1047 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int8_t pktc_idx, int 1048 qsize, uint8_t esize) 1049 { 1050 ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS); 1051 ASSERT(pktc_idx < SGE_NCOUNTERS); /* -ve is ok, means don't use */ 1052 1053 iq->flags = 0; 1054 iq->adapter = sc; 1055 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx); 1056 iq->intr_pktc_idx = SGE_NCOUNTERS - 1; 1057 if (pktc_idx >= 0) { 1058 iq->intr_params |= F_QINTR_CNT_EN; 1059 iq->intr_pktc_idx = pktc_idx; 1060 } 1061 iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */ 1062 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */ 1063 } 1064 1065 static inline void 1066 init_fl(struct sge_fl *fl, uint16_t qsize) 1067 { 1068 1069 fl->qsize = qsize; 1070 } 1071 1072 static inline void 1073 init_eq(struct sge_eq *eq, uint16_t eqtype, uint16_t qsize, uint8_t tx_chan, 1074 uint16_t iqid) 1075 { 1076 ASSERT(tx_chan < NCHAN); 1077 ASSERT(eqtype <= EQ_TYPEMASK); 1078 1079 eq->flags = eqtype & EQ_TYPEMASK; 1080 eq->tx_chan = tx_chan; 1081 eq->iqid = iqid; 1082 eq->qsize = qsize; 1083 } 1084 1085 /* 1086 * Allocates the ring for an ingress queue and an optional freelist. If the 1087 * freelist is specified it will be allocated and then associated with the 1088 * ingress queue. 1089 * 1090 * Returns errno on failure. Resources allocated up to that point may still be 1091 * allocated. Caller is responsible for cleanup in case this function fails. 1092 * 1093 * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then 1094 * the intr_idx specifies the vector, starting from 0. Otherwise it specifies 1095 * the index of the queue to which its interrupts will be forwarded. 1096 */ 1097 static int 1098 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl, 1099 int intr_idx, int cong) 1100 { 1101 int rc, cntxt_id; 1102 size_t len; 1103 struct fw_iq_cmd c; 1104 struct adapter *sc = iq->adapter; 1105 uint32_t v = 0; 1106 1107 len = iq->qsize * iq->esize; 1108 rc = alloc_desc_ring(sc, len, DDI_DMA_READ, &iq->dhdl, &iq->ahdl, 1109 &iq->ba, (caddr_t *)&iq->desc); 1110 if (rc != 0) 1111 return (rc); 1112 1113 bzero(&c, sizeof (c)); 1114 c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | 1115 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) | 1116 V_FW_IQ_CMD_VFN(0)); 1117 1118 c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART | 1119 FW_LEN16(c)); 1120 1121 /* Special handling for firmware event queue */ 1122 if (iq == &sc->sge.fwq) 1123 v |= F_FW_IQ_CMD_IQASYNCH; 1124 1125 if (iq->flags & IQ_INTR) 1126 ASSERT(intr_idx < sc->intr_count); 1127 else 1128 v |= F_FW_IQ_CMD_IQANDST; 1129 v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx); 1130 1131 c.type_to_iqandstindex = cpu_to_be32(v | 1132 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | 1133 V_FW_IQ_CMD_VIID(pi->viid) | 1134 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT)); 1135 c.iqdroprss_to_iqesize = cpu_to_be16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | 1136 F_FW_IQ_CMD_IQGTSMODE | 1137 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) | 1138 V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4)); 1139 c.iqsize = cpu_to_be16(iq->qsize); 1140 c.iqaddr = cpu_to_be64(iq->ba); 1141 if (cong >= 0) 1142 c.iqns_to_fl0congen = BE_32(F_FW_IQ_CMD_IQFLINTCONGEN); 1143 1144 if (fl != NULL) { 1145 mutex_init(&fl->lock, NULL, MUTEX_DRIVER, 1146 DDI_INTR_PRI(sc->intr_pri)); 1147 fl->flags |= FL_MTX; 1148 1149 len = fl->qsize * RX_FL_ESIZE; 1150 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &fl->dhdl, 1151 &fl->ahdl, &fl->ba, (caddr_t *)&fl->desc); 1152 if (rc != 0) 1153 return (rc); 1154 1155 /* Allocate space for one software descriptor per buffer. */ 1156 fl->cap = (fl->qsize - SPG_SIZE / RX_FL_ESIZE) * 8; 1157 fl->sdesc = kmem_zalloc(sizeof (struct fl_sdesc) * fl->cap, 1158 KM_SLEEP); 1159 fl->needed = fl->cap; 1160 fl->lowat = roundup(sc->sge.fl_starve_threshold, 8); 1161 1162 c.iqns_to_fl0congen |= 1163 cpu_to_be32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) | 1164 F_FW_IQ_CMD_FL0PACKEN | F_FW_IQ_CMD_FL0PADEN); 1165 if (cong >= 0) { 1166 c.iqns_to_fl0congen |= 1167 BE_32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) | 1168 F_FW_IQ_CMD_FL0CONGCIF | 1169 F_FW_IQ_CMD_FL0CONGEN); 1170 } 1171 c.fl0dcaen_to_fl0cidxfthresh = 1172 cpu_to_be16(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_64B) | 1173 V_FW_IQ_CMD_FL0FBMAX(X_FETCHBURSTMAX_512B)); 1174 c.fl0size = cpu_to_be16(fl->qsize); 1175 c.fl0addr = cpu_to_be64(fl->ba); 1176 } 1177 1178 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1179 if (rc != 0) { 1180 cxgb_printf(sc->dip, CE_WARN, 1181 "failed to create ingress queue: %d", rc); 1182 return (rc); 1183 } 1184 1185 iq->cdesc = iq->desc; 1186 iq->cidx = 0; 1187 iq->gen = 1; 1188 iq->intr_next = iq->intr_params; 1189 iq->adapter = sc; 1190 iq->cntxt_id = be16_to_cpu(c.iqid); 1191 iq->abs_id = be16_to_cpu(c.physiqid); 1192 iq->flags |= IQ_ALLOCATED; 1193 1194 cntxt_id = iq->cntxt_id - sc->sge.iq_start; 1195 if (cntxt_id >= sc->sge.niq) { 1196 panic("%s: iq->cntxt_id (%d) more than the max (%d)", __func__, 1197 cntxt_id, sc->sge.niq - 1); 1198 } 1199 sc->sge.iqmap[cntxt_id] = iq; 1200 1201 if (fl != NULL) { 1202 fl->cntxt_id = be16_to_cpu(c.fl0id); 1203 fl->pidx = fl->cidx = 0; 1204 fl->copy_threshold = rx_copy_threshold; 1205 1206 cntxt_id = fl->cntxt_id - sc->sge.eq_start; 1207 if (cntxt_id >= sc->sge.neq) { 1208 panic("%s: fl->cntxt_id (%d) more than the max (%d)", 1209 __func__, cntxt_id, sc->sge.neq - 1); 1210 } 1211 sc->sge.eqmap[cntxt_id] = (void *)fl; 1212 1213 FL_LOCK(fl); 1214 (void) refill_fl(sc, fl, fl->lowat); 1215 FL_UNLOCK(fl); 1216 1217 iq->flags |= IQ_HAS_FL; 1218 } 1219 1220 /* Enable IQ interrupts */ 1221 iq->state = IQS_IDLE; 1222 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) | 1223 V_INGRESSQID(iq->cntxt_id)); 1224 1225 return (0); 1226 } 1227 1228 static int 1229 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl) 1230 { 1231 int rc; 1232 struct adapter *sc = iq->adapter; 1233 dev_info_t *dip; 1234 1235 dip = pi ? pi->dip : sc->dip; 1236 1237 if (iq != NULL) { 1238 if (iq->flags & IQ_ALLOCATED) { 1239 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, 1240 FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id, 1241 fl ? fl->cntxt_id : 0xffff, 0xffff); 1242 if (rc != 0) { 1243 cxgb_printf(dip, CE_WARN, 1244 "failed to free queue %p: %d", iq, rc); 1245 return (rc); 1246 } 1247 iq->flags &= ~IQ_ALLOCATED; 1248 } 1249 1250 if (iq->desc != NULL) { 1251 (void) free_desc_ring(&iq->dhdl, &iq->ahdl); 1252 iq->desc = NULL; 1253 } 1254 1255 bzero(iq, sizeof (*iq)); 1256 } 1257 1258 if (fl != NULL) { 1259 if (fl->sdesc != NULL) { 1260 FL_LOCK(fl); 1261 free_fl_bufs(fl); 1262 FL_UNLOCK(fl); 1263 1264 kmem_free(fl->sdesc, sizeof (struct fl_sdesc) * 1265 fl->cap); 1266 fl->sdesc = NULL; 1267 } 1268 1269 if (fl->desc != NULL) { 1270 (void) free_desc_ring(&fl->dhdl, &fl->ahdl); 1271 fl->desc = NULL; 1272 } 1273 1274 if (fl->flags & FL_MTX) { 1275 mutex_destroy(&fl->lock); 1276 fl->flags &= ~FL_MTX; 1277 } 1278 1279 bzero(fl, sizeof (struct sge_fl)); 1280 } 1281 1282 return (0); 1283 } 1284 1285 static int 1286 alloc_fwq(struct adapter *sc) 1287 { 1288 int rc, intr_idx; 1289 struct sge_iq *fwq = &sc->sge.fwq; 1290 1291 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE); 1292 fwq->flags |= IQ_INTR; /* always */ 1293 intr_idx = sc->intr_count > 1 ? 1 : 0; 1294 rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1); 1295 if (rc != 0) { 1296 cxgb_printf(sc->dip, CE_WARN, 1297 "failed to create firmware event queue: %d.", rc); 1298 return (rc); 1299 } 1300 1301 return (0); 1302 } 1303 1304 static int 1305 free_fwq(struct adapter *sc) 1306 { 1307 1308 return (free_iq_fl(NULL, &sc->sge.fwq, NULL)); 1309 } 1310 1311 static int 1312 alloc_mgmtq(struct adapter *sc) 1313 { 1314 int rc; 1315 struct sge_wrq *mgmtq = &sc->sge.mgmtq; 1316 1317 init_eq(&mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan, 1318 sc->sge.fwq.cntxt_id); 1319 rc = alloc_wrq(sc, NULL, mgmtq, 0); 1320 if (rc != 0) { 1321 cxgb_printf(sc->dip, CE_WARN, 1322 "failed to create management queue: %d\n", rc); 1323 return (rc); 1324 } 1325 1326 return (0); 1327 } 1328 1329 static int 1330 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int i) 1331 { 1332 int rc; 1333 1334 rxq->port = pi; 1335 rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx, 1 << pi->tx_chan); 1336 if (rc != 0) 1337 return (rc); 1338 1339 rxq->ksp = setup_rxq_kstats(pi, rxq, i); 1340 1341 return (rc); 1342 } 1343 1344 static int 1345 free_rxq(struct port_info *pi, struct sge_rxq *rxq) 1346 { 1347 int rc; 1348 1349 if (rxq->ksp != NULL) { 1350 kstat_delete(rxq->ksp); 1351 rxq->ksp = NULL; 1352 } 1353 1354 rc = free_iq_fl(pi, &rxq->iq, &rxq->fl); 1355 if (rc == 0) 1356 bzero(&rxq->fl, sizeof (*rxq) - offsetof(struct sge_rxq, fl)); 1357 1358 return (rc); 1359 } 1360 1361 #ifndef TCP_OFFLOAD_DISABLE 1362 static int 1363 alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq, 1364 int intr_idx) 1365 { 1366 int rc; 1367 1368 rc = alloc_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx, 1369 1 << pi->tx_chan); 1370 if (rc != 0) 1371 return (rc); 1372 1373 return (rc); 1374 } 1375 1376 static int 1377 free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq) 1378 { 1379 int rc; 1380 1381 rc = free_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl); 1382 if (rc == 0) 1383 bzero(&ofld_rxq->fl, sizeof (*ofld_rxq) - 1384 offsetof(struct sge_ofld_rxq, fl)); 1385 1386 return (rc); 1387 } 1388 #endif 1389 1390 static int 1391 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) 1392 { 1393 int rc, cntxt_id; 1394 struct fw_eq_ctrl_cmd c; 1395 1396 bzero(&c, sizeof (c)); 1397 1398 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST | 1399 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) | 1400 V_FW_EQ_CTRL_CMD_VFN(0)); 1401 c.alloc_to_len16 = BE_32(F_FW_EQ_CTRL_CMD_ALLOC | 1402 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c)); 1403 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* TODO */ 1404 c.physeqid_pkd = BE_32(0); 1405 c.fetchszm_to_iqid = 1406 BE_32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1407 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) | 1408 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid)); 1409 c.dcaen_to_eqsize = 1410 BE_32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1411 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1412 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1413 V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize)); 1414 c.eqaddr = BE_64(eq->ba); 1415 1416 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1417 if (rc != 0) { 1418 cxgb_printf(sc->dip, CE_WARN, 1419 "failed to create control queue %d: %d", eq->tx_chan, rc); 1420 return (rc); 1421 } 1422 eq->flags |= EQ_ALLOCATED; 1423 1424 eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(BE_32(c.cmpliqid_eqid)); 1425 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1426 if (cntxt_id >= sc->sge.neq) 1427 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1428 cntxt_id, sc->sge.neq - 1); 1429 sc->sge.eqmap[cntxt_id] = eq; 1430 1431 return (rc); 1432 } 1433 1434 static int 1435 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1436 { 1437 int rc, cntxt_id; 1438 struct fw_eq_eth_cmd c; 1439 1440 bzero(&c, sizeof (c)); 1441 1442 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST | 1443 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) | 1444 V_FW_EQ_ETH_CMD_VFN(0)); 1445 c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC | 1446 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c)); 1447 c.viid_pkd = BE_32(V_FW_EQ_ETH_CMD_VIID(pi->viid)); 1448 c.fetchszm_to_iqid = 1449 BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1450 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | 1451 V_FW_EQ_ETH_CMD_IQID(eq->iqid)); 1452 c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1453 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1454 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1455 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize)); 1456 c.eqaddr = BE_64(eq->ba); 1457 1458 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1459 if (rc != 0) { 1460 cxgb_printf(pi->dip, CE_WARN, 1461 "failed to create Ethernet egress queue: %d", rc); 1462 return (rc); 1463 } 1464 eq->flags |= EQ_ALLOCATED; 1465 1466 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd)); 1467 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1468 if (cntxt_id >= sc->sge.neq) 1469 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1470 cntxt_id, sc->sge.neq - 1); 1471 sc->sge.eqmap[cntxt_id] = eq; 1472 1473 return (rc); 1474 } 1475 1476 #ifndef TCP_OFFLOAD_DISABLE 1477 static int 1478 ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1479 { 1480 int rc, cntxt_id; 1481 struct fw_eq_ofld_cmd c; 1482 1483 bzero(&c, sizeof (c)); 1484 1485 c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST | 1486 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) | 1487 V_FW_EQ_OFLD_CMD_VFN(0)); 1488 c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC | 1489 F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c)); 1490 c.fetchszm_to_iqid = 1491 htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1492 V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) | 1493 F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid)); 1494 c.dcaen_to_eqsize = 1495 BE_32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1496 V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1497 V_FW_EQ_OFLD_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1498 V_FW_EQ_OFLD_CMD_EQSIZE(eq->qsize)); 1499 c.eqaddr = BE_64(eq->ba); 1500 1501 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1502 if (rc != 0) { 1503 cxgb_printf(pi->dip, CE_WARN, 1504 "failed to create egress queue for TCP offload: %d", rc); 1505 return (rc); 1506 } 1507 eq->flags |= EQ_ALLOCATED; 1508 1509 eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(BE_32(c.eqid_pkd)); 1510 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1511 if (cntxt_id >= sc->sge.neq) 1512 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1513 cntxt_id, sc->sge.neq - 1); 1514 sc->sge.eqmap[cntxt_id] = eq; 1515 1516 return (rc); 1517 } 1518 #endif 1519 1520 static int 1521 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1522 { 1523 int rc; 1524 size_t len; 1525 1526 mutex_init(&eq->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(sc->intr_pri)); 1527 eq->flags |= EQ_MTX; 1528 1529 len = eq->qsize * EQ_ESIZE; 1530 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &eq->desc_dhdl, 1531 &eq->desc_ahdl, &eq->ba, (caddr_t *)&eq->desc); 1532 if (rc != 0) 1533 return (rc); 1534 1535 eq->cap = eq->qsize - SPG_SIZE / EQ_ESIZE; 1536 eq->spg = (void *)&eq->desc[eq->cap]; 1537 eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */ 1538 eq->pidx = eq->cidx = 0; 1539 1540 switch (eq->flags & EQ_TYPEMASK) { 1541 case EQ_CTRL: 1542 rc = ctrl_eq_alloc(sc, eq); 1543 break; 1544 1545 case EQ_ETH: 1546 rc = eth_eq_alloc(sc, pi, eq); 1547 break; 1548 1549 #ifndef TCP_OFFLOAD_DISABLE 1550 case EQ_OFLD: 1551 rc = ofld_eq_alloc(sc, pi, eq); 1552 break; 1553 #endif 1554 1555 default: 1556 panic("%s: invalid eq type %d.", __func__, 1557 eq->flags & EQ_TYPEMASK); 1558 } 1559 if (rc != 0) { 1560 cxgb_printf(sc->dip, CE_WARN, 1561 "failed to allocate egress queue(%d): %d", 1562 eq->flags & EQ_TYPEMASK, rc); 1563 } 1564 1565 return (rc); 1566 } 1567 1568 static int 1569 free_eq(struct adapter *sc, struct sge_eq *eq) 1570 { 1571 int rc; 1572 1573 if (eq->flags & EQ_ALLOCATED) { 1574 switch (eq->flags & EQ_TYPEMASK) { 1575 case EQ_CTRL: 1576 rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0, 1577 eq->cntxt_id); 1578 break; 1579 1580 case EQ_ETH: 1581 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, 1582 eq->cntxt_id); 1583 break; 1584 1585 #ifndef TCP_OFFLOAD_DISABLE 1586 case EQ_OFLD: 1587 rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0, 1588 eq->cntxt_id); 1589 break; 1590 #endif 1591 1592 default: 1593 panic("%s: invalid eq type %d.", __func__, 1594 eq->flags & EQ_TYPEMASK); 1595 } 1596 if (rc != 0) { 1597 cxgb_printf(sc->dip, CE_WARN, 1598 "failed to free egress queue (%d): %d", 1599 eq->flags & EQ_TYPEMASK, rc); 1600 return (rc); 1601 } 1602 eq->flags &= ~EQ_ALLOCATED; 1603 } 1604 1605 if (eq->desc != NULL) { 1606 (void) free_desc_ring(&eq->desc_dhdl, &eq->desc_ahdl); 1607 eq->desc = NULL; 1608 } 1609 1610 if (eq->flags & EQ_MTX) 1611 mutex_destroy(&eq->lock); 1612 1613 bzero(eq, sizeof (*eq)); 1614 return (0); 1615 } 1616 1617 /* ARGSUSED */ 1618 static int 1619 alloc_wrq(struct adapter *sc, struct port_info *pi, struct sge_wrq *wrq, 1620 int idx) 1621 { 1622 int rc; 1623 1624 rc = alloc_eq(sc, pi, &wrq->eq); 1625 if (rc != 0) 1626 return (rc); 1627 1628 wrq->adapter = sc; 1629 wrq->wr_list.head = NULL; 1630 wrq->wr_list.tail = NULL; 1631 1632 /* 1633 * TODO: use idx to figure out what kind of wrq this is and install 1634 * useful kstats for it. 1635 */ 1636 1637 return (rc); 1638 } 1639 1640 static int 1641 free_wrq(struct adapter *sc, struct sge_wrq *wrq) 1642 { 1643 int rc; 1644 1645 rc = free_eq(sc, &wrq->eq); 1646 if (rc != 0) 1647 return (rc); 1648 1649 bzero(wrq, sizeof (*wrq)); 1650 return (0); 1651 } 1652 1653 static int 1654 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx) 1655 { 1656 int rc, i; 1657 struct adapter *sc = pi->adapter; 1658 struct sge_eq *eq = &txq->eq; 1659 1660 rc = alloc_eq(sc, pi, eq); 1661 if (rc != 0) 1662 return (rc); 1663 1664 txq->port = pi; 1665 txq->sdesc = kmem_zalloc(sizeof (struct tx_sdesc) * eq->cap, KM_SLEEP); 1666 txq->txb_size = eq->qsize * tx_copy_threshold; 1667 rc = alloc_tx_copybuffer(sc, txq->txb_size, &txq->txb_dhdl, 1668 &txq->txb_ahdl, &txq->txb_ba, &txq->txb_va); 1669 if (rc == 0) 1670 txq->txb_avail = txq->txb_size; 1671 else 1672 txq->txb_avail = txq->txb_size = 0; 1673 1674 /* 1675 * TODO: is this too low? Worst case would need around 4 times qsize 1676 * (all tx descriptors filled to the brim with SGLs, with each entry in 1677 * the SGL coming from a distinct DMA handle). Increase tx_dhdl_total 1678 * if you see too many dma_hdl_failed. 1679 */ 1680 txq->tx_dhdl_total = eq->qsize * 2; 1681 txq->tx_dhdl = kmem_zalloc(sizeof (ddi_dma_handle_t) * 1682 txq->tx_dhdl_total, KM_SLEEP); 1683 for (i = 0; i < txq->tx_dhdl_total; i++) { 1684 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx, 1685 DDI_DMA_SLEEP, 0, &txq->tx_dhdl[i]); 1686 if (rc != DDI_SUCCESS) { 1687 cxgb_printf(sc->dip, CE_WARN, 1688 "%s: failed to allocate DMA handle (%d)", 1689 __func__, rc); 1690 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 1691 } 1692 txq->tx_dhdl_avail++; 1693 } 1694 1695 txq->ksp = setup_txq_kstats(pi, txq, idx); 1696 1697 return (rc); 1698 } 1699 1700 static int 1701 free_txq(struct port_info *pi, struct sge_txq *txq) 1702 { 1703 int i; 1704 struct adapter *sc = pi->adapter; 1705 struct sge_eq *eq = &txq->eq; 1706 1707 if (txq->ksp != NULL) { 1708 kstat_delete(txq->ksp); 1709 txq->ksp = NULL; 1710 } 1711 1712 if (txq->txb_va != NULL) { 1713 (void) free_desc_ring(&txq->txb_dhdl, &txq->txb_ahdl); 1714 txq->txb_va = NULL; 1715 } 1716 1717 if (txq->sdesc != NULL) { 1718 struct tx_sdesc *sd; 1719 ddi_dma_handle_t hdl; 1720 1721 TXQ_LOCK(txq); 1722 while (eq->cidx != eq->pidx) { 1723 sd = &txq->sdesc[eq->cidx]; 1724 1725 for (i = sd->hdls_used; i; i--) { 1726 hdl = txq->tx_dhdl[txq->tx_dhdl_cidx]; 1727 (void) ddi_dma_unbind_handle(hdl); 1728 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total) 1729 txq->tx_dhdl_cidx = 0; 1730 } 1731 1732 ASSERT(sd->m); 1733 freemsgchain(sd->m); 1734 1735 eq->cidx += sd->desc_used; 1736 if (eq->cidx >= eq->cap) 1737 eq->cidx -= eq->cap; 1738 1739 txq->txb_avail += txq->txb_used; 1740 } 1741 ASSERT(txq->tx_dhdl_cidx == txq->tx_dhdl_pidx); 1742 ASSERT(txq->txb_avail == txq->txb_size); 1743 TXQ_UNLOCK(txq); 1744 1745 kmem_free(txq->sdesc, sizeof (struct tx_sdesc) * eq->cap); 1746 txq->sdesc = NULL; 1747 } 1748 1749 if (txq->tx_dhdl != NULL) { 1750 for (i = 0; i < txq->tx_dhdl_total; i++) { 1751 if (txq->tx_dhdl[i] != NULL) 1752 ddi_dma_free_handle(&txq->tx_dhdl[i]); 1753 } 1754 } 1755 1756 (void) free_eq(sc, &txq->eq); 1757 1758 bzero(txq, sizeof (*txq)); 1759 return (0); 1760 } 1761 1762 /* 1763 * Allocates a block of contiguous memory for DMA. Can be used to allocate 1764 * memory for descriptor rings or for tx/rx copy buffers. 1765 * 1766 * Caller does not have to clean up anything if this function fails, it cleans 1767 * up after itself. 1768 * 1769 * Caller provides the following: 1770 * len length of the block of memory to allocate. 1771 * flags DDI_DMA_* flags to use (CONSISTENT/STREAMING, READ/WRITE/RDWR) 1772 * acc_attr device access attributes for the allocation. 1773 * dma_attr DMA attributes for the allocation 1774 * 1775 * If the function is successful it fills up this information: 1776 * dma_hdl DMA handle for the allocated memory 1777 * acc_hdl access handle for the allocated memory 1778 * ba bus address of the allocated memory 1779 * va KVA of the allocated memory. 1780 */ 1781 static int 1782 alloc_dma_memory(struct adapter *sc, size_t len, int flags, 1783 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr, 1784 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 1785 uint64_t *pba, caddr_t *pva) 1786 { 1787 int rc; 1788 ddi_dma_handle_t dhdl; 1789 ddi_acc_handle_t ahdl; 1790 ddi_dma_cookie_t cookie; 1791 uint_t ccount; 1792 caddr_t va; 1793 size_t real_len; 1794 1795 *pva = NULL; 1796 1797 /* 1798 * DMA handle. 1799 */ 1800 rc = ddi_dma_alloc_handle(sc->dip, dma_attr, DDI_DMA_SLEEP, 0, &dhdl); 1801 if (rc != DDI_SUCCESS) { 1802 cxgb_printf(sc->dip, CE_WARN, 1803 "failed to allocate DMA handle: %d", rc); 1804 1805 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 1806 } 1807 1808 /* 1809 * Memory suitable for DMA. 1810 */ 1811 rc = ddi_dma_mem_alloc(dhdl, len, acc_attr, 1812 flags & DDI_DMA_CONSISTENT ? DDI_DMA_CONSISTENT : DDI_DMA_STREAMING, 1813 DDI_DMA_SLEEP, 0, &va, &real_len, &ahdl); 1814 if (rc != DDI_SUCCESS) { 1815 cxgb_printf(sc->dip, CE_WARN, 1816 "failed to allocate DMA memory: %d", rc); 1817 1818 ddi_dma_free_handle(&dhdl); 1819 return (ENOMEM); 1820 } 1821 1822 if (len != real_len) { 1823 cxgb_printf(sc->dip, CE_WARN, 1824 "%s: len (%u) != real_len (%u)\n", len, real_len); 1825 } 1826 1827 /* 1828 * DMA bindings. 1829 */ 1830 rc = ddi_dma_addr_bind_handle(dhdl, NULL, va, real_len, flags, NULL, 1831 NULL, &cookie, &ccount); 1832 if (rc != DDI_DMA_MAPPED) { 1833 cxgb_printf(sc->dip, CE_WARN, 1834 "failed to map DMA memory: %d", rc); 1835 1836 ddi_dma_mem_free(&ahdl); 1837 ddi_dma_free_handle(&dhdl); 1838 return (ENOMEM); 1839 } 1840 if (ccount != 1) { 1841 cxgb_printf(sc->dip, CE_WARN, 1842 "unusable DMA mapping (%d segments)", ccount); 1843 (void) free_desc_ring(&dhdl, &ahdl); 1844 } 1845 1846 bzero(va, real_len); 1847 *dma_hdl = dhdl; 1848 *acc_hdl = ahdl; 1849 *pba = cookie.dmac_laddress; 1850 *pva = va; 1851 1852 return (0); 1853 } 1854 1855 static int 1856 free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl) 1857 { 1858 (void) ddi_dma_unbind_handle(*dhdl); 1859 ddi_dma_mem_free(ahdl); 1860 ddi_dma_free_handle(dhdl); 1861 1862 return (0); 1863 } 1864 1865 static int 1866 alloc_desc_ring(struct adapter *sc, size_t len, int rw, 1867 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 1868 uint64_t *pba, caddr_t *pva) 1869 { 1870 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_desc; 1871 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; 1872 1873 return (alloc_dma_memory(sc, len, DDI_DMA_CONSISTENT | rw, acc_attr, 1874 dma_attr, dma_hdl, acc_hdl, pba, pva)); 1875 } 1876 1877 static int 1878 free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl) 1879 { 1880 return (free_dma_memory(dhdl, ahdl)); 1881 } 1882 1883 static int 1884 alloc_tx_copybuffer(struct adapter *sc, size_t len, 1885 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 1886 uint64_t *pba, caddr_t *pva) 1887 { 1888 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_tx; 1889 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; /* NOT dma_attr_tx */ 1890 1891 return (alloc_dma_memory(sc, len, DDI_DMA_STREAMING | DDI_DMA_WRITE, 1892 acc_attr, dma_attr, dma_hdl, acc_hdl, pba, pva)); 1893 } 1894 1895 static inline bool 1896 is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl) 1897 { 1898 (void) ddi_dma_sync(iq->dhdl, (uintptr_t)iq->cdesc - 1899 (uintptr_t)iq->desc, iq->esize, DDI_DMA_SYNC_FORKERNEL); 1900 1901 *ctrl = (void *)((uintptr_t)iq->cdesc + 1902 (iq->esize - sizeof (struct rsp_ctrl))); 1903 1904 return ((((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen)); 1905 } 1906 1907 static inline void 1908 iq_next(struct sge_iq *iq) 1909 { 1910 iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize); 1911 if (++iq->cidx == iq->qsize - 1) { 1912 iq->cidx = 0; 1913 iq->gen ^= 1; 1914 iq->cdesc = iq->desc; 1915 } 1916 } 1917 1918 /* 1919 * Fill up the freelist by upto nbufs and maybe ring its doorbell. 1920 * 1921 * Returns non-zero to indicate that it should be added to the list of starving 1922 * freelists. 1923 */ 1924 static int 1925 refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs) 1926 { 1927 uint64_t *d = &fl->desc[fl->pidx]; 1928 struct fl_sdesc *sd = &fl->sdesc[fl->pidx]; 1929 1930 FL_LOCK_ASSERT_OWNED(fl); 1931 ASSERT(nbufs >= 0); 1932 1933 if (nbufs > fl->needed) 1934 nbufs = fl->needed; 1935 1936 while (nbufs--) { 1937 if (sd->rxb != NULL) { 1938 if (sd->rxb->ref_cnt == 1) { 1939 /* 1940 * Buffer is available for recycling. Two ways 1941 * this can happen: 1942 * 1943 * a) All the packets DMA'd into it last time 1944 * around were within the rx_copy_threshold 1945 * and no part of the buffer was ever passed 1946 * up (ref_cnt never went over 1). 1947 * 1948 * b) Packets DMA'd into the buffer were passed 1949 * up but have all been freed by the upper 1950 * layers by now (ref_cnt went over 1 but is 1951 * now back to 1). 1952 * 1953 * Either way the bus address in the descriptor 1954 * ring is already valid. 1955 */ 1956 ASSERT(*d == cpu_to_be64(sd->rxb->ba)); 1957 d++; 1958 goto recycled; 1959 } else { 1960 /* 1961 * Buffer still in use and we need a 1962 * replacement. But first release our reference 1963 * on the existing buffer. 1964 */ 1965 rxbuf_free(sd->rxb); 1966 } 1967 } 1968 1969 sd->rxb = rxbuf_alloc(sc->sge.rxbuf_cache, KM_NOSLEEP, 1); 1970 if (sd->rxb == NULL) 1971 break; 1972 *d++ = cpu_to_be64(sd->rxb->ba); 1973 1974 recycled: fl->pending++; 1975 sd++; 1976 fl->needed--; 1977 if (++fl->pidx == fl->cap) { 1978 fl->pidx = 0; 1979 sd = fl->sdesc; 1980 d = fl->desc; 1981 } 1982 } 1983 1984 if (fl->pending >= 8) 1985 ring_fl_db(sc, fl); 1986 1987 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING)); 1988 } 1989 1990 #ifndef TAILQ_FOREACH_SAFE 1991 #define TAILQ_FOREACH_SAFE(var, head, field, tvar) \ 1992 for ((var) = TAILQ_FIRST((head)); \ 1993 (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \ 1994 (var) = (tvar)) 1995 #endif 1996 1997 /* 1998 * Attempt to refill all starving freelists. 1999 */ 2000 static void 2001 refill_sfl(void *arg) 2002 { 2003 struct adapter *sc = arg; 2004 struct sge_fl *fl, *fl_temp; 2005 2006 mutex_enter(&sc->sfl_lock); 2007 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) { 2008 FL_LOCK(fl); 2009 (void) refill_fl(sc, fl, 64); 2010 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) { 2011 TAILQ_REMOVE(&sc->sfl, fl, link); 2012 fl->flags &= ~FL_STARVING; 2013 } 2014 FL_UNLOCK(fl); 2015 } 2016 2017 if (!TAILQ_EMPTY(&sc->sfl) != 0) 2018 sc->sfl_timer = timeout(refill_sfl, sc, drv_usectohz(100000)); 2019 mutex_exit(&sc->sfl_lock); 2020 } 2021 2022 static void 2023 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl) 2024 { 2025 mutex_enter(&sc->sfl_lock); 2026 FL_LOCK(fl); 2027 if ((fl->flags & FL_DOOMED) == 0) { 2028 if (TAILQ_EMPTY(&sc->sfl) != 0) { 2029 sc->sfl_timer = timeout(refill_sfl, sc, 2030 drv_usectohz(100000)); 2031 } 2032 fl->flags |= FL_STARVING; 2033 TAILQ_INSERT_TAIL(&sc->sfl, fl, link); 2034 } 2035 FL_UNLOCK(fl); 2036 mutex_exit(&sc->sfl_lock); 2037 } 2038 2039 static void 2040 free_fl_bufs(struct sge_fl *fl) 2041 { 2042 struct fl_sdesc *sd; 2043 unsigned int i; 2044 2045 FL_LOCK_ASSERT_OWNED(fl); 2046 2047 for (i = 0; i < fl->cap; i++) { 2048 sd = &fl->sdesc[i]; 2049 2050 if (sd->rxb != NULL) { 2051 rxbuf_free(sd->rxb); 2052 sd->rxb = NULL; 2053 } 2054 } 2055 } 2056 2057 /* 2058 * Note that fl->cidx and fl->offset are left unchanged in case of failure. 2059 */ 2060 static mblk_t * 2061 get_fl_payload(struct sge_fl *fl, uint32_t len_newbuf, int *fl_bufs_used) 2062 { 2063 struct mblk_pair frame = {0}; 2064 struct rxbuf *rxb; 2065 mblk_t *m = NULL; 2066 uint_t nbuf = 0, len, copy, n; 2067 uint32_t cidx, offset; 2068 2069 /* 2070 * The SGE won't pack a new frame into the current buffer if the entire 2071 * payload doesn't fit in the remaining space. Move on to the next buf 2072 * in that case. 2073 */ 2074 if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) { 2075 fl->offset = 0; 2076 if (++fl->cidx == fl->cap) 2077 fl->cidx = 0; 2078 nbuf++; 2079 } 2080 cidx = fl->cidx; 2081 offset = fl->offset; 2082 2083 len = G_RSPD_LEN(len_newbuf); /* pktshift + payload length */ 2084 copy = (len <= fl->copy_threshold); 2085 if (copy != 0) { 2086 frame.head = m = allocb(len, BPRI_HI); 2087 if (m == NULL) 2088 return (NULL); 2089 } 2090 2091 while (len) { 2092 rxb = fl->sdesc[cidx].rxb; 2093 n = min(len, rxb->buf_size - offset); 2094 2095 (void) ddi_dma_sync(rxb->dhdl, offset, n, 2096 DDI_DMA_SYNC_FORKERNEL); 2097 2098 if (copy != 0) 2099 bcopy(rxb->va + offset, m->b_wptr, n); 2100 else { 2101 m = desballoc((unsigned char *)rxb->va + offset, n, 2102 BPRI_HI, &rxb->freefunc); 2103 if (m == NULL) { 2104 freemsg(frame.head); 2105 return (NULL); 2106 } 2107 atomic_inc_uint(&rxb->ref_cnt); 2108 if (frame.head != NULL) 2109 frame.tail->b_cont = m; 2110 else 2111 frame.head = m; 2112 frame.tail = m; 2113 } 2114 m->b_wptr += n; 2115 len -= n; 2116 offset += roundup(n, FL_ALIGN); 2117 ASSERT(offset <= rxb->buf_size); 2118 if (offset == rxb->buf_size) { 2119 offset = 0; 2120 if (++cidx == fl->cap) 2121 cidx = 0; 2122 nbuf++; 2123 } 2124 } 2125 2126 fl->cidx = cidx; 2127 fl->offset = offset; 2128 (*fl_bufs_used) += nbuf; 2129 2130 ASSERT(frame.head != NULL); 2131 return (frame.head); 2132 } 2133 2134 /* 2135 * We'll do immediate data tx for non-LSO, but only when not coalescing. We're 2136 * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes 2137 * of immediate data. 2138 */ 2139 #define IMM_LEN ( \ 2140 2 * EQ_ESIZE \ 2141 - sizeof (struct fw_eth_tx_pkt_wr) \ 2142 - sizeof (struct cpl_tx_pkt_core)) 2143 2144 /* 2145 * Returns non-zero on failure, no need to cleanup anything in that case. 2146 * 2147 * Note 1: We always try to pull up the mblk if required and return E2BIG only 2148 * if this fails. 2149 * 2150 * Note 2: We'll also pullup incoming mblk if HW_LSO is set and the first mblk 2151 * does not have the TCP header in it. 2152 */ 2153 static int 2154 get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, struct txinfo *txinfo, 2155 int sgl_only) 2156 { 2157 uint32_t flags = 0, len, n; 2158 mblk_t *m = *fp; 2159 int rc; 2160 2161 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb and dma_hdls */ 2162 2163 mac_hcksum_get(m, NULL, NULL, NULL, NULL, &flags); 2164 txinfo->flags = flags; 2165 2166 mac_lso_get(m, &txinfo->mss, &flags); 2167 txinfo->flags |= flags; 2168 2169 if (flags & HW_LSO) 2170 sgl_only = 1; /* Do not allow immediate data with LSO */ 2171 2172 start: txinfo->nsegs = 0; 2173 txinfo->hdls_used = 0; 2174 txinfo->txb_used = 0; 2175 txinfo->len = 0; 2176 2177 /* total length and a rough estimate of # of segments */ 2178 n = 0; 2179 for (; m; m = m->b_cont) { 2180 len = MBLKL(m); 2181 n += (len / PAGE_SIZE) + 1; 2182 txinfo->len += len; 2183 } 2184 m = *fp; 2185 2186 if (n >= TX_SGL_SEGS || (flags & HW_LSO && MBLKL(m) < 50)) { 2187 txq->pullup_early++; 2188 m = msgpullup(*fp, -1); 2189 if (m == NULL) { 2190 txq->pullup_failed++; 2191 return (E2BIG); /* (*fp) left as it was */ 2192 } 2193 freemsg(*fp); 2194 *fp = m; 2195 mac_hcksum_set(m, NULL, NULL, NULL, NULL, txinfo->flags); 2196 } 2197 2198 if (txinfo->len <= IMM_LEN && !sgl_only) 2199 return (0); /* nsegs = 0 tells caller to use imm. tx */ 2200 2201 if (txinfo->len <= txq->copy_threshold && 2202 copy_into_txb(txq, m, txinfo->len, txinfo) == 0) 2203 goto done; 2204 2205 for (; m; m = m->b_cont) { 2206 2207 len = MBLKL(m); 2208 2209 /* Use tx copy buffer if this mblk is small enough */ 2210 if (len <= txq->copy_threshold && 2211 copy_into_txb(txq, m, len, txinfo) == 0) 2212 continue; 2213 2214 /* Add DMA bindings for this mblk to the SGL */ 2215 rc = add_mblk(txq, txinfo, m, len); 2216 2217 if (rc == E2BIG || 2218 (txinfo->nsegs == TX_SGL_SEGS && m->b_cont)) { 2219 2220 txq->pullup_late++; 2221 m = msgpullup(*fp, -1); 2222 if (m != NULL) { 2223 free_txinfo_resources(txq, txinfo); 2224 freemsg(*fp); 2225 *fp = m; 2226 mac_hcksum_set(m, NULL, NULL, NULL, NULL, 2227 txinfo->flags); 2228 goto start; 2229 } 2230 2231 txq->pullup_failed++; 2232 rc = E2BIG; 2233 } 2234 2235 if (rc != 0) { 2236 free_txinfo_resources(txq, txinfo); 2237 return (rc); 2238 } 2239 } 2240 2241 ASSERT(txinfo->nsegs > 0 && txinfo->nsegs <= TX_SGL_SEGS); 2242 2243 done: 2244 2245 /* 2246 * Store the # of flits required to hold this frame's SGL in nflits. An 2247 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by 2248 * multiple (len0 + len1, addr0, addr1) tuples. If addr1 is not used 2249 * then len1 must be set to 0. 2250 */ 2251 n = txinfo->nsegs - 1; 2252 txinfo->nflits = (3 * n) / 2 + (n & 1) + 2; 2253 if (n & 1) 2254 txinfo->sgl.sge[n / 2].len[1] = cpu_to_be32(0); 2255 2256 txinfo->sgl.cmd_nsge = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_DSGL) | 2257 V_ULPTX_NSGE(txinfo->nsegs)); 2258 2259 return (0); 2260 } 2261 2262 static inline int 2263 fits_in_txb(struct sge_txq *txq, int len, int *waste) 2264 { 2265 if (txq->txb_avail < len) 2266 return (0); 2267 2268 if (txq->txb_next + len <= txq->txb_size) { 2269 *waste = 0; 2270 return (1); 2271 } 2272 2273 *waste = txq->txb_size - txq->txb_next; 2274 2275 return (txq->txb_avail - *waste < len ? 0 : 1); 2276 } 2277 2278 #define TXB_CHUNK 64 2279 2280 /* 2281 * Copies the specified # of bytes into txq's tx copy buffer and updates txinfo 2282 * and txq to indicate resources used. Caller has to make sure that those many 2283 * bytes are available in the mblk chain (b_cont linked). 2284 */ 2285 static inline int 2286 copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, struct txinfo *txinfo) 2287 { 2288 int waste, n; 2289 2290 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb */ 2291 2292 if (!fits_in_txb(txq, len, &waste)) { 2293 txq->txb_full++; 2294 return (ENOMEM); 2295 } 2296 2297 if (waste != 0) { 2298 ASSERT((waste & (TXB_CHUNK - 1)) == 0); 2299 txinfo->txb_used += waste; 2300 txq->txb_avail -= waste; 2301 txq->txb_next = 0; 2302 } 2303 2304 for (n = 0; n < len; m = m->b_cont) { 2305 bcopy(m->b_rptr, txq->txb_va + txq->txb_next + n, MBLKL(m)); 2306 n += MBLKL(m); 2307 } 2308 2309 add_seg(txinfo, txq->txb_ba + txq->txb_next, len); 2310 2311 n = roundup(len, TXB_CHUNK); 2312 txinfo->txb_used += n; 2313 txq->txb_avail -= n; 2314 txq->txb_next += n; 2315 ASSERT(txq->txb_next <= txq->txb_size); 2316 if (txq->txb_next == txq->txb_size) 2317 txq->txb_next = 0; 2318 2319 return (0); 2320 } 2321 2322 static inline void 2323 add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len) 2324 { 2325 ASSERT(txinfo->nsegs < TX_SGL_SEGS); /* must have room */ 2326 2327 if (txinfo->nsegs != 0) { 2328 int idx = txinfo->nsegs - 1; 2329 txinfo->sgl.sge[idx / 2].len[idx & 1] = cpu_to_be32(len); 2330 txinfo->sgl.sge[idx / 2].addr[idx & 1] = cpu_to_be64(ba); 2331 } else { 2332 txinfo->sgl.len0 = cpu_to_be32(len); 2333 txinfo->sgl.addr0 = cpu_to_be64(ba); 2334 } 2335 txinfo->nsegs++; 2336 } 2337 2338 /* 2339 * This function cleans up any partially allocated resources when it fails so 2340 * there's nothing for the caller to clean up in that case. 2341 * 2342 * EIO indicates permanent failure. Caller should drop the frame containing 2343 * this mblk and continue. 2344 * 2345 * E2BIG indicates that the SGL length for this mblk exceeds the hardware 2346 * limit. Caller should pull up the frame before trying to send it out. 2347 * (This error means our pullup_early heuristic did not work for this frame) 2348 * 2349 * ENOMEM indicates a temporary shortage of resources (DMA handles, other DMA 2350 * resources, etc.). Caller should suspend the tx queue and wait for reclaim to 2351 * free up resources. 2352 */ 2353 static inline int 2354 add_mblk(struct sge_txq *txq, struct txinfo *txinfo, mblk_t *m, int len) 2355 { 2356 ddi_dma_handle_t dhdl; 2357 ddi_dma_cookie_t cookie; 2358 uint_t ccount = 0; 2359 int rc; 2360 2361 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate dhdls */ 2362 2363 if (txq->tx_dhdl_avail == 0) { 2364 txq->dma_hdl_failed++; 2365 return (ENOMEM); 2366 } 2367 2368 dhdl = txq->tx_dhdl[txq->tx_dhdl_pidx]; 2369 rc = ddi_dma_addr_bind_handle(dhdl, NULL, (caddr_t)m->b_rptr, len, 2370 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie, 2371 &ccount); 2372 if (rc != DDI_DMA_MAPPED) { 2373 txq->dma_map_failed++; 2374 2375 ASSERT(rc != DDI_DMA_INUSE && rc != DDI_DMA_PARTIAL_MAP); 2376 2377 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EIO); 2378 } 2379 2380 if (ccount + txinfo->nsegs > TX_SGL_SEGS) { 2381 (void) ddi_dma_unbind_handle(dhdl); 2382 return (E2BIG); 2383 } 2384 2385 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size); 2386 while (--ccount) { 2387 ddi_dma_nextcookie(dhdl, &cookie); 2388 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size); 2389 } 2390 2391 if (++txq->tx_dhdl_pidx == txq->tx_dhdl_total) 2392 txq->tx_dhdl_pidx = 0; 2393 txq->tx_dhdl_avail--; 2394 txinfo->hdls_used++; 2395 2396 return (0); 2397 } 2398 2399 /* 2400 * Releases all the txq resources used up in the specified txinfo. 2401 */ 2402 static void 2403 free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo) 2404 { 2405 int n; 2406 2407 TXQ_LOCK_ASSERT_OWNED(txq); /* dhdls, txb */ 2408 2409 n = txinfo->txb_used; 2410 if (n > 0) { 2411 txq->txb_avail += n; 2412 if (n <= txq->txb_next) 2413 txq->txb_next -= n; 2414 else { 2415 n -= txq->txb_next; 2416 txq->txb_next = txq->txb_size - n; 2417 } 2418 } 2419 2420 for (n = txinfo->hdls_used; n > 0; n--) { 2421 if (txq->tx_dhdl_pidx > 0) 2422 txq->tx_dhdl_pidx--; 2423 else 2424 txq->tx_dhdl_pidx = txq->tx_dhdl_total - 1; 2425 txq->tx_dhdl_avail++; 2426 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_pidx]); 2427 } 2428 } 2429 2430 /* 2431 * Returns 0 to indicate that m has been accepted into a coalesced tx work 2432 * request. It has either been folded into txpkts or txpkts was flushed and m 2433 * has started a new coalesced work request (as the first frame in a fresh 2434 * txpkts). 2435 * 2436 * Returns non-zero to indicate a failure - caller is responsible for 2437 * transmitting m, if there was anything in txpkts it has been flushed. 2438 */ 2439 static int 2440 add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m, 2441 struct txinfo *txinfo) 2442 { 2443 struct sge_eq *eq = &txq->eq; 2444 int can_coalesce; 2445 struct tx_sdesc *txsd; 2446 uint8_t flits; 2447 2448 TXQ_LOCK_ASSERT_OWNED(txq); 2449 2450 if (txpkts->npkt > 0) { 2451 flits = TXPKTS_PKT_HDR + txinfo->nflits; 2452 can_coalesce = (txinfo->flags & HW_LSO) == 0 && 2453 txpkts->nflits + flits <= TX_WR_FLITS && 2454 txpkts->nflits + flits <= eq->avail * 8 && 2455 txpkts->plen + txinfo->len < 65536; 2456 2457 if (can_coalesce != 0) { 2458 txpkts->tail->b_next = m; 2459 txpkts->tail = m; 2460 txpkts->npkt++; 2461 txpkts->nflits += flits; 2462 txpkts->plen += txinfo->len; 2463 2464 txsd = &txq->sdesc[eq->pidx]; 2465 txsd->txb_used += txinfo->txb_used; 2466 txsd->hdls_used += txinfo->hdls_used; 2467 2468 return (0); 2469 } 2470 2471 /* 2472 * Couldn't coalesce m into txpkts. The first order of business 2473 * is to send txpkts on its way. Then we'll revisit m. 2474 */ 2475 write_txpkts_wr(txq, txpkts); 2476 } 2477 2478 /* 2479 * Check if we can start a new coalesced tx work request with m as 2480 * the first packet in it. 2481 */ 2482 2483 ASSERT(txpkts->npkt == 0); 2484 ASSERT(txinfo->len < 65536); 2485 2486 flits = TXPKTS_WR_HDR + txinfo->nflits; 2487 can_coalesce = (txinfo->flags & HW_LSO) == 0 && 2488 flits <= eq->avail * 8 && flits <= TX_WR_FLITS; 2489 2490 if (can_coalesce == 0) 2491 return (EINVAL); 2492 2493 /* 2494 * Start a fresh coalesced tx WR with m as the first frame in it. 2495 */ 2496 txpkts->tail = m; 2497 txpkts->npkt = 1; 2498 txpkts->nflits = flits; 2499 txpkts->flitp = &eq->desc[eq->pidx].flit[2]; 2500 txpkts->plen = txinfo->len; 2501 2502 txsd = &txq->sdesc[eq->pidx]; 2503 txsd->m = m; 2504 txsd->txb_used = txinfo->txb_used; 2505 txsd->hdls_used = txinfo->hdls_used; 2506 2507 return (0); 2508 } 2509 2510 /* 2511 * Note that write_txpkts_wr can never run out of hardware descriptors (but 2512 * write_txpkt_wr can). add_to_txpkts ensures that a frame is accepted for 2513 * coalescing only if sufficient hardware descriptors are available. 2514 */ 2515 static void 2516 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts) 2517 { 2518 struct sge_eq *eq = &txq->eq; 2519 struct fw_eth_tx_pkts_wr *wr; 2520 struct tx_sdesc *txsd; 2521 uint32_t ctrl; 2522 uint16_t ndesc; 2523 2524 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */ 2525 2526 ndesc = howmany(txpkts->nflits, 8); 2527 2528 wr = (void *)&eq->desc[eq->pidx]; 2529 wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR) | 2530 V_FW_WR_IMMDLEN(0)); /* immdlen does not matter in this WR */ 2531 ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2)); 2532 if (eq->avail == ndesc) 2533 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ; 2534 wr->equiq_to_len16 = cpu_to_be32(ctrl); 2535 wr->plen = cpu_to_be16(txpkts->plen); 2536 wr->npkt = txpkts->npkt; 2537 wr->r3 = wr->type = 0; 2538 2539 /* Everything else already written */ 2540 2541 txsd = &txq->sdesc[eq->pidx]; 2542 txsd->desc_used = ndesc; 2543 2544 txq->txb_used += txsd->txb_used / TXB_CHUNK; 2545 txq->hdl_used += txsd->hdls_used; 2546 2547 ASSERT(eq->avail >= ndesc); 2548 2549 eq->pending += ndesc; 2550 eq->avail -= ndesc; 2551 eq->pidx += ndesc; 2552 if (eq->pidx >= eq->cap) 2553 eq->pidx -= eq->cap; 2554 2555 txq->txpkts_pkts += txpkts->npkt; 2556 txq->txpkts_wrs++; 2557 txpkts->npkt = 0; /* emptied */ 2558 } 2559 2560 static int 2561 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m, 2562 struct txinfo *txinfo) 2563 { 2564 struct sge_eq *eq = &txq->eq; 2565 struct fw_eth_tx_pkt_wr *wr; 2566 struct cpl_tx_pkt_core *cpl; 2567 uint32_t ctrl; /* used in many unrelated places */ 2568 uint64_t ctrl1; 2569 int nflits, ndesc; 2570 struct tx_sdesc *txsd; 2571 caddr_t dst; 2572 2573 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */ 2574 2575 /* 2576 * Do we have enough flits to send this frame out? 2577 */ 2578 ctrl = sizeof (struct cpl_tx_pkt_core); 2579 if (txinfo->flags & HW_LSO) { 2580 nflits = TXPKT_LSO_WR_HDR; 2581 ctrl += sizeof (struct cpl_tx_pkt_lso); 2582 } else 2583 nflits = TXPKT_WR_HDR; 2584 if (txinfo->nsegs > 0) 2585 nflits += txinfo->nflits; 2586 else { 2587 nflits += howmany(txinfo->len, 8); 2588 ctrl += txinfo->len; 2589 } 2590 ndesc = howmany(nflits, 8); 2591 if (ndesc > eq->avail) 2592 return (ENOMEM); 2593 2594 /* Firmware work request header */ 2595 wr = (void *)&eq->desc[eq->pidx]; 2596 wr->op_immdlen = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | 2597 V_FW_WR_IMMDLEN(ctrl)); 2598 ctrl = V_FW_WR_LEN16(howmany(nflits, 2)); 2599 if (eq->avail == ndesc) 2600 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ; 2601 wr->equiq_to_len16 = cpu_to_be32(ctrl); 2602 wr->r3 = 0; 2603 2604 if (txinfo->flags & HW_LSO) { 2605 struct cpl_tx_pkt_lso *lso = (void *)(wr + 1); 2606 char *p = (void *)m->b_rptr; 2607 ctrl = V_LSO_OPCODE((u32)CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | 2608 F_LSO_LAST_SLICE; 2609 2610 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 2611 if (((struct ether_header *)p)->ether_type == 2612 htons(ETHERTYPE_VLAN)) { 2613 ctrl |= V_LSO_ETHHDR_LEN(1); 2614 p += sizeof (struct ether_vlan_header); 2615 } else 2616 p += sizeof (struct ether_header); 2617 2618 /* LINTED: E_BAD_PTR_CAST_ALIGN for IPH_HDR_LENGTH() */ 2619 ctrl |= V_LSO_IPHDR_LEN(IPH_HDR_LENGTH(p) / 4); 2620 /* LINTED: E_BAD_PTR_CAST_ALIGN for IPH_HDR_LENGTH() */ 2621 p += IPH_HDR_LENGTH(p); 2622 ctrl |= V_LSO_TCPHDR_LEN(TCP_HDR_LENGTH((tcph_t *)p) / 4); 2623 2624 lso->lso_ctrl = cpu_to_be32(ctrl); 2625 lso->ipid_ofst = cpu_to_be16(0); 2626 lso->mss = cpu_to_be16(txinfo->mss); 2627 lso->seqno_offset = cpu_to_be32(0); 2628 lso->len = cpu_to_be32(txinfo->len); 2629 2630 cpl = (void *)(lso + 1); 2631 2632 txq->tso_wrs++; 2633 } else 2634 cpl = (void *)(wr + 1); 2635 2636 /* Checksum offload */ 2637 ctrl1 = 0; 2638 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM)) 2639 ctrl1 |= F_TXPKT_IPCSUM_DIS; 2640 if (!(txinfo->flags & HCK_FULLCKSUM)) 2641 ctrl1 |= F_TXPKT_L4CSUM_DIS; 2642 if (ctrl1 == 0) 2643 txq->txcsum++; /* some hardware assistance provided */ 2644 2645 /* CPL header */ 2646 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) | 2647 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf)); 2648 cpl->pack = 0; 2649 cpl->len = cpu_to_be16(txinfo->len); 2650 cpl->ctrl1 = cpu_to_be64(ctrl1); 2651 2652 /* Software descriptor */ 2653 txsd = &txq->sdesc[eq->pidx]; 2654 txsd->m = m; 2655 txsd->txb_used = txinfo->txb_used; 2656 txsd->hdls_used = txinfo->hdls_used; 2657 /* LINTED: E_ASSIGN_NARROW_CONV */ 2658 txsd->desc_used = ndesc; 2659 2660 txq->txb_used += txinfo->txb_used / TXB_CHUNK; 2661 txq->hdl_used += txinfo->hdls_used; 2662 2663 eq->pending += ndesc; 2664 eq->avail -= ndesc; 2665 eq->pidx += ndesc; 2666 if (eq->pidx >= eq->cap) 2667 eq->pidx -= eq->cap; 2668 2669 /* SGL */ 2670 dst = (void *)(cpl + 1); 2671 if (txinfo->nsegs > 0) { 2672 txq->sgl_wrs++; 2673 copy_to_txd(eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8); 2674 2675 /* Need to zero-pad to a 16 byte boundary if not on one */ 2676 if ((uintptr_t)dst & 0xf) 2677 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 2678 *(uint64_t *)dst = 0; 2679 2680 } else { 2681 txq->imm_wrs++; 2682 #ifdef DEBUG 2683 ctrl = txinfo->len; 2684 #endif 2685 for (; m; m = m->b_cont) { 2686 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m)); 2687 #ifdef DEBUG 2688 ctrl -= MBLKL(m); 2689 #endif 2690 } 2691 ASSERT(ctrl == 0); 2692 } 2693 2694 txq->txpkt_wrs++; 2695 return (0); 2696 } 2697 2698 static inline void 2699 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq, 2700 struct txpkts *txpkts, struct txinfo *txinfo) 2701 { 2702 struct ulp_txpkt *ulpmc; 2703 struct ulptx_idata *ulpsc; 2704 struct cpl_tx_pkt_core *cpl; 2705 uintptr_t flitp, start, end; 2706 uint64_t ctrl; 2707 caddr_t dst; 2708 2709 ASSERT(txpkts->npkt > 0); 2710 2711 start = (uintptr_t)txq->eq.desc; 2712 end = (uintptr_t)txq->eq.spg; 2713 2714 /* Checksum offload */ 2715 ctrl = 0; 2716 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM)) 2717 ctrl |= F_TXPKT_IPCSUM_DIS; 2718 if (!(txinfo->flags & HCK_FULLCKSUM)) 2719 ctrl |= F_TXPKT_L4CSUM_DIS; 2720 if (ctrl == 0) 2721 txq->txcsum++; /* some hardware assistance provided */ 2722 2723 /* 2724 * The previous packet's SGL must have ended at a 16 byte boundary (this 2725 * is required by the firmware/hardware). It follows that flitp cannot 2726 * wrap around between the ULPTX master command and ULPTX subcommand (8 2727 * bytes each), and that it can not wrap around in the middle of the 2728 * cpl_tx_pkt_core either. 2729 */ 2730 flitp = (uintptr_t)txpkts->flitp; 2731 ASSERT((flitp & 0xf) == 0); 2732 2733 /* ULP master command */ 2734 ulpmc = (void *)flitp; 2735 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); 2736 ulpmc->len = htonl(howmany(sizeof (*ulpmc) + sizeof (*ulpsc) + 2737 sizeof (*cpl) + 8 * txinfo->nflits, 16)); 2738 2739 /* ULP subcommand */ 2740 ulpsc = (void *)(ulpmc + 1); 2741 ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) | 2742 F_ULP_TX_SC_MORE); 2743 ulpsc->len = cpu_to_be32(sizeof (struct cpl_tx_pkt_core)); 2744 2745 flitp += sizeof (*ulpmc) + sizeof (*ulpsc); 2746 if (flitp == end) 2747 flitp = start; 2748 2749 /* CPL_TX_PKT */ 2750 cpl = (void *)flitp; 2751 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) | 2752 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf)); 2753 cpl->pack = 0; 2754 cpl->len = cpu_to_be16(txinfo->len); 2755 cpl->ctrl1 = cpu_to_be64(ctrl); 2756 2757 flitp += sizeof (*cpl); 2758 if (flitp == end) 2759 flitp = start; 2760 2761 /* SGL for this frame */ 2762 dst = (caddr_t)flitp; 2763 copy_to_txd(&txq->eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8); 2764 flitp = (uintptr_t)dst; 2765 2766 /* Zero pad and advance to a 16 byte boundary if not already at one. */ 2767 if (flitp & 0xf) { 2768 2769 /* no matter what, flitp should be on an 8 byte boundary */ 2770 ASSERT((flitp & 0x7) == 0); 2771 2772 *(uint64_t *)flitp = 0; 2773 flitp += sizeof (uint64_t); 2774 txpkts->nflits++; 2775 } 2776 2777 if (flitp == end) 2778 flitp = start; 2779 2780 txpkts->flitp = (void *)flitp; 2781 } 2782 2783 static inline void 2784 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) 2785 { 2786 if ((uintptr_t)(*to) + len <= (uintptr_t)eq->spg) { 2787 bcopy(from, *to, len); 2788 (*to) += len; 2789 } else { 2790 int portion = (uintptr_t)eq->spg - (uintptr_t)(*to); 2791 2792 bcopy(from, *to, portion); 2793 from += portion; 2794 portion = len - portion; /* remaining */ 2795 bcopy(from, (void *)eq->desc, portion); 2796 (*to) = (caddr_t)eq->desc + portion; 2797 } 2798 } 2799 2800 static inline void 2801 ring_tx_db(struct adapter *sc, struct sge_eq *eq) 2802 { 2803 if (eq->pending > eq->pidx) { 2804 int offset = eq->cap - (eq->pending - eq->pidx); 2805 2806 /* pidx has wrapped around since last doorbell */ 2807 2808 (void) ddi_dma_sync(eq->desc_dhdl, 2809 offset * sizeof (struct tx_desc), 0, 2810 DDI_DMA_SYNC_FORDEV); 2811 (void) ddi_dma_sync(eq->desc_dhdl, 2812 0, eq->pidx * sizeof (struct tx_desc), 2813 DDI_DMA_SYNC_FORDEV); 2814 } else if (eq->pending > 0) { 2815 (void) ddi_dma_sync(eq->desc_dhdl, 2816 (eq->pidx - eq->pending) * sizeof (struct tx_desc), 2817 eq->pending * sizeof (struct tx_desc), 2818 DDI_DMA_SYNC_FORDEV); 2819 } 2820 2821 membar_producer(); 2822 2823 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), 2824 V_QID(eq->cntxt_id) | V_PIDX(eq->pending)); 2825 2826 eq->pending = 0; 2827 } 2828 2829 static int 2830 reclaim_tx_descs(struct sge_txq *txq, int howmany) 2831 { 2832 struct tx_sdesc *txsd; 2833 uint_t cidx, can_reclaim, reclaimed, txb_freed, hdls_freed; 2834 struct sge_eq *eq = &txq->eq; 2835 2836 EQ_LOCK_ASSERT_OWNED(eq); 2837 2838 cidx = eq->spg->cidx; /* stable snapshot */ 2839 cidx = be16_to_cpu(cidx); 2840 2841 if (cidx >= eq->cidx) 2842 can_reclaim = cidx - eq->cidx; 2843 else 2844 can_reclaim = cidx + eq->cap - eq->cidx; 2845 2846 if (can_reclaim == 0) 2847 return (0); 2848 2849 txb_freed = hdls_freed = reclaimed = 0; 2850 do { 2851 int ndesc; 2852 2853 txsd = &txq->sdesc[eq->cidx]; 2854 ndesc = txsd->desc_used; 2855 2856 /* Firmware doesn't return "partial" credits. */ 2857 ASSERT(can_reclaim >= ndesc); 2858 2859 /* 2860 * We always keep mblk around, even for immediate data. If mblk 2861 * is NULL, this has to be the software descriptor for a credit 2862 * flush work request. 2863 */ 2864 if (txsd->m != NULL) 2865 freemsgchain(txsd->m); 2866 #ifdef DEBUG 2867 else { 2868 ASSERT(txsd->txb_used == 0); 2869 ASSERT(txsd->hdls_used == 0); 2870 ASSERT(ndesc == 1); 2871 } 2872 #endif 2873 2874 txb_freed += txsd->txb_used; 2875 hdls_freed += txsd->hdls_used; 2876 reclaimed += ndesc; 2877 2878 eq->cidx += ndesc; 2879 if (eq->cidx >= eq->cap) 2880 eq->cidx -= eq->cap; 2881 2882 can_reclaim -= ndesc; 2883 2884 } while (can_reclaim && reclaimed < howmany); 2885 2886 eq->avail += reclaimed; 2887 ASSERT(eq->avail < eq->cap); /* avail tops out at (cap - 1) */ 2888 2889 txq->txb_avail += txb_freed; 2890 2891 txq->tx_dhdl_avail += hdls_freed; 2892 ASSERT(txq->tx_dhdl_avail <= txq->tx_dhdl_total); 2893 for (; hdls_freed; hdls_freed--) { 2894 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_cidx]); 2895 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total) 2896 txq->tx_dhdl_cidx = 0; 2897 } 2898 2899 return (reclaimed); 2900 } 2901 2902 static void 2903 write_txqflush_wr(struct sge_txq *txq) 2904 { 2905 struct sge_eq *eq = &txq->eq; 2906 struct fw_eq_flush_wr *wr; 2907 struct tx_sdesc *txsd; 2908 2909 EQ_LOCK_ASSERT_OWNED(eq); 2910 ASSERT(eq->avail > 0); 2911 2912 wr = (void *)&eq->desc[eq->pidx]; 2913 bzero(wr, sizeof (*wr)); 2914 wr->opcode = FW_EQ_FLUSH_WR; 2915 wr->equiq_to_len16 = cpu_to_be32(V_FW_WR_LEN16(sizeof (*wr) / 16) | 2916 F_FW_WR_EQUEQ | F_FW_WR_EQUIQ); 2917 2918 txsd = &txq->sdesc[eq->pidx]; 2919 txsd->m = NULL; 2920 txsd->txb_used = 0; 2921 txsd->hdls_used = 0; 2922 txsd->desc_used = 1; 2923 2924 eq->pending++; 2925 eq->avail--; 2926 if (++eq->pidx == eq->cap) 2927 eq->pidx = 0; 2928 } 2929 2930 static int 2931 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m) 2932 { 2933 struct sge_rxq *rxq = (void *)iq; 2934 struct mblk_pair chain = {0}; 2935 const struct cpl_rx_pkt *cpl = (const void *)(rss + 1); 2936 2937 iq->intr_next = iq->intr_params; 2938 2939 m->b_rptr += FL_PKTSHIFT; 2940 2941 /* TODO: what about cpl->ip_frag? */ 2942 if (cpl->csum_calc && !cpl->err_vec && !cpl->ip_frag) { 2943 mac_hcksum_set(m, 0, 0, 0, 0xffff, 2944 HCK_FULLCKSUM_OK | HCK_FULLCKSUM | 2945 HCK_IPV4_HDRCKSUM_OK); 2946 rxq->rxcsum++; 2947 } 2948 2949 /* Add to the chain that we'll send up */ 2950 if (chain.head != NULL) 2951 chain.tail->b_next = m; 2952 else 2953 chain.head = m; 2954 chain.tail = m; 2955 2956 t4_mac_rx(rxq->port, rxq, chain.head); 2957 2958 return (0); 2959 } 2960 2961 #define FL_HW_IDX(idx) ((idx) >> 3) 2962 2963 static inline void 2964 ring_fl_db(struct adapter *sc, struct sge_fl *fl) 2965 { 2966 int desc_start, desc_last, ndesc; 2967 2968 ndesc = FL_HW_IDX(fl->pending); 2969 2970 /* Hold back one credit if pidx = cidx */ 2971 if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx)) 2972 ndesc--; 2973 2974 /* 2975 * There are chances of ndesc modified above (to avoid pidx = cidx). 2976 * If there is nothing to post, return. 2977 */ 2978 if (ndesc <= 0) 2979 return; 2980 2981 desc_last = FL_HW_IDX(fl->pidx); 2982 2983 if (fl->pidx < fl->pending) { 2984 /* There was a wrap */ 2985 desc_start = FL_HW_IDX(fl->pidx + fl->cap - fl->pending); 2986 2987 /* From desc_start to the end of list */ 2988 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 0, 2989 DDI_DMA_SYNC_FORDEV); 2990 2991 /* From start of list to the desc_last */ 2992 if (desc_last != 0) 2993 (void) ddi_dma_sync(fl->dhdl, 0, desc_last * 2994 RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV); 2995 } else { 2996 /* There was no wrap, sync from start_desc to last_desc */ 2997 desc_start = FL_HW_IDX(fl->pidx - fl->pending); 2998 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 2999 ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV); 3000 } 3001 3002 membar_producer(); 3003 3004 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), F_DBPRIO | 3005 V_QID(fl->cntxt_id) | V_PIDX(ndesc)); 3006 3007 /* 3008 * Update pending count: 3009 * Deduct the number of descriptors posted 3010 */ 3011 fl->pending -= ndesc * 8; 3012 } 3013 3014 /* ARGSUSED */ 3015 static int 3016 handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m) 3017 { 3018 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1); 3019 3020 ASSERT(m == NULL); 3021 3022 if (cpl->type == FW6_TYPE_CMD_RPL) 3023 (void) t4_handle_fw_rpl(iq->adapter, cpl->data); 3024 3025 return (0); 3026 } 3027 3028 int 3029 t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count, 3030 int flags) 3031 { 3032 int i, rc; 3033 3034 txmaps->map_total = count; 3035 txmaps->map_avail = txmaps->map_cidx = txmaps->map_pidx = 0; 3036 3037 txmaps->map = kmem_zalloc(sizeof (ddi_dma_handle_t) * 3038 txmaps->map_total, flags); 3039 3040 for (i = 0; i < count; i++) { 3041 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx, 3042 DDI_DMA_SLEEP, 0, &txmaps->map[i]); 3043 if (rc != DDI_SUCCESS) { 3044 cxgb_printf(sc->dip, CE_WARN, 3045 "%s: failed to allocate DMA handle (%d)", 3046 __func__, rc); 3047 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 3048 } 3049 txmaps->map_avail++; 3050 } 3051 3052 return (0); 3053 } 3054 3055 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG) 3056 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR) 3057 #define KS_U_SET(x, y) kstatp->x.value.ul = (y) 3058 #define KS_U_FROM(x, y) kstatp->x.value.ul = (y)->x 3059 #define KS_C_SET(x, ...) \ 3060 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__) 3061 3062 /* 3063 * cxgbe:X:config 3064 */ 3065 struct cxgbe_port_config_kstats { 3066 kstat_named_t idx; 3067 kstat_named_t nrxq; 3068 kstat_named_t ntxq; 3069 kstat_named_t first_rxq; 3070 kstat_named_t first_txq; 3071 kstat_named_t controller; 3072 kstat_named_t factory_mac_address; 3073 }; 3074 3075 /* 3076 * cxgbe:X:info 3077 */ 3078 struct cxgbe_port_info_kstats { 3079 kstat_named_t transceiver; 3080 kstat_named_t rx_ovflow0; 3081 kstat_named_t rx_ovflow1; 3082 kstat_named_t rx_ovflow2; 3083 kstat_named_t rx_ovflow3; 3084 kstat_named_t rx_trunc0; 3085 kstat_named_t rx_trunc1; 3086 kstat_named_t rx_trunc2; 3087 kstat_named_t rx_trunc3; 3088 kstat_named_t tx_pause; 3089 kstat_named_t rx_pause; 3090 }; 3091 3092 static kstat_t * 3093 setup_port_config_kstats(struct port_info *pi) 3094 { 3095 kstat_t *ksp; 3096 struct cxgbe_port_config_kstats *kstatp; 3097 int ndata; 3098 dev_info_t *pdip = ddi_get_parent(pi->dip); 3099 uint8_t *ma = &pi->hw_addr[0]; 3100 3101 ndata = sizeof (struct cxgbe_port_config_kstats) / 3102 sizeof (kstat_named_t); 3103 3104 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "config", 3105 "net", KSTAT_TYPE_NAMED, ndata, 0); 3106 if (ksp == NULL) { 3107 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats."); 3108 return (NULL); 3109 } 3110 3111 kstatp = (struct cxgbe_port_config_kstats *)ksp->ks_data; 3112 3113 KS_UINIT(idx); 3114 KS_UINIT(nrxq); 3115 KS_UINIT(ntxq); 3116 KS_UINIT(first_rxq); 3117 KS_UINIT(first_txq); 3118 KS_CINIT(controller); 3119 KS_CINIT(factory_mac_address); 3120 3121 KS_U_SET(idx, pi->port_id); 3122 KS_U_SET(nrxq, pi->nrxq); 3123 KS_U_SET(ntxq, pi->ntxq); 3124 KS_U_SET(first_rxq, pi->first_rxq); 3125 KS_U_SET(first_txq, pi->first_txq); 3126 KS_C_SET(controller, "%s%d", ddi_driver_name(pdip), 3127 ddi_get_instance(pdip)); 3128 KS_C_SET(factory_mac_address, "%02X%02X%02X%02X%02X%02X", 3129 ma[0], ma[1], ma[2], ma[3], ma[4], ma[5]); 3130 3131 /* Do NOT set ksp->ks_update. These kstats do not change. */ 3132 3133 /* Install the kstat */ 3134 ksp->ks_private = (void *)pi; 3135 kstat_install(ksp); 3136 3137 return (ksp); 3138 } 3139 3140 static kstat_t * 3141 setup_port_info_kstats(struct port_info *pi) 3142 { 3143 kstat_t *ksp; 3144 struct cxgbe_port_info_kstats *kstatp; 3145 int ndata; 3146 3147 ndata = sizeof (struct cxgbe_port_info_kstats) / sizeof (kstat_named_t); 3148 3149 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "info", 3150 "net", KSTAT_TYPE_NAMED, ndata, 0); 3151 if (ksp == NULL) { 3152 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats."); 3153 return (NULL); 3154 } 3155 3156 kstatp = (struct cxgbe_port_info_kstats *)ksp->ks_data; 3157 3158 KS_CINIT(transceiver); 3159 KS_UINIT(rx_ovflow0); 3160 KS_UINIT(rx_ovflow1); 3161 KS_UINIT(rx_ovflow2); 3162 KS_UINIT(rx_ovflow3); 3163 KS_UINIT(rx_trunc0); 3164 KS_UINIT(rx_trunc1); 3165 KS_UINIT(rx_trunc2); 3166 KS_UINIT(rx_trunc3); 3167 KS_UINIT(tx_pause); 3168 KS_UINIT(rx_pause); 3169 3170 /* Install the kstat */ 3171 ksp->ks_update = update_port_info_kstats; 3172 ksp->ks_private = (void *)pi; 3173 kstat_install(ksp); 3174 3175 return (ksp); 3176 } 3177 3178 static int 3179 update_port_info_kstats(kstat_t *ksp, int rw) 3180 { 3181 struct cxgbe_port_info_kstats *kstatp = 3182 (struct cxgbe_port_info_kstats *)ksp->ks_data; 3183 struct port_info *pi = ksp->ks_private; 3184 static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX", 3185 "active TWINAX", "LRM" }; 3186 uint32_t bgmap; 3187 3188 if (rw == KSTAT_WRITE) 3189 return (0); 3190 3191 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 3192 KS_C_SET(transceiver, "unplugged"); 3193 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 3194 KS_C_SET(transceiver, "unknown"); 3195 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 3196 KS_C_SET(transceiver, "unsupported"); 3197 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str)) 3198 KS_C_SET(transceiver, "%s", mod_str[pi->mod_type]); 3199 else 3200 KS_C_SET(transceiver, "type %d", pi->mod_type); 3201 3202 #define GET_STAT(name) t4_read_reg64(pi->adapter, \ 3203 PORT_REG(pi->port_id, A_MPS_PORT_STAT_##name##_L)) 3204 #define GET_STAT_COM(name) t4_read_reg64(pi->adapter, \ 3205 A_MPS_STAT_##name##_L) 3206 3207 bgmap = G_NUMPORTS(t4_read_reg(pi->adapter, A_MPS_CMN_CTL)); 3208 if (bgmap == 0) 3209 bgmap = (pi->port_id == 0) ? 0xf : 0; 3210 else if (bgmap == 1) 3211 bgmap = (pi->port_id < 2) ? (3 << (2 * pi->port_id)) : 0; 3212 else 3213 bgmap = 1; 3214 3215 KS_U_SET(rx_ovflow0, (bgmap & 1) ? 3216 GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0); 3217 KS_U_SET(rx_ovflow1, (bgmap & 2) ? 3218 GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0); 3219 KS_U_SET(rx_ovflow2, (bgmap & 4) ? 3220 GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0); 3221 KS_U_SET(rx_ovflow3, (bgmap & 8) ? 3222 GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0); 3223 KS_U_SET(rx_trunc0, (bgmap & 1) ? 3224 GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0); 3225 KS_U_SET(rx_trunc1, (bgmap & 2) ? 3226 GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0); 3227 KS_U_SET(rx_trunc2, (bgmap & 4) ? 3228 GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0); 3229 KS_U_SET(rx_trunc3, (bgmap & 8) ? 3230 GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0); 3231 3232 KS_U_SET(tx_pause, GET_STAT(TX_PORT_PAUSE)); 3233 KS_U_SET(rx_pause, GET_STAT(RX_PORT_PAUSE)); 3234 3235 return (0); 3236 3237 } 3238 3239 /* 3240 * cxgbe:X:rxqY 3241 */ 3242 struct rxq_kstats { 3243 kstat_named_t rxcsum; 3244 kstat_named_t nomem; 3245 }; 3246 3247 static kstat_t * 3248 setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, int idx) 3249 { 3250 struct kstat *ksp; 3251 struct rxq_kstats *kstatp; 3252 int ndata; 3253 char str[16]; 3254 3255 ndata = sizeof (struct rxq_kstats) / sizeof (kstat_named_t); 3256 (void) snprintf(str, sizeof (str), "rxq%u", idx); 3257 3258 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "rxq", 3259 KSTAT_TYPE_NAMED, ndata, 0); 3260 if (ksp == NULL) { 3261 cxgb_printf(pi->dip, CE_WARN, 3262 "%s: failed to initialize rxq kstats for queue %d.", 3263 __func__, idx); 3264 return (NULL); 3265 } 3266 3267 kstatp = (struct rxq_kstats *)ksp->ks_data; 3268 3269 KS_UINIT(rxcsum); 3270 KS_UINIT(nomem); 3271 3272 ksp->ks_update = update_rxq_kstats; 3273 ksp->ks_private = (void *)rxq; 3274 kstat_install(ksp); 3275 3276 return (ksp); 3277 } 3278 3279 static int 3280 update_rxq_kstats(kstat_t *ksp, int rw) 3281 { 3282 struct rxq_kstats *kstatp = (struct rxq_kstats *)ksp->ks_data; 3283 struct sge_rxq *rxq = ksp->ks_private; 3284 3285 if (rw == KSTAT_WRITE) 3286 return (0); 3287 3288 KS_U_FROM(rxcsum, rxq); 3289 KS_U_FROM(nomem, rxq); 3290 3291 return (0); 3292 } 3293 3294 /* 3295 * cxgbe:X:txqY 3296 */ 3297 struct txq_kstats { 3298 kstat_named_t txcsum; 3299 kstat_named_t tso_wrs; 3300 kstat_named_t imm_wrs; 3301 kstat_named_t sgl_wrs; 3302 kstat_named_t txpkt_wrs; 3303 kstat_named_t txpkts_wrs; 3304 kstat_named_t txpkts_pkts; 3305 kstat_named_t txb_used; 3306 kstat_named_t hdl_used; 3307 kstat_named_t txb_full; 3308 kstat_named_t dma_hdl_failed; 3309 kstat_named_t dma_map_failed; 3310 kstat_named_t qfull; 3311 kstat_named_t qflush; 3312 kstat_named_t pullup_early; 3313 kstat_named_t pullup_late; 3314 kstat_named_t pullup_failed; 3315 }; 3316 3317 static kstat_t * 3318 setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, int idx) 3319 { 3320 struct kstat *ksp; 3321 struct txq_kstats *kstatp; 3322 int ndata; 3323 char str[16]; 3324 3325 ndata = sizeof (struct txq_kstats) / sizeof (kstat_named_t); 3326 (void) snprintf(str, sizeof (str), "txq%u", idx); 3327 3328 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "txq", 3329 KSTAT_TYPE_NAMED, ndata, 0); 3330 if (ksp == NULL) { 3331 cxgb_printf(pi->dip, CE_WARN, 3332 "%s: failed to initialize txq kstats for queue %d.", 3333 __func__, idx); 3334 return (NULL); 3335 } 3336 3337 kstatp = (struct txq_kstats *)ksp->ks_data; 3338 3339 KS_UINIT(txcsum); 3340 KS_UINIT(tso_wrs); 3341 KS_UINIT(imm_wrs); 3342 KS_UINIT(sgl_wrs); 3343 KS_UINIT(txpkt_wrs); 3344 KS_UINIT(txpkts_wrs); 3345 KS_UINIT(txpkts_pkts); 3346 KS_UINIT(txb_used); 3347 KS_UINIT(hdl_used); 3348 KS_UINIT(txb_full); 3349 KS_UINIT(dma_hdl_failed); 3350 KS_UINIT(dma_map_failed); 3351 KS_UINIT(qfull); 3352 KS_UINIT(qflush); 3353 KS_UINIT(pullup_early); 3354 KS_UINIT(pullup_late); 3355 KS_UINIT(pullup_failed); 3356 3357 ksp->ks_update = update_txq_kstats; 3358 ksp->ks_private = (void *)txq; 3359 kstat_install(ksp); 3360 3361 return (ksp); 3362 } 3363 3364 static int 3365 update_txq_kstats(kstat_t *ksp, int rw) 3366 { 3367 struct txq_kstats *kstatp = (struct txq_kstats *)ksp->ks_data; 3368 struct sge_txq *txq = ksp->ks_private; 3369 3370 if (rw == KSTAT_WRITE) 3371 return (0); 3372 3373 KS_U_FROM(txcsum, txq); 3374 KS_U_FROM(tso_wrs, txq); 3375 KS_U_FROM(imm_wrs, txq); 3376 KS_U_FROM(sgl_wrs, txq); 3377 KS_U_FROM(txpkt_wrs, txq); 3378 KS_U_FROM(txpkts_wrs, txq); 3379 KS_U_FROM(txpkts_pkts, txq); 3380 KS_U_FROM(txb_used, txq); 3381 KS_U_FROM(hdl_used, txq); 3382 KS_U_FROM(txb_full, txq); 3383 KS_U_FROM(dma_hdl_failed, txq); 3384 KS_U_FROM(dma_map_failed, txq); 3385 KS_U_FROM(qfull, txq); 3386 KS_U_FROM(qflush, txq); 3387 KS_U_FROM(pullup_early, txq); 3388 KS_U_FROM(pullup_late, txq); 3389 KS_U_FROM(pullup_failed, txq); 3390 3391 return (0); 3392 } 3393