1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * This file is part of the Chelsio T4 support code.
14 *
15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved.
16 *
17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
20 * release for licensing terms and conditions.
21 */
22
23 /*
24 * Copyright 2025 Oxide Computer Company
25 */
26
27 #include <sys/ddi.h>
28 #include <sys/sunddi.h>
29 #include <sys/sunndi.h>
30 #include <sys/atomic.h>
31 #include <sys/dlpi.h>
32 #include <sys/pattr.h>
33 #include <sys/strsubr.h>
34 #include <sys/stream.h>
35 #include <sys/strsun.h>
36 #include <inet/ip.h>
37 #include <inet/tcp.h>
38
39 #include "version.h"
40 #include "common/common.h"
41 #include "common/t4_msg.h"
42 #include "common/t4_regs.h"
43 #include "common/t4_regs_values.h"
44
45 /* TODO: Tune. */
46 int rx_buf_size = 8192;
47 int tx_copy_threshold = 256;
48 uint16_t rx_copy_threshold = 256;
49
50 /* Used to track coalesced tx work request */
51 struct txpkts {
52 mblk_t *tail; /* head is in the software descriptor */
53 uint64_t *flitp; /* ptr to flit where next pkt should start */
54 uint8_t npkt; /* # of packets in this work request */
55 uint8_t nflits; /* # of flits used by this work request */
56 uint16_t plen; /* total payload (sum of all packets) */
57 };
58
59 /* All information needed to tx a frame */
60 struct txinfo {
61 uint32_t len; /* Total length of frame */
62 uint32_t flags; /* Checksum and LSO flags */
63 uint32_t mss; /* MSS for LSO */
64 uint8_t nsegs; /* # of segments in the SGL, 0 means imm. tx */
65 uint8_t nflits; /* # of flits needed for the SGL */
66 uint8_t hdls_used; /* # of DMA handles used */
67 uint32_t txb_used; /* txb_space used */
68 mac_ether_offload_info_t meoi; /* pkt hdr info for offloads */
69 struct ulptx_sgl sgl __attribute__((aligned(8)));
70 struct ulptx_sge_pair reserved[TX_SGL_SEGS / 2];
71 };
72
73 struct mblk_pair {
74 mblk_t *head, *tail;
75 };
76
77 struct rxbuf {
78 kmem_cache_t *cache; /* the kmem_cache this rxb came from */
79 ddi_dma_handle_t dhdl;
80 ddi_acc_handle_t ahdl;
81 caddr_t va; /* KVA of buffer */
82 uint64_t ba; /* bus address of buffer */
83 frtn_t freefunc;
84 uint_t buf_size;
85 volatile uint_t ref_cnt;
86 };
87
88 static int service_iq(struct sge_iq *iq, int budget);
89 static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx,
90 int8_t pktc_idx, int qsize, uint8_t esize);
91 static inline void init_fl(struct sge_fl *fl, uint16_t qsize);
92 static inline void init_eq(struct adapter *sc, struct sge_eq *eq,
93 uint16_t eqtype, uint16_t qsize, uint8_t tx_chan, uint16_t iqid);
94 static int alloc_iq_fl(struct port_info *pi, struct sge_iq *iq,
95 struct sge_fl *fl, int intr_idx, int cong);
96 static int free_iq_fl(struct port_info *pi, struct sge_iq *iq,
97 struct sge_fl *fl);
98 static int alloc_fwq(struct adapter *sc);
99 static int free_fwq(struct adapter *sc);
100 static int alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx,
101 int i);
102 static int free_rxq(struct port_info *pi, struct sge_rxq *rxq);
103 static int ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq);
104 static int eth_eq_alloc(struct adapter *sc, struct port_info *pi,
105 struct sge_eq *eq);
106 static int alloc_eq(struct adapter *sc, struct port_info *pi,
107 struct sge_eq *eq);
108 static int free_eq(struct adapter *sc, struct sge_eq *eq);
109 static int alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx);
110 static int free_txq(struct port_info *pi, struct sge_txq *txq);
111 static int alloc_dma_memory(struct adapter *sc, size_t len, int flags,
112 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr,
113 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
114 caddr_t *pva);
115 static int free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl);
116 static int alloc_desc_ring(struct adapter *sc, size_t len, int rw,
117 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
118 caddr_t *pva);
119 static int free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl);
120 static int alloc_tx_copybuffer(struct adapter *sc, size_t len,
121 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
122 caddr_t *pva);
123 static inline bool is_new_response(const struct sge_iq *iq,
124 struct rsp_ctrl **ctrl);
125 static inline void iq_next(struct sge_iq *iq);
126 static int refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs);
127 static void refill_sfl(void *arg);
128 static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl);
129 static void free_fl_bufs(struct sge_fl *fl);
130 static mblk_t *get_fl_payload(struct adapter *sc, struct sge_fl *fl,
131 uint32_t len_newbuf, int *fl_bufs_used);
132 static int get_frame_txinfo(struct sge_txq *txq, mblk_t **fp,
133 struct txinfo *txinfo, int sgl_only);
134 static inline int fits_in_txb(struct sge_txq *txq, int len, int *waste);
135 static inline int copy_into_txb(struct sge_txq *txq, mblk_t *m, int len,
136 struct txinfo *txinfo);
137 static inline void add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len);
138 static inline int add_mblk(struct sge_txq *txq, struct txinfo *txinfo,
139 mblk_t *m, int len);
140 static void free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo);
141 static int add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m,
142 struct txinfo *txinfo);
143 static void write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts);
144 static int write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m,
145 struct txinfo *txinfo);
146 static inline void write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
147 struct txpkts *txpkts, struct txinfo *txinfo);
148 static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to,
149 int len);
150 static inline void ring_tx_db(struct adapter *sc, struct sge_eq *eq);
151 static int reclaim_tx_descs(struct sge_txq *txq, int howmany);
152 static void write_txqflush_wr(struct sge_txq *txq);
153 static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss,
154 mblk_t *m);
155 static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl);
156 static kstat_t *setup_port_config_kstats(struct port_info *pi);
157 static kstat_t *setup_port_info_kstats(struct port_info *pi);
158 static kstat_t *setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq,
159 int idx);
160 static int update_rxq_kstats(kstat_t *ksp, int rw);
161 static int update_port_info_kstats(kstat_t *ksp, int rw);
162 static kstat_t *setup_txq_kstats(struct port_info *pi, struct sge_txq *txq,
163 int idx);
164 static int update_txq_kstats(kstat_t *ksp, int rw);
165 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *,
166 mblk_t *);
167 static int handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss,
168 mblk_t *m);
169
170 static kmem_cache_t *rxbuf_cache_create(struct rxbuf_cache_params *);
171 static struct rxbuf *rxbuf_alloc(kmem_cache_t *, int, uint_t);
172 static void rxbuf_free(struct rxbuf *);
173 static int rxbuf_ctor(void *, void *, int);
174 static void rxbuf_dtor(void *, void *);
175
176 static inline int
reclaimable(struct sge_eq * eq)177 reclaimable(struct sge_eq *eq)
178 {
179 unsigned int cidx;
180
181 cidx = eq->spg->cidx; /* stable snapshot */
182 cidx = be16_to_cpu(cidx);
183
184 if (cidx >= eq->cidx)
185 return (cidx - eq->cidx);
186 else
187 return (cidx + eq->cap - eq->cidx);
188 }
189
190 void
t4_sge_init(struct adapter * sc)191 t4_sge_init(struct adapter *sc)
192 {
193 struct driver_properties *p = &sc->props;
194 ddi_dma_attr_t *dma_attr;
195 ddi_device_acc_attr_t *acc_attr;
196 uint32_t sge_control, sge_conm_ctrl;
197 int egress_threshold;
198
199 /*
200 * Device access and DMA attributes for descriptor rings
201 */
202 acc_attr = &sc->sge.acc_attr_desc;
203 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
204 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
205 acc_attr->devacc_attr_dataorder = DDI_STRICTORDER_ACC;
206
207 dma_attr = &sc->sge.dma_attr_desc;
208 dma_attr->dma_attr_version = DMA_ATTR_V0;
209 dma_attr->dma_attr_addr_lo = 0;
210 dma_attr->dma_attr_addr_hi = UINT64_MAX;
211 dma_attr->dma_attr_count_max = UINT64_MAX;
212 dma_attr->dma_attr_align = 512;
213 dma_attr->dma_attr_burstsizes = 0xfff;
214 dma_attr->dma_attr_minxfer = 1;
215 dma_attr->dma_attr_maxxfer = UINT64_MAX;
216 dma_attr->dma_attr_seg = UINT64_MAX;
217 dma_attr->dma_attr_sgllen = 1;
218 dma_attr->dma_attr_granular = 1;
219 dma_attr->dma_attr_flags = 0;
220
221 /*
222 * Device access and DMA attributes for tx buffers
223 */
224 acc_attr = &sc->sge.acc_attr_tx;
225 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
226 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
227
228 dma_attr = &sc->sge.dma_attr_tx;
229 dma_attr->dma_attr_version = DMA_ATTR_V0;
230 dma_attr->dma_attr_addr_lo = 0;
231 dma_attr->dma_attr_addr_hi = UINT64_MAX;
232 dma_attr->dma_attr_count_max = UINT64_MAX;
233 dma_attr->dma_attr_align = 1;
234 dma_attr->dma_attr_burstsizes = 0xfff;
235 dma_attr->dma_attr_minxfer = 1;
236 dma_attr->dma_attr_maxxfer = UINT64_MAX;
237 dma_attr->dma_attr_seg = UINT64_MAX;
238 dma_attr->dma_attr_sgllen = TX_SGL_SEGS;
239 dma_attr->dma_attr_granular = 1;
240 dma_attr->dma_attr_flags = 0;
241
242 /*
243 * Ingress Padding Boundary and Egress Status Page Size are set up by
244 * t4_fixup_host_params().
245 */
246 sge_control = t4_read_reg(sc, A_SGE_CONTROL);
247 sc->sge.pktshift = G_PKTSHIFT(sge_control);
248 sc->sge.stat_len = (sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64;
249
250 /* t4_nex uses FLM packed mode */
251 sc->sge.fl_align = t4_fl_pkt_align(sc, true);
252
253 /*
254 * Device access and DMA attributes for rx buffers
255 */
256 sc->sge.rxb_params.dip = sc->dip;
257 sc->sge.rxb_params.buf_size = rx_buf_size;
258
259 acc_attr = &sc->sge.rxb_params.acc_attr_rx;
260 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
261 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
262
263 dma_attr = &sc->sge.rxb_params.dma_attr_rx;
264 dma_attr->dma_attr_version = DMA_ATTR_V0;
265 dma_attr->dma_attr_addr_lo = 0;
266 dma_attr->dma_attr_addr_hi = UINT64_MAX;
267 dma_attr->dma_attr_count_max = UINT64_MAX;
268 /*
269 * Low 4 bits of an rx buffer address have a special meaning to the SGE
270 * and an rx buf cannot have an address with any of these bits set.
271 * FL_ALIGN is >= 32 so we're sure things are ok.
272 */
273 dma_attr->dma_attr_align = sc->sge.fl_align;
274 dma_attr->dma_attr_burstsizes = 0xfff;
275 dma_attr->dma_attr_minxfer = 1;
276 dma_attr->dma_attr_maxxfer = UINT64_MAX;
277 dma_attr->dma_attr_seg = UINT64_MAX;
278 dma_attr->dma_attr_sgllen = 1;
279 dma_attr->dma_attr_granular = 1;
280 dma_attr->dma_attr_flags = 0;
281
282 sc->sge.rxbuf_cache = rxbuf_cache_create(&sc->sge.rxb_params);
283
284 /*
285 * A FL with <= fl_starve_thres buffers is starving and a periodic
286 * timer will attempt to refill it. This needs to be larger than the
287 * SGE's Egress Congestion Threshold. If it isn't, then we can get
288 * stuck waiting for new packets while the SGE is waiting for us to
289 * give it more Free List entries. (Note that the SGE's Egress
290 * Congestion Threshold is in units of 2 Free List pointers.) For T4,
291 * there was only a single field to control this. For T5 there's the
292 * original field which now only applies to Unpacked Mode Free List
293 * buffers and a new field which only applies to Packed Mode Free List
294 * buffers.
295 */
296
297 sge_conm_ctrl = t4_read_reg(sc, A_SGE_CONM_CTRL);
298 switch (CHELSIO_CHIP_VERSION(sc->params.chip)) {
299 case CHELSIO_T4:
300 egress_threshold = G_EGRTHRESHOLD(sge_conm_ctrl);
301 break;
302 case CHELSIO_T5:
303 egress_threshold = G_EGRTHRESHOLDPACKING(sge_conm_ctrl);
304 break;
305 case CHELSIO_T6:
306 default:
307 egress_threshold = G_T6_EGRTHRESHOLDPACKING(sge_conm_ctrl);
308 }
309 sc->sge.fl_starve_threshold = 2*egress_threshold + 1;
310
311 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, rx_buf_size);
312
313 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD,
314 V_THRESHOLD_0(p->counter_val[0]) |
315 V_THRESHOLD_1(p->counter_val[1]) |
316 V_THRESHOLD_2(p->counter_val[2]) |
317 V_THRESHOLD_3(p->counter_val[3]));
318
319 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1,
320 V_TIMERVALUE0(us_to_core_ticks(sc, p->timer_val[0])) |
321 V_TIMERVALUE1(us_to_core_ticks(sc, p->timer_val[1])));
322 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3,
323 V_TIMERVALUE2(us_to_core_ticks(sc, p->timer_val[2])) |
324 V_TIMERVALUE3(us_to_core_ticks(sc, p->timer_val[3])));
325 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5,
326 V_TIMERVALUE4(us_to_core_ticks(sc, p->timer_val[4])) |
327 V_TIMERVALUE5(us_to_core_ticks(sc, p->timer_val[5])));
328
329 (void) t4_register_cpl_handler(sc, CPL_FW4_MSG, handle_fw_rpl);
330 (void) t4_register_cpl_handler(sc, CPL_FW6_MSG, handle_fw_rpl);
331 (void) t4_register_cpl_handler(sc, CPL_SGE_EGR_UPDATE,
332 handle_sge_egr_update);
333 (void) t4_register_cpl_handler(sc, CPL_RX_PKT, t4_eth_rx);
334 (void) t4_register_fw_msg_handler(sc, FW6_TYPE_CMD_RPL,
335 t4_handle_fw_rpl);
336 }
337
338 /*
339 * Allocate and initialize the firmware event queue and the forwarded interrupt
340 * queues, if any. The adapter owns all these queues as they are not associated
341 * with any particular port.
342 *
343 * Returns errno on failure. Resources allocated up to that point may still be
344 * allocated. Caller is responsible for cleanup in case this function fails.
345 */
346 int
t4_setup_adapter_queues(struct adapter * sc)347 t4_setup_adapter_queues(struct adapter *sc)
348 {
349 int rc;
350
351 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
352
353 /*
354 * Firmware event queue
355 */
356 rc = alloc_fwq(sc);
357 if (rc != 0)
358 return (rc);
359
360 return (rc);
361 }
362
363 /*
364 * Idempotent
365 */
366 int
t4_teardown_adapter_queues(struct adapter * sc)367 t4_teardown_adapter_queues(struct adapter *sc)
368 {
369
370 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
371
372 (void) free_fwq(sc);
373
374 return (0);
375 }
376
377 static inline int
first_vector(struct port_info * pi)378 first_vector(struct port_info *pi)
379 {
380 struct adapter *sc = pi->adapter;
381 int rc = T4_EXTRA_INTR, i;
382
383 if (sc->intr_count == 1)
384 return (0);
385
386 for_each_port(sc, i) {
387 struct port_info *p = sc->port[i];
388
389 if (i == pi->port_id)
390 break;
391
392 /*
393 * Not compiled with offload support and intr_count > 1. Only
394 * NIC queues exist and they'd better be taking direct
395 * interrupts.
396 */
397 ASSERT(!(sc->flags & INTR_FWD));
398 rc += p->nrxq;
399 }
400 return (rc);
401 }
402
403 /*
404 * Given an arbitrary "index," come up with an iq that can be used by other
405 * queues (of this port) for interrupt forwarding, SGE egress updates, etc.
406 * The iq returned is guaranteed to be something that takes direct interrupts.
407 */
408 static struct sge_iq *
port_intr_iq(struct port_info * pi,int idx)409 port_intr_iq(struct port_info *pi, int idx)
410 {
411 struct adapter *sc = pi->adapter;
412 struct sge *s = &sc->sge;
413 struct sge_iq *iq = NULL;
414
415 if (sc->intr_count == 1)
416 return (&sc->sge.fwq);
417
418 /*
419 * Not compiled with offload support and intr_count > 1. Only NIC
420 * queues exist and they'd better be taking direct interrupts.
421 */
422 ASSERT(!(sc->flags & INTR_FWD));
423
424 idx %= pi->nrxq;
425 iq = &s->rxq[pi->first_rxq + idx].iq;
426
427 return (iq);
428 }
429
430 int
t4_setup_port_queues(struct port_info * pi)431 t4_setup_port_queues(struct port_info *pi)
432 {
433 int rc = 0, i, intr_idx, j;
434 struct sge_rxq *rxq;
435 struct sge_txq *txq;
436 struct adapter *sc = pi->adapter;
437 struct driver_properties *p = &sc->props;
438
439 pi->ksp_config = setup_port_config_kstats(pi);
440 pi->ksp_info = setup_port_info_kstats(pi);
441
442 /* Interrupt vector to start from (when using multiple vectors) */
443 intr_idx = first_vector(pi);
444
445 /*
446 * First pass over all rx queues (NIC and TOE):
447 * a) initialize iq and fl
448 * b) allocate queue iff it will take direct interrupts.
449 */
450
451 for_each_rxq(pi, i, rxq) {
452
453 init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, p->qsize_rxq,
454 RX_IQ_ESIZE);
455
456 init_fl(&rxq->fl, p->qsize_rxq / 8); /* 8 bufs in each entry */
457
458 if ((!(sc->flags & INTR_FWD)) ||
459 (sc->intr_count > 1 && pi->nrxq)) {
460 rxq->iq.flags |= IQ_INTR;
461 rc = alloc_rxq(pi, rxq, intr_idx, i);
462 if (rc != 0)
463 goto done;
464 intr_idx++;
465 }
466
467 }
468
469 /*
470 * Second pass over all rx queues (NIC and TOE). The queues forwarding
471 * their interrupts are allocated now.
472 */
473 j = 0;
474 for_each_rxq(pi, i, rxq) {
475 if (rxq->iq.flags & IQ_INTR)
476 continue;
477
478 intr_idx = port_intr_iq(pi, j)->abs_id;
479
480 rc = alloc_rxq(pi, rxq, intr_idx, i);
481 if (rc != 0)
482 goto done;
483 j++;
484 }
485
486 /*
487 * Now the tx queues. Only one pass needed.
488 */
489 j = 0;
490 for_each_txq(pi, i, txq) {
491 uint16_t iqid;
492
493 iqid = port_intr_iq(pi, j)->cntxt_id;
494 init_eq(sc, &txq->eq, EQ_ETH, p->qsize_txq, pi->tx_chan, iqid);
495 rc = alloc_txq(pi, txq, i);
496 if (rc != 0)
497 goto done;
498 }
499
500 done:
501 if (rc != 0)
502 (void) t4_teardown_port_queues(pi);
503
504 return (rc);
505 }
506
507 /*
508 * Idempotent
509 */
510 int
t4_teardown_port_queues(struct port_info * pi)511 t4_teardown_port_queues(struct port_info *pi)
512 {
513 int i;
514 struct sge_rxq *rxq;
515 struct sge_txq *txq;
516
517 if (pi->ksp_config != NULL) {
518 kstat_delete(pi->ksp_config);
519 pi->ksp_config = NULL;
520 }
521 if (pi->ksp_info != NULL) {
522 kstat_delete(pi->ksp_info);
523 pi->ksp_info = NULL;
524 }
525
526 for_each_txq(pi, i, txq) {
527 (void) free_txq(pi, txq);
528 }
529
530 for_each_rxq(pi, i, rxq) {
531 if ((rxq->iq.flags & IQ_INTR) == 0)
532 (void) free_rxq(pi, rxq);
533 }
534
535 /*
536 * Then take down the rx queues that take direct interrupts.
537 */
538
539 for_each_rxq(pi, i, rxq) {
540 if (rxq->iq.flags & IQ_INTR)
541 (void) free_rxq(pi, rxq);
542 }
543
544 return (0);
545 }
546
547 /* Deals with errors and forwarded interrupts */
548 uint_t
t4_intr_all(caddr_t arg1,caddr_t arg2)549 t4_intr_all(caddr_t arg1, caddr_t arg2)
550 {
551
552 (void) t4_intr_err(arg1, arg2);
553 (void) t4_intr(arg1, arg2);
554
555 return (DDI_INTR_CLAIMED);
556 }
557
558 static void
t4_intr_rx_work(struct sge_iq * iq)559 t4_intr_rx_work(struct sge_iq *iq)
560 {
561 mblk_t *mp = NULL;
562 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
563 RXQ_LOCK(rxq);
564 if (!iq->polling) {
565 mp = t4_ring_rx(rxq, iq->qsize/8);
566 t4_write_reg(iq->adapter, MYPF_REG(A_SGE_PF_GTS),
567 V_INGRESSQID((u32)iq->cntxt_id) |
568 V_SEINTARM(iq->intr_next));
569 }
570 RXQ_UNLOCK(rxq);
571 if (mp != NULL) {
572 mac_rx_ring(rxq->port->mh, rxq->ring_handle, mp,
573 rxq->ring_gen_num);
574 }
575 }
576
577 /* Deals with interrupts on the given ingress queue */
578 /* ARGSUSED */
579 uint_t
t4_intr(caddr_t arg1,caddr_t arg2)580 t4_intr(caddr_t arg1, caddr_t arg2)
581 {
582 struct sge_iq *iq = (struct sge_iq *)arg2;
583 int state;
584
585 /*
586 * Right now receive polling is only enabled for MSI-X and
587 * when we have enough msi-x vectors i.e no interrupt forwarding.
588 */
589 if (iq->adapter->props.multi_rings) {
590 t4_intr_rx_work(iq);
591 } else {
592 state = atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY);
593 if (state == IQS_IDLE) {
594 (void) service_iq(iq, 0);
595 (void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE);
596 }
597 }
598 return (DDI_INTR_CLAIMED);
599 }
600
601 /* Deals with error interrupts */
602 /* ARGSUSED */
603 uint_t
t4_intr_err(caddr_t arg1,caddr_t arg2)604 t4_intr_err(caddr_t arg1, caddr_t arg2)
605 {
606 /* LINTED: E_BAD_PTR_CAST_ALIGN */
607 struct adapter *sc = (struct adapter *)arg1;
608
609 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0);
610 (void) t4_slow_intr_handler(sc);
611
612 return (DDI_INTR_CLAIMED);
613 }
614
615 /*
616 * t4_ring_rx - Process responses from an SGE response queue.
617 *
618 * This function processes responses from an SGE response queue up to the
619 * supplied budget. Responses include received packets as well as control
620 * messages from FW or HW.
621 *
622 * It returns a chain of mblks containing the received data, to be
623 * passed up to mac_rx_ring().
624 */
625 mblk_t *
t4_ring_rx(struct sge_rxq * rxq,int budget)626 t4_ring_rx(struct sge_rxq *rxq, int budget)
627 {
628 struct sge_iq *iq = &rxq->iq;
629 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
630 struct adapter *sc = iq->adapter;
631 struct rsp_ctrl *ctrl;
632 const struct rss_header *rss;
633 int ndescs = 0, fl_bufs_used = 0;
634 int rsp_type;
635 uint32_t lq;
636 mblk_t *mblk_head = NULL, **mblk_tail, *m;
637 struct cpl_rx_pkt *cpl;
638 uint32_t received_bytes = 0, pkt_len = 0;
639 bool csum_ok;
640 uint16_t err_vec;
641
642 mblk_tail = &mblk_head;
643
644 while (is_new_response(iq, &ctrl)) {
645
646 membar_consumer();
647
648 m = NULL;
649 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
650 lq = be32_to_cpu(ctrl->pldbuflen_qid);
651 rss = (const void *)iq->cdesc;
652
653 switch (rsp_type) {
654 case X_RSPD_TYPE_FLBUF:
655
656 ASSERT(iq->flags & IQ_HAS_FL);
657
658 if (CPL_RX_PKT == rss->opcode) {
659 cpl = (void *)(rss + 1);
660 pkt_len = be16_to_cpu(cpl->len);
661
662 if (iq->polling &&
663 ((received_bytes + pkt_len) > budget))
664 goto done;
665
666 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
667 if (m == NULL)
668 goto done;
669
670 iq->intr_next = iq->intr_params;
671 m->b_rptr += sc->sge.pktshift;
672 if (sc->params.tp.rx_pkt_encap) {
673 /* Enabled only in T6 config file */
674 err_vec = G_T6_COMPR_RXERR_VEC(
675 ntohs(cpl->err_vec));
676 } else {
677 err_vec = ntohs(cpl->err_vec);
678 }
679
680 csum_ok = cpl->csum_calc && !err_vec;
681
682 /* TODO: what about cpl->ip_frag? */
683 if (csum_ok && !cpl->ip_frag) {
684 mac_hcksum_set(m, 0, 0, 0, 0xffff,
685 HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
686 HCK_IPV4_HDRCKSUM_OK);
687 rxq->rxcsum++;
688 }
689 rxq->rxpkts++;
690 rxq->rxbytes += pkt_len;
691 received_bytes += pkt_len;
692
693 *mblk_tail = m;
694 mblk_tail = &m->b_next;
695
696 break;
697 }
698
699 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
700 if (m == NULL)
701 goto done;
702 /* FALLTHROUGH */
703
704 case X_RSPD_TYPE_CPL:
705 ASSERT(rss->opcode < NUM_CPL_CMDS);
706 sc->cpl_handler[rss->opcode](iq, rss, m);
707 break;
708
709 default:
710 break;
711 }
712 iq_next(iq);
713 ++ndescs;
714 if (!iq->polling && (ndescs == budget))
715 break;
716 }
717
718 done:
719
720 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
721 V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) |
722 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
723
724 if ((fl_bufs_used > 0) || (iq->flags & IQ_HAS_FL)) {
725 int starved;
726 FL_LOCK(fl);
727 fl->needed += fl_bufs_used;
728 starved = refill_fl(sc, fl, fl->cap / 8);
729 FL_UNLOCK(fl);
730 if (starved)
731 add_fl_to_sfl(sc, fl);
732 }
733 return (mblk_head);
734 }
735
736 /*
737 * Deals with anything and everything on the given ingress queue.
738 */
739 static int
service_iq(struct sge_iq * iq,int budget)740 service_iq(struct sge_iq *iq, int budget)
741 {
742 struct sge_iq *q;
743 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
744 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
745 struct adapter *sc = iq->adapter;
746 struct rsp_ctrl *ctrl;
747 const struct rss_header *rss;
748 int ndescs = 0, limit, fl_bufs_used = 0;
749 int rsp_type;
750 uint32_t lq;
751 int starved;
752 mblk_t *m;
753 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
754
755 limit = budget ? budget : iq->qsize / 8;
756
757 /*
758 * We always come back and check the descriptor ring for new indirect
759 * interrupts and other responses after running a single handler.
760 */
761 for (;;) {
762 while (is_new_response(iq, &ctrl)) {
763
764 membar_consumer();
765
766 m = NULL;
767 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
768 lq = be32_to_cpu(ctrl->pldbuflen_qid);
769 rss = (const void *)iq->cdesc;
770
771 switch (rsp_type) {
772 case X_RSPD_TYPE_FLBUF:
773
774 ASSERT(iq->flags & IQ_HAS_FL);
775
776 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
777 if (m == NULL) {
778 /*
779 * Rearm the iq with a
780 * longer-than-default timer
781 */
782 const uint32_t timer_idx =
783 V_QINTR_TIMER_IDX(SGE_NTIMERS-1);
784 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
785 V_CIDXINC(ndescs) |
786 V_INGRESSQID((u32)iq->cntxt_id) |
787 V_SEINTARM(timer_idx));
788 if (fl_bufs_used > 0) {
789 ASSERT(iq->flags & IQ_HAS_FL);
790 FL_LOCK(fl);
791 fl->needed += fl_bufs_used;
792 starved = refill_fl(sc, fl,
793 fl->cap / 8);
794 FL_UNLOCK(fl);
795 if (starved)
796 add_fl_to_sfl(sc, fl);
797 }
798 return (0);
799 }
800
801 /* FALLTHRU */
802 case X_RSPD_TYPE_CPL:
803
804 ASSERT(rss->opcode < NUM_CPL_CMDS);
805 sc->cpl_handler[rss->opcode](iq, rss, m);
806 break;
807
808 case X_RSPD_TYPE_INTR:
809
810 /*
811 * Interrupts should be forwarded only to queues
812 * that are not forwarding their interrupts.
813 * This means service_iq can recurse but only 1
814 * level deep.
815 */
816 ASSERT(budget == 0);
817
818 q = sc->sge.iqmap[lq - sc->sge.iq_start];
819 if (atomic_cas_uint(&q->state, IQS_IDLE,
820 IQS_BUSY) == IQS_IDLE) {
821 if (service_iq(q, q->qsize / 8) == 0) {
822 (void) atomic_cas_uint(
823 &q->state, IQS_BUSY,
824 IQS_IDLE);
825 } else {
826 STAILQ_INSERT_TAIL(&iql, q,
827 link);
828 }
829 }
830 break;
831
832 default:
833 break;
834 }
835
836 iq_next(iq);
837 if (++ndescs == limit) {
838 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
839 V_CIDXINC(ndescs) |
840 V_INGRESSQID(iq->cntxt_id) |
841 V_SEINTARM(V_QINTR_TIMER_IDX(
842 X_TIMERREG_UPDATE_CIDX)));
843 ndescs = 0;
844
845 if (fl_bufs_used > 0) {
846 ASSERT(iq->flags & IQ_HAS_FL);
847 FL_LOCK(fl);
848 fl->needed += fl_bufs_used;
849 (void) refill_fl(sc, fl, fl->cap / 8);
850 FL_UNLOCK(fl);
851 fl_bufs_used = 0;
852 }
853
854 if (budget != 0)
855 return (EINPROGRESS);
856 }
857 }
858
859 if (STAILQ_EMPTY(&iql) != 0)
860 break;
861
862 /*
863 * Process the head only, and send it to the back of the list if
864 * it's still not done.
865 */
866 q = STAILQ_FIRST(&iql);
867 STAILQ_REMOVE_HEAD(&iql, link);
868 if (service_iq(q, q->qsize / 8) == 0)
869 (void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE);
870 else
871 STAILQ_INSERT_TAIL(&iql, q, link);
872 }
873
874 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
875 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
876
877 if (iq->flags & IQ_HAS_FL) {
878
879 FL_LOCK(fl);
880 fl->needed += fl_bufs_used;
881 starved = refill_fl(sc, fl, fl->cap / 4);
882 FL_UNLOCK(fl);
883 if (starved != 0)
884 add_fl_to_sfl(sc, fl);
885 }
886
887 return (0);
888 }
889
890 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */
891 #define TXPKTS_PKT_HDR ((\
892 sizeof (struct ulp_txpkt) + \
893 sizeof (struct ulptx_idata) + \
894 sizeof (struct cpl_tx_pkt_core)) / 8)
895
896 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */
897 #define TXPKTS_WR_HDR (\
898 sizeof (struct fw_eth_tx_pkts_wr) / 8 + \
899 TXPKTS_PKT_HDR)
900
901 /* Header of a tx WR, before SGL of first packet (in flits) */
902 #define TXPKT_WR_HDR ((\
903 sizeof (struct fw_eth_tx_pkt_wr) + \
904 sizeof (struct cpl_tx_pkt_core)) / 8)
905
906 /* Header of a tx LSO WR, before SGL of first packet (in flits) */
907 #define TXPKT_LSO_WR_HDR ((\
908 sizeof (struct fw_eth_tx_pkt_wr) + \
909 sizeof (struct cpl_tx_pkt_lso_core) + \
910 sizeof (struct cpl_tx_pkt_core)) / 8)
911
912 mblk_t *
t4_eth_tx(void * arg,mblk_t * frame)913 t4_eth_tx(void *arg, mblk_t *frame)
914 {
915 struct sge_txq *txq = (struct sge_txq *)arg;
916 struct port_info *pi = txq->port;
917 struct adapter *sc = pi->adapter;
918 struct sge_eq *eq = &txq->eq;
919 mblk_t *next_frame;
920 int rc, coalescing;
921 struct txpkts txpkts;
922 struct txinfo txinfo;
923
924 txpkts.npkt = 0; /* indicates there's nothing in txpkts */
925 coalescing = 0;
926
927 TXQ_LOCK(txq);
928 if (eq->avail < 8)
929 (void) reclaim_tx_descs(txq, 8);
930 for (; frame; frame = next_frame) {
931
932 if (eq->avail < 8)
933 break;
934
935 next_frame = frame->b_next;
936 frame->b_next = NULL;
937
938 if (next_frame != NULL)
939 coalescing = 1;
940
941 rc = get_frame_txinfo(txq, &frame, &txinfo, coalescing);
942 if (rc != 0) {
943 if (rc == ENOMEM) {
944
945 /* Short of resources, suspend tx */
946
947 frame->b_next = next_frame;
948 break;
949 }
950
951 /*
952 * Unrecoverable error for this frame, throw it
953 * away and move on to the next.
954 */
955
956 freemsg(frame);
957 continue;
958 }
959
960 if (coalescing != 0 &&
961 add_to_txpkts(txq, &txpkts, frame, &txinfo) == 0) {
962
963 /* Successfully absorbed into txpkts */
964
965 write_ulp_cpl_sgl(pi, txq, &txpkts, &txinfo);
966 goto doorbell;
967 }
968
969 /*
970 * We weren't coalescing to begin with, or current frame could
971 * not be coalesced (add_to_txpkts flushes txpkts if a frame
972 * given to it can't be coalesced). Either way there should be
973 * nothing in txpkts.
974 */
975 ASSERT(txpkts.npkt == 0);
976
977 /* We're sending out individual frames now */
978 coalescing = 0;
979
980 if (eq->avail < 8)
981 (void) reclaim_tx_descs(txq, 8);
982 rc = write_txpkt_wr(pi, txq, frame, &txinfo);
983 if (rc != 0) {
984
985 /* Short of hardware descriptors, suspend tx */
986
987 /*
988 * This is an unlikely but expensive failure. We've
989 * done all the hard work (DMA bindings etc.) and now we
990 * can't send out the frame. What's worse, we have to
991 * spend even more time freeing up everything in txinfo.
992 */
993 txq->qfull++;
994 free_txinfo_resources(txq, &txinfo);
995
996 frame->b_next = next_frame;
997 break;
998 }
999
1000 doorbell:
1001 /* Fewer and fewer doorbells as the queue fills up */
1002 if (eq->pending >= (1 << (fls(eq->qsize - eq->avail) / 2))) {
1003 txq->txbytes += txinfo.len;
1004 txq->txpkts++;
1005 ring_tx_db(sc, eq);
1006 }
1007 (void) reclaim_tx_descs(txq, 32);
1008 }
1009
1010 if (txpkts.npkt > 0)
1011 write_txpkts_wr(txq, &txpkts);
1012
1013 /*
1014 * frame not NULL means there was an error but we haven't thrown it
1015 * away. This can happen when we're short of tx descriptors (qfull) or
1016 * maybe even DMA handles (dma_hdl_failed). Either way, a credit flush
1017 * and reclaim will get things going again.
1018 *
1019 * If eq->avail is already 0 we know a credit flush was requested in the
1020 * WR that reduced it to 0 so we don't need another flush (we don't have
1021 * any descriptor for a flush WR anyway, duh).
1022 */
1023 if (frame && eq->avail > 0)
1024 write_txqflush_wr(txq);
1025
1026 if (eq->pending != 0)
1027 ring_tx_db(sc, eq);
1028
1029 (void) reclaim_tx_descs(txq, eq->qsize);
1030 TXQ_UNLOCK(txq);
1031
1032 return (frame);
1033 }
1034
1035 static inline void
init_iq(struct sge_iq * iq,struct adapter * sc,int tmr_idx,int8_t pktc_idx,int qsize,uint8_t esize)1036 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int8_t pktc_idx,
1037 int qsize, uint8_t esize)
1038 {
1039 ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS);
1040 ASSERT(pktc_idx < SGE_NCOUNTERS); /* -ve is ok, means don't use */
1041
1042 iq->flags = 0;
1043 iq->adapter = sc;
1044 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
1045 iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
1046 if (pktc_idx >= 0) {
1047 iq->intr_params |= F_QINTR_CNT_EN;
1048 iq->intr_pktc_idx = pktc_idx;
1049 }
1050 iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */
1051 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */
1052 }
1053
1054 static inline void
init_fl(struct sge_fl * fl,uint16_t qsize)1055 init_fl(struct sge_fl *fl, uint16_t qsize)
1056 {
1057
1058 fl->qsize = qsize;
1059 fl->allocb_fail = 0;
1060 }
1061
1062 static inline void
init_eq(struct adapter * sc,struct sge_eq * eq,uint16_t eqtype,uint16_t qsize,uint8_t tx_chan,uint16_t iqid)1063 init_eq(struct adapter *sc, struct sge_eq *eq, uint16_t eqtype, uint16_t qsize,
1064 uint8_t tx_chan, uint16_t iqid)
1065 {
1066 struct sge *s = &sc->sge;
1067 uint32_t r;
1068
1069 ASSERT(tx_chan < NCHAN);
1070 ASSERT(eqtype <= EQ_TYPEMASK);
1071
1072 if (is_t5(sc->params.chip)) {
1073 r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF);
1074 r >>= S_QUEUESPERPAGEPF0 +
1075 (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf;
1076 s->s_qpp = r & M_QUEUESPERPAGEPF0;
1077 }
1078
1079 eq->flags = eqtype & EQ_TYPEMASK;
1080 eq->tx_chan = tx_chan;
1081 eq->iqid = iqid;
1082 eq->qsize = qsize;
1083 }
1084
1085 /*
1086 * Allocates the ring for an ingress queue and an optional freelist. If the
1087 * freelist is specified it will be allocated and then associated with the
1088 * ingress queue.
1089 *
1090 * Returns errno on failure. Resources allocated up to that point may still be
1091 * allocated. Caller is responsible for cleanup in case this function fails.
1092 *
1093 * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then
1094 * the intr_idx specifies the vector, starting from 0. Otherwise it specifies
1095 * the index of the queue to which its interrupts will be forwarded.
1096 */
1097 static int
alloc_iq_fl(struct port_info * pi,struct sge_iq * iq,struct sge_fl * fl,int intr_idx,int cong)1098 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
1099 int intr_idx, int cong)
1100 {
1101 int rc, i, cntxt_id;
1102 size_t len;
1103 struct fw_iq_cmd c;
1104 struct adapter *sc = iq->adapter;
1105 uint32_t v = 0;
1106
1107 len = iq->qsize * iq->esize;
1108 rc = alloc_desc_ring(sc, len, DDI_DMA_READ, &iq->dhdl, &iq->ahdl,
1109 &iq->ba, (caddr_t *)&iq->desc);
1110 if (rc != 0)
1111 return (rc);
1112
1113 bzero(&c, sizeof (c));
1114 c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
1115 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
1116 V_FW_IQ_CMD_VFN(0));
1117
1118 c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
1119 FW_LEN16(c));
1120
1121 /* Special handling for firmware event queue */
1122 if (iq == &sc->sge.fwq)
1123 v |= F_FW_IQ_CMD_IQASYNCH;
1124
1125 if (iq->flags & IQ_INTR)
1126 ASSERT(intr_idx < sc->intr_count);
1127 else
1128 v |= F_FW_IQ_CMD_IQANDST;
1129 v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx);
1130
1131 c.type_to_iqandstindex = cpu_to_be32(v |
1132 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
1133 V_FW_IQ_CMD_VIID(pi->viid) |
1134 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
1135 c.iqdroprss_to_iqesize = cpu_to_be16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
1136 F_FW_IQ_CMD_IQGTSMODE |
1137 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) |
1138 V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4));
1139 c.iqsize = cpu_to_be16(iq->qsize);
1140 c.iqaddr = cpu_to_be64(iq->ba);
1141 if (cong >= 0) {
1142 const uint32_t iq_type =
1143 cong ? FW_IQ_IQTYPE_NIC : FW_IQ_IQTYPE_OFLD;
1144 c.iqns_to_fl0congen = BE_32(F_FW_IQ_CMD_IQFLINTCONGEN |
1145 V_FW_IQ_CMD_IQTYPE(iq_type));
1146 }
1147
1148 if (fl != NULL) {
1149 unsigned int chip_ver = CHELSIO_CHIP_VERSION(sc->params.chip);
1150
1151 mutex_init(&fl->lock, NULL, MUTEX_DRIVER,
1152 DDI_INTR_PRI(sc->intr_pri));
1153 fl->flags |= FL_MTX;
1154
1155 len = fl->qsize * RX_FL_ESIZE;
1156 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &fl->dhdl,
1157 &fl->ahdl, &fl->ba, (caddr_t *)&fl->desc);
1158 if (rc != 0)
1159 return (rc);
1160
1161 /* Allocate space for one software descriptor per buffer. */
1162 fl->cap = (fl->qsize - sc->sge.stat_len / RX_FL_ESIZE) * 8;
1163 fl->sdesc = kmem_zalloc(sizeof (struct fl_sdesc) * fl->cap,
1164 KM_SLEEP);
1165 fl->needed = fl->cap;
1166 fl->lowat = roundup(sc->sge.fl_starve_threshold, 8);
1167
1168 c.iqns_to_fl0congen |=
1169 cpu_to_be32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
1170 F_FW_IQ_CMD_FL0PACKEN | F_FW_IQ_CMD_FL0PADEN);
1171 if (cong >= 0) {
1172 c.iqns_to_fl0congen |=
1173 BE_32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) |
1174 F_FW_IQ_CMD_FL0CONGCIF |
1175 F_FW_IQ_CMD_FL0CONGEN);
1176 }
1177
1178 /*
1179 * In T6, for egress queue type FL there is internal overhead
1180 * of 16B for header going into FLM module. Hence the maximum
1181 * allowed burst size is 448 bytes. For T4/T5, the hardware
1182 * doesn't coalesce fetch requests if more than 64 bytes of
1183 * Free List pointers are provided, so we use a 128-byte Fetch
1184 * Burst Minimum there (T6 implements coalescing so we can use
1185 * the smaller 64-byte value there).
1186 */
1187
1188 c.fl0dcaen_to_fl0cidxfthresh = cpu_to_be16(
1189 V_FW_IQ_CMD_FL0FBMIN(chip_ver <= CHELSIO_T5 ?
1190 X_FETCHBURSTMIN_128B : X_FETCHBURSTMIN_64B) |
1191 V_FW_IQ_CMD_FL0FBMAX(chip_ver <= CHELSIO_T5 ?
1192 X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B));
1193 c.fl0size = cpu_to_be16(fl->qsize);
1194 c.fl0addr = cpu_to_be64(fl->ba);
1195 }
1196
1197 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1198 if (rc != 0) {
1199 cxgb_printf(sc->dip, CE_WARN,
1200 "failed to create ingress queue: %d", rc);
1201 return (rc);
1202 }
1203
1204 iq->cdesc = iq->desc;
1205 iq->cidx = 0;
1206 iq->gen = 1;
1207 iq->intr_next = iq->intr_params;
1208 iq->adapter = sc;
1209 iq->cntxt_id = be16_to_cpu(c.iqid);
1210 iq->abs_id = be16_to_cpu(c.physiqid);
1211 iq->flags |= IQ_ALLOCATED;
1212 mutex_init(&iq->lock, NULL, MUTEX_DRIVER,
1213 DDI_INTR_PRI(DDI_INTR_PRI(sc->intr_pri)));
1214 iq->polling = 0;
1215
1216 cntxt_id = iq->cntxt_id - sc->sge.iq_start;
1217 if (cntxt_id >= sc->sge.iqmap_sz) {
1218 panic("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
1219 cntxt_id, sc->sge.iqmap_sz - 1);
1220 }
1221 sc->sge.iqmap[cntxt_id] = iq;
1222
1223 if (fl != NULL) {
1224 fl->cntxt_id = be16_to_cpu(c.fl0id);
1225 fl->pidx = fl->cidx = 0;
1226 fl->copy_threshold = rx_copy_threshold;
1227
1228 cntxt_id = fl->cntxt_id - sc->sge.eq_start;
1229 if (cntxt_id >= sc->sge.eqmap_sz) {
1230 panic("%s: fl->cntxt_id (%d) more than the max (%d)",
1231 __func__, cntxt_id, sc->sge.eqmap_sz - 1);
1232 }
1233 sc->sge.eqmap[cntxt_id] = (void *)fl;
1234
1235 FL_LOCK(fl);
1236 (void) refill_fl(sc, fl, fl->lowat);
1237 FL_UNLOCK(fl);
1238
1239 iq->flags |= IQ_HAS_FL;
1240 }
1241
1242 if (is_t5(sc->params.chip) && cong >= 0) {
1243 uint32_t param, val;
1244
1245 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1246 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
1247 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id);
1248 if (cong == 0)
1249 val = 1 << 19;
1250 else {
1251 val = 2 << 19;
1252 for (i = 0; i < 4; i++) {
1253 if (cong & (1 << i))
1254 val |= 1 << (i << 2);
1255 }
1256 }
1257
1258 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
1259 if (rc != 0) {
1260 /* report error but carry on */
1261 cxgb_printf(sc->dip, CE_WARN,
1262 "failed to set congestion manager context for "
1263 "ingress queue %d: %d", iq->cntxt_id, rc);
1264 }
1265 }
1266
1267 /* Enable IQ interrupts */
1268 iq->state = IQS_IDLE;
1269 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) |
1270 V_INGRESSQID(iq->cntxt_id));
1271
1272 return (0);
1273 }
1274
1275 static int
free_iq_fl(struct port_info * pi,struct sge_iq * iq,struct sge_fl * fl)1276 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl)
1277 {
1278 int rc;
1279
1280 if (iq != NULL) {
1281 struct adapter *sc = iq->adapter;
1282 dev_info_t *dip;
1283
1284 dip = pi ? pi->dip : sc->dip;
1285 if (iq->flags & IQ_ALLOCATED) {
1286 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0,
1287 FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id,
1288 fl ? fl->cntxt_id : 0xffff, 0xffff);
1289 if (rc != 0) {
1290 cxgb_printf(dip, CE_WARN,
1291 "failed to free queue %p: %d", iq, rc);
1292 return (rc);
1293 }
1294 mutex_destroy(&iq->lock);
1295 iq->flags &= ~IQ_ALLOCATED;
1296 }
1297
1298 if (iq->desc != NULL) {
1299 (void) free_desc_ring(&iq->dhdl, &iq->ahdl);
1300 iq->desc = NULL;
1301 }
1302
1303 bzero(iq, sizeof (*iq));
1304 }
1305
1306 if (fl != NULL) {
1307 if (fl->sdesc != NULL) {
1308 FL_LOCK(fl);
1309 free_fl_bufs(fl);
1310 FL_UNLOCK(fl);
1311
1312 kmem_free(fl->sdesc, sizeof (struct fl_sdesc) *
1313 fl->cap);
1314 fl->sdesc = NULL;
1315 }
1316
1317 if (fl->desc != NULL) {
1318 (void) free_desc_ring(&fl->dhdl, &fl->ahdl);
1319 fl->desc = NULL;
1320 }
1321
1322 if (fl->flags & FL_MTX) {
1323 mutex_destroy(&fl->lock);
1324 fl->flags &= ~FL_MTX;
1325 }
1326
1327 bzero(fl, sizeof (struct sge_fl));
1328 }
1329
1330 return (0);
1331 }
1332
1333 static int
alloc_fwq(struct adapter * sc)1334 alloc_fwq(struct adapter *sc)
1335 {
1336 int rc, intr_idx;
1337 struct sge_iq *fwq = &sc->sge.fwq;
1338
1339 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE);
1340 fwq->flags |= IQ_INTR; /* always */
1341 intr_idx = sc->intr_count > 1 ? 1 : 0;
1342 rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1);
1343 if (rc != 0) {
1344 cxgb_printf(sc->dip, CE_WARN,
1345 "failed to create firmware event queue: %d.", rc);
1346 return (rc);
1347 }
1348
1349 return (0);
1350 }
1351
1352 static int
free_fwq(struct adapter * sc)1353 free_fwq(struct adapter *sc)
1354 {
1355
1356 return (free_iq_fl(NULL, &sc->sge.fwq, NULL));
1357 }
1358
1359 static int
alloc_rxq(struct port_info * pi,struct sge_rxq * rxq,int intr_idx,int i)1360 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int i)
1361 {
1362 int rc;
1363
1364 rxq->port = pi;
1365 rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx,
1366 t4_get_tp_ch_map(pi->adapter, pi->tx_chan));
1367 if (rc != 0)
1368 return (rc);
1369
1370 rxq->ksp = setup_rxq_kstats(pi, rxq, i);
1371
1372 return (rc);
1373 }
1374
1375 static int
free_rxq(struct port_info * pi,struct sge_rxq * rxq)1376 free_rxq(struct port_info *pi, struct sge_rxq *rxq)
1377 {
1378 int rc;
1379
1380 if (rxq->ksp != NULL) {
1381 kstat_delete(rxq->ksp);
1382 rxq->ksp = NULL;
1383 }
1384
1385 rc = free_iq_fl(pi, &rxq->iq, &rxq->fl);
1386 if (rc == 0)
1387 bzero(&rxq->fl, sizeof (*rxq) - offsetof(struct sge_rxq, fl));
1388
1389 return (rc);
1390 }
1391
1392 static int
ctrl_eq_alloc(struct adapter * sc,struct sge_eq * eq)1393 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
1394 {
1395 int rc, cntxt_id;
1396 struct fw_eq_ctrl_cmd c;
1397
1398 bzero(&c, sizeof (c));
1399
1400 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
1401 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) |
1402 V_FW_EQ_CTRL_CMD_VFN(0));
1403 c.alloc_to_len16 = BE_32(F_FW_EQ_CTRL_CMD_ALLOC |
1404 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c));
1405 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* TODO */
1406 c.physeqid_pkd = BE_32(0);
1407 c.fetchszm_to_iqid =
1408 BE_32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1409 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) |
1410 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
1411 c.dcaen_to_eqsize =
1412 BE_32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1413 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1414 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1415 V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize));
1416 c.eqaddr = BE_64(eq->ba);
1417
1418 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1419 if (rc != 0) {
1420 cxgb_printf(sc->dip, CE_WARN,
1421 "failed to create control queue %d: %d", eq->tx_chan, rc);
1422 return (rc);
1423 }
1424 eq->flags |= EQ_ALLOCATED;
1425
1426 eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(BE_32(c.cmpliqid_eqid));
1427 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1428 if (cntxt_id >= sc->sge.eqmap_sz)
1429 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1430 cntxt_id, sc->sge.eqmap_sz - 1);
1431 sc->sge.eqmap[cntxt_id] = eq;
1432
1433 return (rc);
1434 }
1435
1436 static int
eth_eq_alloc(struct adapter * sc,struct port_info * pi,struct sge_eq * eq)1437 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1438 {
1439 int rc, cntxt_id;
1440 struct fw_eq_eth_cmd c;
1441
1442 bzero(&c, sizeof (c));
1443
1444 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
1445 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
1446 V_FW_EQ_ETH_CMD_VFN(0));
1447 c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC |
1448 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
1449 c.autoequiqe_to_viid = BE_32(F_FW_EQ_ETH_CMD_AUTOEQUIQE |
1450 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(pi->viid));
1451 c.fetchszm_to_iqid =
1452 BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1453 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
1454 V_FW_EQ_ETH_CMD_IQID(eq->iqid));
1455 c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1456 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1457 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1458 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize));
1459 c.eqaddr = BE_64(eq->ba);
1460
1461 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1462 if (rc != 0) {
1463 cxgb_printf(pi->dip, CE_WARN,
1464 "failed to create Ethernet egress queue: %d", rc);
1465 return (rc);
1466 }
1467 eq->flags |= EQ_ALLOCATED;
1468
1469 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd));
1470 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1471 if (cntxt_id >= sc->sge.eqmap_sz)
1472 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1473 cntxt_id, sc->sge.eqmap_sz - 1);
1474 sc->sge.eqmap[cntxt_id] = eq;
1475
1476 return (rc);
1477 }
1478
1479 static int
alloc_eq(struct adapter * sc,struct port_info * pi,struct sge_eq * eq)1480 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1481 {
1482 int rc;
1483 size_t len;
1484
1485 mutex_init(&eq->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(sc->intr_pri));
1486 eq->flags |= EQ_MTX;
1487
1488 len = eq->qsize * EQ_ESIZE;
1489 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &eq->desc_dhdl,
1490 &eq->desc_ahdl, &eq->ba, (caddr_t *)&eq->desc);
1491 if (rc != 0)
1492 return (rc);
1493
1494 eq->cap = eq->qsize - sc->sge.stat_len / EQ_ESIZE;
1495 eq->spg = (void *)&eq->desc[eq->cap];
1496 eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */
1497 eq->pidx = eq->cidx = 0;
1498 eq->doorbells = sc->doorbells;
1499
1500 switch (eq->flags & EQ_TYPEMASK) {
1501 case EQ_CTRL:
1502 rc = ctrl_eq_alloc(sc, eq);
1503 break;
1504
1505 case EQ_ETH:
1506 rc = eth_eq_alloc(sc, pi, eq);
1507 break;
1508
1509 default:
1510 panic("%s: invalid eq type %d.", __func__,
1511 eq->flags & EQ_TYPEMASK);
1512 }
1513
1514 if (eq->doorbells & (DOORBELL_UDB | DOORBELL_UDBWC | DOORBELL_WCWR)) {
1515 uint32_t s_qpp = sc->sge.s_qpp;
1516 uint32_t mask = (1 << s_qpp) - 1;
1517 volatile uint8_t *udb;
1518
1519 udb = (volatile uint8_t *)sc->reg1p + UDBS_DB_OFFSET;
1520 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */
1521 eq->udb_qid = eq->cntxt_id & mask; /* id in page */
1522 if (eq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE)
1523 eq->doorbells &= ~DOORBELL_WCWR;
1524 else {
1525 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */
1526 eq->udb_qid = 0;
1527 }
1528 eq->udb = (volatile void *)udb;
1529 }
1530
1531 if (rc != 0) {
1532 cxgb_printf(sc->dip, CE_WARN,
1533 "failed to allocate egress queue(%d): %d",
1534 eq->flags & EQ_TYPEMASK, rc);
1535 }
1536
1537 return (rc);
1538 }
1539
1540 static int
free_eq(struct adapter * sc,struct sge_eq * eq)1541 free_eq(struct adapter *sc, struct sge_eq *eq)
1542 {
1543 int rc;
1544
1545 if (eq->flags & EQ_ALLOCATED) {
1546 switch (eq->flags & EQ_TYPEMASK) {
1547 case EQ_CTRL:
1548 rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0,
1549 eq->cntxt_id);
1550 break;
1551
1552 case EQ_ETH:
1553 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0,
1554 eq->cntxt_id);
1555 break;
1556 default:
1557 panic("%s: invalid eq type %d.", __func__,
1558 eq->flags & EQ_TYPEMASK);
1559 }
1560 if (rc != 0) {
1561 cxgb_printf(sc->dip, CE_WARN,
1562 "failed to free egress queue (%d): %d",
1563 eq->flags & EQ_TYPEMASK, rc);
1564 return (rc);
1565 }
1566 eq->flags &= ~EQ_ALLOCATED;
1567 }
1568
1569 if (eq->desc != NULL) {
1570 (void) free_desc_ring(&eq->desc_dhdl, &eq->desc_ahdl);
1571 eq->desc = NULL;
1572 }
1573
1574 if (eq->flags & EQ_MTX)
1575 mutex_destroy(&eq->lock);
1576
1577 bzero(eq, sizeof (*eq));
1578 return (0);
1579 }
1580
1581 static int
alloc_txq(struct port_info * pi,struct sge_txq * txq,int idx)1582 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx)
1583 {
1584 int rc, i;
1585 struct adapter *sc = pi->adapter;
1586 struct sge_eq *eq = &txq->eq;
1587
1588 rc = alloc_eq(sc, pi, eq);
1589 if (rc != 0)
1590 return (rc);
1591
1592 txq->port = pi;
1593 txq->sdesc = kmem_zalloc(sizeof (struct tx_sdesc) * eq->cap, KM_SLEEP);
1594 txq->txb_size = eq->qsize * tx_copy_threshold;
1595 rc = alloc_tx_copybuffer(sc, txq->txb_size, &txq->txb_dhdl,
1596 &txq->txb_ahdl, &txq->txb_ba, &txq->txb_va);
1597 if (rc == 0)
1598 txq->txb_avail = txq->txb_size;
1599 else
1600 txq->txb_avail = txq->txb_size = 0;
1601
1602 /*
1603 * TODO: is this too low? Worst case would need around 4 times qsize
1604 * (all tx descriptors filled to the brim with SGLs, with each entry in
1605 * the SGL coming from a distinct DMA handle). Increase tx_dhdl_total
1606 * if you see too many dma_hdl_failed.
1607 */
1608 txq->tx_dhdl_total = eq->qsize * 2;
1609 txq->tx_dhdl = kmem_zalloc(sizeof (ddi_dma_handle_t) *
1610 txq->tx_dhdl_total, KM_SLEEP);
1611 for (i = 0; i < txq->tx_dhdl_total; i++) {
1612 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx,
1613 DDI_DMA_SLEEP, 0, &txq->tx_dhdl[i]);
1614 if (rc != DDI_SUCCESS) {
1615 cxgb_printf(sc->dip, CE_WARN,
1616 "%s: failed to allocate DMA handle (%d)",
1617 __func__, rc);
1618 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
1619 }
1620 txq->tx_dhdl_avail++;
1621 }
1622
1623 txq->ksp = setup_txq_kstats(pi, txq, idx);
1624
1625 return (rc);
1626 }
1627
1628 static int
free_txq(struct port_info * pi,struct sge_txq * txq)1629 free_txq(struct port_info *pi, struct sge_txq *txq)
1630 {
1631 int i;
1632 struct adapter *sc = pi->adapter;
1633 struct sge_eq *eq = &txq->eq;
1634
1635 if (txq->ksp != NULL) {
1636 kstat_delete(txq->ksp);
1637 txq->ksp = NULL;
1638 }
1639
1640 if (txq->txb_va != NULL) {
1641 (void) free_desc_ring(&txq->txb_dhdl, &txq->txb_ahdl);
1642 txq->txb_va = NULL;
1643 }
1644
1645 if (txq->sdesc != NULL) {
1646 struct tx_sdesc *sd;
1647 ddi_dma_handle_t hdl;
1648
1649 TXQ_LOCK(txq);
1650 while (eq->cidx != eq->pidx) {
1651 sd = &txq->sdesc[eq->cidx];
1652
1653 for (i = sd->hdls_used; i; i--) {
1654 hdl = txq->tx_dhdl[txq->tx_dhdl_cidx];
1655 (void) ddi_dma_unbind_handle(hdl);
1656 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total)
1657 txq->tx_dhdl_cidx = 0;
1658 }
1659
1660 ASSERT(sd->m);
1661 freemsgchain(sd->m);
1662
1663 eq->cidx += sd->desc_used;
1664 if (eq->cidx >= eq->cap)
1665 eq->cidx -= eq->cap;
1666
1667 txq->txb_avail += txq->txb_used;
1668 }
1669 ASSERT(txq->tx_dhdl_cidx == txq->tx_dhdl_pidx);
1670 ASSERT(txq->txb_avail == txq->txb_size);
1671 TXQ_UNLOCK(txq);
1672
1673 kmem_free(txq->sdesc, sizeof (struct tx_sdesc) * eq->cap);
1674 txq->sdesc = NULL;
1675 }
1676
1677 if (txq->tx_dhdl != NULL) {
1678 for (i = 0; i < txq->tx_dhdl_total; i++) {
1679 if (txq->tx_dhdl[i] != NULL)
1680 ddi_dma_free_handle(&txq->tx_dhdl[i]);
1681 }
1682 kmem_free(txq->tx_dhdl,
1683 sizeof (ddi_dma_handle_t) * txq->tx_dhdl_total);
1684 txq->tx_dhdl = NULL;
1685 }
1686
1687 (void) free_eq(sc, &txq->eq);
1688
1689 bzero(txq, sizeof (*txq));
1690 return (0);
1691 }
1692
1693 /*
1694 * Allocates a block of contiguous memory for DMA. Can be used to allocate
1695 * memory for descriptor rings or for tx/rx copy buffers.
1696 *
1697 * Caller does not have to clean up anything if this function fails, it cleans
1698 * up after itself.
1699 *
1700 * Caller provides the following:
1701 * len length of the block of memory to allocate.
1702 * flags DDI_DMA_* flags to use (CONSISTENT/STREAMING, READ/WRITE/RDWR)
1703 * acc_attr device access attributes for the allocation.
1704 * dma_attr DMA attributes for the allocation
1705 *
1706 * If the function is successful it fills up this information:
1707 * dma_hdl DMA handle for the allocated memory
1708 * acc_hdl access handle for the allocated memory
1709 * ba bus address of the allocated memory
1710 * va KVA of the allocated memory.
1711 */
1712 static int
alloc_dma_memory(struct adapter * sc,size_t len,int flags,ddi_device_acc_attr_t * acc_attr,ddi_dma_attr_t * dma_attr,ddi_dma_handle_t * dma_hdl,ddi_acc_handle_t * acc_hdl,uint64_t * pba,caddr_t * pva)1713 alloc_dma_memory(struct adapter *sc, size_t len, int flags,
1714 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr,
1715 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
1716 uint64_t *pba, caddr_t *pva)
1717 {
1718 int rc;
1719 ddi_dma_handle_t dhdl;
1720 ddi_acc_handle_t ahdl;
1721 ddi_dma_cookie_t cookie;
1722 uint_t ccount;
1723 caddr_t va;
1724 size_t real_len;
1725
1726 *pva = NULL;
1727
1728 /*
1729 * DMA handle.
1730 */
1731 rc = ddi_dma_alloc_handle(sc->dip, dma_attr, DDI_DMA_SLEEP, 0, &dhdl);
1732 if (rc != DDI_SUCCESS) {
1733 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
1734 }
1735
1736 /*
1737 * Memory suitable for DMA.
1738 */
1739 rc = ddi_dma_mem_alloc(dhdl, len, acc_attr,
1740 flags & DDI_DMA_CONSISTENT ? DDI_DMA_CONSISTENT : DDI_DMA_STREAMING,
1741 DDI_DMA_SLEEP, 0, &va, &real_len, &ahdl);
1742 if (rc != DDI_SUCCESS) {
1743 ddi_dma_free_handle(&dhdl);
1744 return (ENOMEM);
1745 }
1746
1747 /*
1748 * DMA bindings.
1749 */
1750 rc = ddi_dma_addr_bind_handle(dhdl, NULL, va, real_len, flags, NULL,
1751 NULL, &cookie, &ccount);
1752 if (rc != DDI_DMA_MAPPED) {
1753 ddi_dma_mem_free(&ahdl);
1754 ddi_dma_free_handle(&dhdl);
1755 return (ENOMEM);
1756 }
1757 if (ccount != 1) {
1758 /* unusable DMA mapping */
1759 (void) free_desc_ring(&dhdl, &ahdl);
1760 return (ENOMEM);
1761 }
1762
1763 bzero(va, real_len);
1764 *dma_hdl = dhdl;
1765 *acc_hdl = ahdl;
1766 *pba = cookie.dmac_laddress;
1767 *pva = va;
1768
1769 return (0);
1770 }
1771
1772 static int
free_dma_memory(ddi_dma_handle_t * dhdl,ddi_acc_handle_t * ahdl)1773 free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl)
1774 {
1775 (void) ddi_dma_unbind_handle(*dhdl);
1776 ddi_dma_mem_free(ahdl);
1777 ddi_dma_free_handle(dhdl);
1778
1779 return (0);
1780 }
1781
1782 static int
alloc_desc_ring(struct adapter * sc,size_t len,int rw,ddi_dma_handle_t * dma_hdl,ddi_acc_handle_t * acc_hdl,uint64_t * pba,caddr_t * pva)1783 alloc_desc_ring(struct adapter *sc, size_t len, int rw,
1784 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
1785 uint64_t *pba, caddr_t *pva)
1786 {
1787 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_desc;
1788 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc;
1789
1790 return (alloc_dma_memory(sc, len, DDI_DMA_CONSISTENT | rw, acc_attr,
1791 dma_attr, dma_hdl, acc_hdl, pba, pva));
1792 }
1793
1794 static int
free_desc_ring(ddi_dma_handle_t * dhdl,ddi_acc_handle_t * ahdl)1795 free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl)
1796 {
1797 return (free_dma_memory(dhdl, ahdl));
1798 }
1799
1800 static int
alloc_tx_copybuffer(struct adapter * sc,size_t len,ddi_dma_handle_t * dma_hdl,ddi_acc_handle_t * acc_hdl,uint64_t * pba,caddr_t * pva)1801 alloc_tx_copybuffer(struct adapter *sc, size_t len,
1802 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
1803 uint64_t *pba, caddr_t *pva)
1804 {
1805 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_tx;
1806 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; /* NOT dma_attr_tx */
1807
1808 return (alloc_dma_memory(sc, len, DDI_DMA_STREAMING | DDI_DMA_WRITE,
1809 acc_attr, dma_attr, dma_hdl, acc_hdl, pba, pva));
1810 }
1811
1812 static inline bool
is_new_response(const struct sge_iq * iq,struct rsp_ctrl ** ctrl)1813 is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl)
1814 {
1815 (void) ddi_dma_sync(iq->dhdl, (uintptr_t)iq->cdesc -
1816 (uintptr_t)iq->desc, iq->esize, DDI_DMA_SYNC_FORKERNEL);
1817
1818 *ctrl = (void *)((uintptr_t)iq->cdesc +
1819 (iq->esize - sizeof (struct rsp_ctrl)));
1820
1821 return ((((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen));
1822 }
1823
1824 static inline void
iq_next(struct sge_iq * iq)1825 iq_next(struct sge_iq *iq)
1826 {
1827 iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize);
1828 if (++iq->cidx == iq->qsize - 1) {
1829 iq->cidx = 0;
1830 iq->gen ^= 1;
1831 iq->cdesc = iq->desc;
1832 }
1833 }
1834
1835 /*
1836 * Fill up the freelist by upto nbufs and maybe ring its doorbell.
1837 *
1838 * Returns non-zero to indicate that it should be added to the list of starving
1839 * freelists.
1840 */
1841 static int
refill_fl(struct adapter * sc,struct sge_fl * fl,int nbufs)1842 refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
1843 {
1844 uint64_t *d = &fl->desc[fl->pidx];
1845 struct fl_sdesc *sd = &fl->sdesc[fl->pidx];
1846
1847 FL_LOCK_ASSERT_OWNED(fl);
1848 ASSERT(nbufs >= 0);
1849
1850 if (nbufs > fl->needed)
1851 nbufs = fl->needed;
1852
1853 while (nbufs--) {
1854 if (sd->rxb != NULL) {
1855 if (sd->rxb->ref_cnt == 1) {
1856 /*
1857 * Buffer is available for recycling. Two ways
1858 * this can happen:
1859 *
1860 * a) All the packets DMA'd into it last time
1861 * around were within the rx_copy_threshold
1862 * and no part of the buffer was ever passed
1863 * up (ref_cnt never went over 1).
1864 *
1865 * b) Packets DMA'd into the buffer were passed
1866 * up but have all been freed by the upper
1867 * layers by now (ref_cnt went over 1 but is
1868 * now back to 1).
1869 *
1870 * Either way the bus address in the descriptor
1871 * ring is already valid.
1872 */
1873 ASSERT(*d == cpu_to_be64(sd->rxb->ba));
1874 d++;
1875 goto recycled;
1876 } else {
1877 /*
1878 * Buffer still in use and we need a
1879 * replacement. But first release our reference
1880 * on the existing buffer.
1881 */
1882 rxbuf_free(sd->rxb);
1883 }
1884 }
1885
1886 sd->rxb = rxbuf_alloc(sc->sge.rxbuf_cache, KM_NOSLEEP, 1);
1887 if (sd->rxb == NULL)
1888 break;
1889 *d++ = cpu_to_be64(sd->rxb->ba);
1890
1891 recycled: fl->pending++;
1892 sd++;
1893 fl->needed--;
1894 if (++fl->pidx == fl->cap) {
1895 fl->pidx = 0;
1896 sd = fl->sdesc;
1897 d = fl->desc;
1898 }
1899 }
1900
1901 if (fl->pending >= 8)
1902 ring_fl_db(sc, fl);
1903
1904 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
1905 }
1906
1907 #ifndef TAILQ_FOREACH_SAFE
1908 #define TAILQ_FOREACH_SAFE(var, head, field, tvar) \
1909 for ((var) = TAILQ_FIRST((head)); \
1910 (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \
1911 (var) = (tvar))
1912 #endif
1913
1914 /*
1915 * Attempt to refill all starving freelists.
1916 */
1917 static void
refill_sfl(void * arg)1918 refill_sfl(void *arg)
1919 {
1920 struct adapter *sc = arg;
1921 struct sge_fl *fl, *fl_temp;
1922
1923 mutex_enter(&sc->sfl_lock);
1924 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) {
1925 FL_LOCK(fl);
1926 (void) refill_fl(sc, fl, 64);
1927 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) {
1928 TAILQ_REMOVE(&sc->sfl, fl, link);
1929 fl->flags &= ~FL_STARVING;
1930 }
1931 FL_UNLOCK(fl);
1932 }
1933
1934 if (!TAILQ_EMPTY(&sc->sfl) != 0)
1935 sc->sfl_timer = timeout(refill_sfl, sc, drv_usectohz(100000));
1936 mutex_exit(&sc->sfl_lock);
1937 }
1938
1939 static void
add_fl_to_sfl(struct adapter * sc,struct sge_fl * fl)1940 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl)
1941 {
1942 mutex_enter(&sc->sfl_lock);
1943 FL_LOCK(fl);
1944 if ((fl->flags & FL_DOOMED) == 0) {
1945 if (TAILQ_EMPTY(&sc->sfl) != 0) {
1946 sc->sfl_timer = timeout(refill_sfl, sc,
1947 drv_usectohz(100000));
1948 }
1949 fl->flags |= FL_STARVING;
1950 TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
1951 }
1952 FL_UNLOCK(fl);
1953 mutex_exit(&sc->sfl_lock);
1954 }
1955
1956 static void
free_fl_bufs(struct sge_fl * fl)1957 free_fl_bufs(struct sge_fl *fl)
1958 {
1959 struct fl_sdesc *sd;
1960 unsigned int i;
1961
1962 FL_LOCK_ASSERT_OWNED(fl);
1963
1964 for (i = 0; i < fl->cap; i++) {
1965 sd = &fl->sdesc[i];
1966
1967 if (sd->rxb != NULL) {
1968 rxbuf_free(sd->rxb);
1969 sd->rxb = NULL;
1970 }
1971 }
1972 }
1973
1974 /*
1975 * Note that fl->cidx and fl->offset are left unchanged in case of failure.
1976 */
1977 static mblk_t *
get_fl_payload(struct adapter * sc,struct sge_fl * fl,uint32_t len_newbuf,int * fl_bufs_used)1978 get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf,
1979 int *fl_bufs_used)
1980 {
1981 struct mblk_pair frame = {0};
1982 struct rxbuf *rxb;
1983 mblk_t *m = NULL;
1984 uint_t nbuf = 0, len, copy, n;
1985 uint32_t cidx, offset, rcidx, roffset;
1986
1987 /*
1988 * The SGE won't pack a new frame into the current buffer if the entire
1989 * payload doesn't fit in the remaining space. Move on to the next buf
1990 * in that case.
1991 */
1992 rcidx = fl->cidx;
1993 roffset = fl->offset;
1994 if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) {
1995 fl->offset = 0;
1996 if (++fl->cidx == fl->cap)
1997 fl->cidx = 0;
1998 nbuf++;
1999 }
2000 cidx = fl->cidx;
2001 offset = fl->offset;
2002
2003 len = G_RSPD_LEN(len_newbuf); /* pktshift + payload length */
2004 copy = (len <= fl->copy_threshold);
2005 if (copy != 0) {
2006 frame.head = m = allocb(len, BPRI_HI);
2007 if (m == NULL) {
2008 fl->allocb_fail++;
2009 DTRACE_PROBE1(t4__fl_alloc_fail, struct sge_fl *, fl);
2010 fl->cidx = rcidx;
2011 fl->offset = roffset;
2012 return (NULL);
2013 }
2014 }
2015
2016 while (len) {
2017 rxb = fl->sdesc[cidx].rxb;
2018 n = min(len, rxb->buf_size - offset);
2019
2020 (void) ddi_dma_sync(rxb->dhdl, offset, n,
2021 DDI_DMA_SYNC_FORKERNEL);
2022
2023 if (copy != 0)
2024 bcopy(rxb->va + offset, m->b_wptr, n);
2025 else {
2026 m = desballoc((unsigned char *)rxb->va + offset, n,
2027 BPRI_HI, &rxb->freefunc);
2028 if (m == NULL) {
2029 fl->allocb_fail++;
2030 DTRACE_PROBE1(t4__fl_alloc_fail,
2031 struct sge_fl *, fl);
2032 if (frame.head)
2033 freemsgchain(frame.head);
2034 fl->cidx = rcidx;
2035 fl->offset = roffset;
2036 return (NULL);
2037 }
2038 atomic_inc_uint(&rxb->ref_cnt);
2039 if (frame.head != NULL)
2040 frame.tail->b_cont = m;
2041 else
2042 frame.head = m;
2043 frame.tail = m;
2044 }
2045 m->b_wptr += n;
2046 len -= n;
2047 offset += roundup(n, sc->sge.fl_align);
2048 ASSERT(offset <= rxb->buf_size);
2049 if (offset == rxb->buf_size) {
2050 offset = 0;
2051 if (++cidx == fl->cap)
2052 cidx = 0;
2053 nbuf++;
2054 }
2055 }
2056
2057 fl->cidx = cidx;
2058 fl->offset = offset;
2059 (*fl_bufs_used) += nbuf;
2060
2061 ASSERT(frame.head != NULL);
2062 return (frame.head);
2063 }
2064
2065 /*
2066 * We'll do immediate data tx for non-LSO, but only when not coalescing. We're
2067 * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes
2068 * of immediate data.
2069 */
2070 #define IMM_LEN ( \
2071 2 * EQ_ESIZE \
2072 - sizeof (struct fw_eth_tx_pkt_wr) \
2073 - sizeof (struct cpl_tx_pkt_core))
2074
2075 /*
2076 * Returns non-zero on failure, no need to cleanup anything in that case.
2077 *
2078 * Note 1: We always try to pull up the mblk if required and return E2BIG only
2079 * if this fails.
2080 *
2081 * Note 2: We'll also pullup incoming mblk if HW_LSO is set and the first mblk
2082 * does not have the TCP header in it.
2083 */
2084 static int
get_frame_txinfo(struct sge_txq * txq,mblk_t ** fp,struct txinfo * txinfo,int sgl_only)2085 get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, struct txinfo *txinfo,
2086 int sgl_only)
2087 {
2088 uint32_t flags = 0, len, n;
2089 mblk_t *m = *fp;
2090 int rc;
2091
2092 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb and dma_hdls */
2093
2094 mac_hcksum_get(m, NULL, NULL, NULL, NULL, &flags);
2095 txinfo->flags = (flags & HCK_TX_FLAGS);
2096
2097 mac_lso_get(m, &txinfo->mss, &flags);
2098 txinfo->flags |= (flags & HW_LSO_FLAGS);
2099
2100 if (flags & HW_LSO)
2101 sgl_only = 1; /* Do not allow immediate data with LSO */
2102
2103 /*
2104 * If checksum or segmentation offloads are requested, gather
2105 * information about the sizes and types of headers in the packet.
2106 */
2107 if (txinfo->flags != 0) {
2108 /*
2109 * Even if this fails, the meoi_flags field will be capable of
2110 * communicating the lack of useful packet information.
2111 */
2112 (void) mac_ether_offload_info(m, &txinfo->meoi);
2113 } else {
2114 bzero(&txinfo->meoi, sizeof (txinfo->meoi));
2115 }
2116
2117 start: txinfo->nsegs = 0;
2118 txinfo->hdls_used = 0;
2119 txinfo->txb_used = 0;
2120 txinfo->len = 0;
2121
2122 /* total length and a rough estimate of # of segments */
2123 n = 0;
2124 for (; m; m = m->b_cont) {
2125 len = MBLKL(m);
2126 n += (len / PAGE_SIZE) + 1;
2127 txinfo->len += len;
2128 }
2129 m = *fp;
2130
2131 if (n >= TX_SGL_SEGS || (flags & HW_LSO && MBLKL(m) < 50)) {
2132 txq->pullup_early++;
2133 m = msgpullup(*fp, -1);
2134 if (m == NULL) {
2135 txq->pullup_failed++;
2136 return (E2BIG); /* (*fp) left as it was */
2137 }
2138 freemsg(*fp);
2139 *fp = m;
2140 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags);
2141 }
2142
2143 if (txinfo->len <= IMM_LEN && !sgl_only)
2144 return (0); /* nsegs = 0 tells caller to use imm. tx */
2145
2146 if (txinfo->len <= txq->copy_threshold &&
2147 copy_into_txb(txq, m, txinfo->len, txinfo) == 0)
2148 goto done;
2149
2150 for (; m; m = m->b_cont) {
2151
2152 len = MBLKL(m);
2153
2154 /* Use tx copy buffer if this mblk is small enough */
2155 if (len <= txq->copy_threshold &&
2156 copy_into_txb(txq, m, len, txinfo) == 0)
2157 continue;
2158
2159 /* Add DMA bindings for this mblk to the SGL */
2160 rc = add_mblk(txq, txinfo, m, len);
2161
2162 if (rc == E2BIG ||
2163 (txinfo->nsegs == TX_SGL_SEGS && m->b_cont)) {
2164
2165 txq->pullup_late++;
2166 m = msgpullup(*fp, -1);
2167 if (m != NULL) {
2168 free_txinfo_resources(txq, txinfo);
2169 freemsg(*fp);
2170 *fp = m;
2171 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags);
2172 goto start;
2173 }
2174
2175 txq->pullup_failed++;
2176 rc = E2BIG;
2177 }
2178
2179 if (rc != 0) {
2180 free_txinfo_resources(txq, txinfo);
2181 return (rc);
2182 }
2183 }
2184
2185 ASSERT(txinfo->nsegs > 0 && txinfo->nsegs <= TX_SGL_SEGS);
2186
2187 done:
2188
2189 /*
2190 * Store the # of flits required to hold this frame's SGL in nflits. An
2191 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by
2192 * multiple (len0 + len1, addr0, addr1) tuples. If addr1 is not used
2193 * then len1 must be set to 0.
2194 */
2195 n = txinfo->nsegs - 1;
2196 txinfo->nflits = (3 * n) / 2 + (n & 1) + 2;
2197 if (n & 1)
2198 txinfo->sgl.sge[n / 2].len[1] = cpu_to_be32(0);
2199
2200 txinfo->sgl.cmd_nsge = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_DSGL) |
2201 V_ULPTX_NSGE(txinfo->nsegs));
2202
2203 return (0);
2204 }
2205
2206 static inline int
fits_in_txb(struct sge_txq * txq,int len,int * waste)2207 fits_in_txb(struct sge_txq *txq, int len, int *waste)
2208 {
2209 if (txq->txb_avail < len)
2210 return (0);
2211
2212 if (txq->txb_next + len <= txq->txb_size) {
2213 *waste = 0;
2214 return (1);
2215 }
2216
2217 *waste = txq->txb_size - txq->txb_next;
2218
2219 return (txq->txb_avail - *waste < len ? 0 : 1);
2220 }
2221
2222 #define TXB_CHUNK 64
2223
2224 /*
2225 * Copies the specified # of bytes into txq's tx copy buffer and updates txinfo
2226 * and txq to indicate resources used. Caller has to make sure that those many
2227 * bytes are available in the mblk chain (b_cont linked).
2228 */
2229 static inline int
copy_into_txb(struct sge_txq * txq,mblk_t * m,int len,struct txinfo * txinfo)2230 copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, struct txinfo *txinfo)
2231 {
2232 int waste, n;
2233
2234 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb */
2235
2236 if (!fits_in_txb(txq, len, &waste)) {
2237 txq->txb_full++;
2238 return (ENOMEM);
2239 }
2240
2241 if (waste != 0) {
2242 ASSERT((waste & (TXB_CHUNK - 1)) == 0);
2243 txinfo->txb_used += waste;
2244 txq->txb_avail -= waste;
2245 txq->txb_next = 0;
2246 }
2247
2248 for (n = 0; n < len; m = m->b_cont) {
2249 bcopy(m->b_rptr, txq->txb_va + txq->txb_next + n, MBLKL(m));
2250 n += MBLKL(m);
2251 }
2252
2253 add_seg(txinfo, txq->txb_ba + txq->txb_next, len);
2254
2255 n = roundup(len, TXB_CHUNK);
2256 txinfo->txb_used += n;
2257 txq->txb_avail -= n;
2258 txq->txb_next += n;
2259 ASSERT(txq->txb_next <= txq->txb_size);
2260 if (txq->txb_next == txq->txb_size)
2261 txq->txb_next = 0;
2262
2263 return (0);
2264 }
2265
2266 static inline void
add_seg(struct txinfo * txinfo,uint64_t ba,uint32_t len)2267 add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len)
2268 {
2269 ASSERT(txinfo->nsegs < TX_SGL_SEGS); /* must have room */
2270
2271 if (txinfo->nsegs != 0) {
2272 int idx = txinfo->nsegs - 1;
2273 txinfo->sgl.sge[idx / 2].len[idx & 1] = cpu_to_be32(len);
2274 txinfo->sgl.sge[idx / 2].addr[idx & 1] = cpu_to_be64(ba);
2275 } else {
2276 txinfo->sgl.len0 = cpu_to_be32(len);
2277 txinfo->sgl.addr0 = cpu_to_be64(ba);
2278 }
2279 txinfo->nsegs++;
2280 }
2281
2282 /*
2283 * This function cleans up any partially allocated resources when it fails so
2284 * there's nothing for the caller to clean up in that case.
2285 *
2286 * EIO indicates permanent failure. Caller should drop the frame containing
2287 * this mblk and continue.
2288 *
2289 * E2BIG indicates that the SGL length for this mblk exceeds the hardware
2290 * limit. Caller should pull up the frame before trying to send it out.
2291 * (This error means our pullup_early heuristic did not work for this frame)
2292 *
2293 * ENOMEM indicates a temporary shortage of resources (DMA handles, other DMA
2294 * resources, etc.). Caller should suspend the tx queue and wait for reclaim to
2295 * free up resources.
2296 */
2297 static inline int
add_mblk(struct sge_txq * txq,struct txinfo * txinfo,mblk_t * m,int len)2298 add_mblk(struct sge_txq *txq, struct txinfo *txinfo, mblk_t *m, int len)
2299 {
2300 ddi_dma_handle_t dhdl;
2301 ddi_dma_cookie_t cookie;
2302 uint_t ccount = 0;
2303 int rc;
2304
2305 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate dhdls */
2306
2307 if (txq->tx_dhdl_avail == 0) {
2308 txq->dma_hdl_failed++;
2309 return (ENOMEM);
2310 }
2311
2312 dhdl = txq->tx_dhdl[txq->tx_dhdl_pidx];
2313 rc = ddi_dma_addr_bind_handle(dhdl, NULL, (caddr_t)m->b_rptr, len,
2314 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie,
2315 &ccount);
2316 if (rc != DDI_DMA_MAPPED) {
2317 txq->dma_map_failed++;
2318
2319 ASSERT(rc != DDI_DMA_INUSE && rc != DDI_DMA_PARTIAL_MAP);
2320
2321 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EIO);
2322 }
2323
2324 if (ccount + txinfo->nsegs > TX_SGL_SEGS) {
2325 (void) ddi_dma_unbind_handle(dhdl);
2326 return (E2BIG);
2327 }
2328
2329 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size);
2330 while (--ccount) {
2331 ddi_dma_nextcookie(dhdl, &cookie);
2332 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size);
2333 }
2334
2335 if (++txq->tx_dhdl_pidx == txq->tx_dhdl_total)
2336 txq->tx_dhdl_pidx = 0;
2337 txq->tx_dhdl_avail--;
2338 txinfo->hdls_used++;
2339
2340 return (0);
2341 }
2342
2343 /*
2344 * Releases all the txq resources used up in the specified txinfo.
2345 */
2346 static void
free_txinfo_resources(struct sge_txq * txq,struct txinfo * txinfo)2347 free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo)
2348 {
2349 int n;
2350
2351 TXQ_LOCK_ASSERT_OWNED(txq); /* dhdls, txb */
2352
2353 n = txinfo->txb_used;
2354 if (n > 0) {
2355 txq->txb_avail += n;
2356 if (n <= txq->txb_next)
2357 txq->txb_next -= n;
2358 else {
2359 n -= txq->txb_next;
2360 txq->txb_next = txq->txb_size - n;
2361 }
2362 }
2363
2364 for (n = txinfo->hdls_used; n > 0; n--) {
2365 if (txq->tx_dhdl_pidx > 0)
2366 txq->tx_dhdl_pidx--;
2367 else
2368 txq->tx_dhdl_pidx = txq->tx_dhdl_total - 1;
2369 txq->tx_dhdl_avail++;
2370 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_pidx]);
2371 }
2372 }
2373
2374 /*
2375 * Returns 0 to indicate that m has been accepted into a coalesced tx work
2376 * request. It has either been folded into txpkts or txpkts was flushed and m
2377 * has started a new coalesced work request (as the first frame in a fresh
2378 * txpkts).
2379 *
2380 * Returns non-zero to indicate a failure - caller is responsible for
2381 * transmitting m, if there was anything in txpkts it has been flushed.
2382 */
2383 static int
add_to_txpkts(struct sge_txq * txq,struct txpkts * txpkts,mblk_t * m,struct txinfo * txinfo)2384 add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m,
2385 struct txinfo *txinfo)
2386 {
2387 struct sge_eq *eq = &txq->eq;
2388 int can_coalesce;
2389 struct tx_sdesc *txsd;
2390 uint8_t flits;
2391
2392 TXQ_LOCK_ASSERT_OWNED(txq);
2393
2394 if (txpkts->npkt > 0) {
2395 flits = TXPKTS_PKT_HDR + txinfo->nflits;
2396 can_coalesce = (txinfo->flags & HW_LSO) == 0 &&
2397 txpkts->nflits + flits <= TX_WR_FLITS &&
2398 txpkts->nflits + flits <= eq->avail * 8 &&
2399 txpkts->plen + txinfo->len < 65536;
2400
2401 if (can_coalesce != 0) {
2402 txpkts->tail->b_next = m;
2403 txpkts->tail = m;
2404 txpkts->npkt++;
2405 txpkts->nflits += flits;
2406 txpkts->plen += txinfo->len;
2407
2408 txsd = &txq->sdesc[eq->pidx];
2409 txsd->txb_used += txinfo->txb_used;
2410 txsd->hdls_used += txinfo->hdls_used;
2411
2412 return (0);
2413 }
2414
2415 /*
2416 * Couldn't coalesce m into txpkts. The first order of business
2417 * is to send txpkts on its way. Then we'll revisit m.
2418 */
2419 write_txpkts_wr(txq, txpkts);
2420 }
2421
2422 /*
2423 * Check if we can start a new coalesced tx work request with m as
2424 * the first packet in it.
2425 */
2426
2427 ASSERT(txpkts->npkt == 0);
2428 ASSERT(txinfo->len < 65536);
2429
2430 flits = TXPKTS_WR_HDR + txinfo->nflits;
2431 can_coalesce = (txinfo->flags & HW_LSO) == 0 &&
2432 flits <= eq->avail * 8 && flits <= TX_WR_FLITS;
2433
2434 if (can_coalesce == 0)
2435 return (EINVAL);
2436
2437 /*
2438 * Start a fresh coalesced tx WR with m as the first frame in it.
2439 */
2440 txpkts->tail = m;
2441 txpkts->npkt = 1;
2442 txpkts->nflits = flits;
2443 txpkts->flitp = &eq->desc[eq->pidx].flit[2];
2444 txpkts->plen = txinfo->len;
2445
2446 txsd = &txq->sdesc[eq->pidx];
2447 txsd->m = m;
2448 txsd->txb_used = txinfo->txb_used;
2449 txsd->hdls_used = txinfo->hdls_used;
2450
2451 return (0);
2452 }
2453
2454 /*
2455 * Note that write_txpkts_wr can never run out of hardware descriptors (but
2456 * write_txpkt_wr can). add_to_txpkts ensures that a frame is accepted for
2457 * coalescing only if sufficient hardware descriptors are available.
2458 */
2459 static void
write_txpkts_wr(struct sge_txq * txq,struct txpkts * txpkts)2460 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts)
2461 {
2462 struct sge_eq *eq = &txq->eq;
2463 struct fw_eth_tx_pkts_wr *wr;
2464 struct tx_sdesc *txsd;
2465 uint32_t ctrl;
2466 uint16_t ndesc;
2467
2468 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */
2469
2470 ndesc = howmany(txpkts->nflits, 8);
2471
2472 wr = (void *)&eq->desc[eq->pidx];
2473 wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR) |
2474 V_FW_WR_IMMDLEN(0)); /* immdlen does not matter in this WR */
2475 ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2));
2476 if (eq->avail == ndesc)
2477 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
2478 wr->equiq_to_len16 = cpu_to_be32(ctrl);
2479 wr->plen = cpu_to_be16(txpkts->plen);
2480 wr->npkt = txpkts->npkt;
2481 wr->r3 = wr->type = 0;
2482
2483 /* Everything else already written */
2484
2485 txsd = &txq->sdesc[eq->pidx];
2486 txsd->desc_used = ndesc;
2487
2488 txq->txb_used += txsd->txb_used / TXB_CHUNK;
2489 txq->hdl_used += txsd->hdls_used;
2490
2491 ASSERT(eq->avail >= ndesc);
2492
2493 eq->pending += ndesc;
2494 eq->avail -= ndesc;
2495 eq->pidx += ndesc;
2496 if (eq->pidx >= eq->cap)
2497 eq->pidx -= eq->cap;
2498
2499 txq->txpkts_pkts += txpkts->npkt;
2500 txq->txpkts_wrs++;
2501 txpkts->npkt = 0; /* emptied */
2502 }
2503
2504 typedef enum {
2505 COS_SUCCESS, /* ctrl flit contains proper bits for csum offload */
2506 COS_IGNORE, /* no csum offload requested */
2507 COS_FAIL, /* csum offload requested, but pkt data missing */
2508 } csum_offload_status_t;
2509 /*
2510 * Build a ctrl1 flit for checksum offload in CPL_TX_PKT_XT command
2511 */
2512 static csum_offload_status_t
csum_to_ctrl(const struct txinfo * txinfo,uint32_t chip_version,uint64_t * ctrlp)2513 csum_to_ctrl(const struct txinfo *txinfo, uint32_t chip_version,
2514 uint64_t *ctrlp)
2515 {
2516 const mac_ether_offload_info_t *meoi = &txinfo->meoi;
2517 const uint32_t tx_flags = txinfo->flags;
2518 const boolean_t needs_l3_csum = (tx_flags & HW_LSO) != 0 ||
2519 (tx_flags & HCK_IPV4_HDRCKSUM) != 0;
2520 const boolean_t needs_l4_csum = (tx_flags & HW_LSO) != 0 ||
2521 (tx_flags & (HCK_FULLCKSUM | HCK_PARTIALCKSUM)) != 0;
2522
2523 /*
2524 * Default to disabling any checksumming both for cases where it is not
2525 * requested, but also if we cannot appropriately interrogate the
2526 * required information from the packet.
2527 */
2528 uint64_t ctrl = F_TXPKT_L4CSUM_DIS | F_TXPKT_IPCSUM_DIS;
2529 if (!needs_l3_csum && !needs_l4_csum) {
2530 *ctrlp = ctrl;
2531 return (COS_IGNORE);
2532 }
2533
2534 if (needs_l3_csum) {
2535 /* Only IPv4 checksums are supported (for L3) */
2536 if ((meoi->meoi_flags & MEOI_L3INFO_SET) == 0 ||
2537 meoi->meoi_l3proto != ETHERTYPE_IP) {
2538 *ctrlp = ctrl;
2539 return (COS_FAIL);
2540 }
2541 ctrl &= ~F_TXPKT_IPCSUM_DIS;
2542 }
2543
2544 if (needs_l4_csum) {
2545 /*
2546 * We need at least all of the L3 header to make decisions about
2547 * the contained L4 protocol. If not all of the L4 information
2548 * is present, we will leave it to the NIC to checksum all it is
2549 * able to.
2550 */
2551 if ((meoi->meoi_flags & MEOI_L3INFO_SET) == 0) {
2552 *ctrlp = ctrl;
2553 return (COS_FAIL);
2554 }
2555
2556 /*
2557 * Since we are parsing the packet anyways, make the checksum
2558 * decision based on the L4 protocol, rather than using the
2559 * Generic TCP/UDP checksum using start & end offsets in the
2560 * packet (like requested with PARTIALCKSUM).
2561 */
2562 int csum_type = -1;
2563 if (meoi->meoi_l3proto == ETHERTYPE_IP &&
2564 meoi->meoi_l4proto == IPPROTO_TCP) {
2565 csum_type = TX_CSUM_TCPIP;
2566 } else if (meoi->meoi_l3proto == ETHERTYPE_IPV6 &&
2567 meoi->meoi_l4proto == IPPROTO_TCP) {
2568 csum_type = TX_CSUM_TCPIP6;
2569 } else if (meoi->meoi_l3proto == ETHERTYPE_IP &&
2570 meoi->meoi_l4proto == IPPROTO_UDP) {
2571 csum_type = TX_CSUM_UDPIP;
2572 } else if (meoi->meoi_l3proto == ETHERTYPE_IPV6 &&
2573 meoi->meoi_l4proto == IPPROTO_UDP) {
2574 csum_type = TX_CSUM_UDPIP6;
2575 } else {
2576 *ctrlp = ctrl;
2577 return (COS_FAIL);
2578 }
2579
2580 ASSERT(csum_type != -1);
2581 ctrl &= ~F_TXPKT_L4CSUM_DIS;
2582 ctrl |= V_TXPKT_CSUM_TYPE(csum_type);
2583 }
2584
2585 if ((ctrl & F_TXPKT_IPCSUM_DIS) == 0 &&
2586 (ctrl & F_TXPKT_L4CSUM_DIS) != 0) {
2587 /*
2588 * If only the IPv4 checksum is requested, we need to set an
2589 * appropriate type in the command for it.
2590 */
2591 ctrl |= V_TXPKT_CSUM_TYPE(TX_CSUM_IP);
2592 }
2593
2594 ASSERT(ctrl != (F_TXPKT_L4CSUM_DIS | F_TXPKT_IPCSUM_DIS));
2595
2596 /*
2597 * Fill in the requisite L2/L3 header length data.
2598 *
2599 * The Ethernet header length is recorded as 'size - 14 bytes'
2600 */
2601 const uint8_t eth_len = meoi->meoi_l2hlen - 14;
2602 if (chip_version >= CHELSIO_T6) {
2603 ctrl |= V_T6_TXPKT_ETHHDR_LEN(eth_len);
2604 } else {
2605 ctrl |= V_TXPKT_ETHHDR_LEN(eth_len);
2606 }
2607 ctrl |= V_TXPKT_IPHDR_LEN(meoi->meoi_l3hlen);
2608
2609 *ctrlp = ctrl;
2610 return (COS_SUCCESS);
2611 }
2612
2613 static int
write_txpkt_wr(struct port_info * pi,struct sge_txq * txq,mblk_t * m,struct txinfo * txinfo)2614 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m,
2615 struct txinfo *txinfo)
2616 {
2617 struct sge_eq *eq = &txq->eq;
2618 struct fw_eth_tx_pkt_wr *wr;
2619 struct cpl_tx_pkt_core *cpl;
2620 uint32_t ctrl; /* used in many unrelated places */
2621 uint64_t ctrl1;
2622 int nflits, ndesc;
2623 struct tx_sdesc *txsd;
2624 caddr_t dst;
2625 const mac_ether_offload_info_t *meoi = &txinfo->meoi;
2626
2627 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */
2628
2629 /*
2630 * Do we have enough flits to send this frame out?
2631 */
2632 ctrl = sizeof (struct cpl_tx_pkt_core);
2633 if (txinfo->flags & HW_LSO) {
2634 nflits = TXPKT_LSO_WR_HDR;
2635 ctrl += sizeof (struct cpl_tx_pkt_lso_core);
2636 } else {
2637 nflits = TXPKT_WR_HDR;
2638 }
2639 if (txinfo->nsegs > 0)
2640 nflits += txinfo->nflits;
2641 else {
2642 nflits += howmany(txinfo->len, 8);
2643 ctrl += txinfo->len;
2644 }
2645 ndesc = howmany(nflits, 8);
2646 if (ndesc > eq->avail)
2647 return (ENOMEM);
2648
2649 /* Firmware work request header */
2650 wr = (void *)&eq->desc[eq->pidx];
2651 wr->op_immdlen = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
2652 V_FW_WR_IMMDLEN(ctrl));
2653 ctrl = V_FW_WR_LEN16(howmany(nflits, 2));
2654 if (eq->avail == ndesc)
2655 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
2656 wr->equiq_to_len16 = cpu_to_be32(ctrl);
2657 wr->r3 = 0;
2658
2659 if (txinfo->flags & HW_LSO &&
2660 (meoi->meoi_flags & MEOI_L4INFO_SET) != 0 &&
2661 meoi->meoi_l4proto == IPPROTO_TCP) {
2662 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
2663
2664 ctrl = V_LSO_OPCODE((u32)CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE |
2665 F_LSO_LAST_SLICE;
2666
2667 if (meoi->meoi_l2hlen > sizeof (struct ether_header)) {
2668 /*
2669 * This presently assumes a standard VLAN header,
2670 * without support for Q-in-Q.
2671 */
2672 ctrl |= V_LSO_ETHHDR_LEN(1);
2673 }
2674
2675 switch (meoi->meoi_l3proto) {
2676 case ETHERTYPE_IPV6:
2677 ctrl |= F_LSO_IPV6;
2678 /* FALLTHROUGH */
2679 case ETHERTYPE_IP:
2680 ctrl |= V_LSO_IPHDR_LEN(meoi->meoi_l3hlen / 4);
2681 break;
2682 default:
2683 break;
2684 }
2685
2686 ctrl |= V_LSO_TCPHDR_LEN(meoi->meoi_l4hlen / 4);
2687
2688 lso->lso_ctrl = cpu_to_be32(ctrl);
2689 lso->ipid_ofst = cpu_to_be16(0);
2690 lso->mss = cpu_to_be16(txinfo->mss);
2691 lso->seqno_offset = cpu_to_be32(0);
2692 if (is_t4(pi->adapter->params.chip))
2693 lso->len = cpu_to_be32(txinfo->len);
2694 else
2695 lso->len = cpu_to_be32(V_LSO_T5_XFER_SIZE(txinfo->len));
2696
2697 cpl = (void *)(lso + 1);
2698
2699 txq->tso_wrs++;
2700 } else {
2701 cpl = (void *)(wr + 1);
2702 }
2703
2704 /* Checksum offload */
2705 switch (csum_to_ctrl(txinfo,
2706 CHELSIO_CHIP_VERSION(pi->adapter->params.chip), &ctrl1)) {
2707 case COS_SUCCESS:
2708 txq->txcsum++;
2709 break;
2710 case COS_FAIL:
2711 /*
2712 * Packet will be going out with checksums which are probably
2713 * wrong but there is little we can do now.
2714 */
2715 txq->csum_failed++;
2716 break;
2717 default:
2718 break;
2719 }
2720
2721 /* CPL header */
2722 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) |
2723 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
2724 cpl->pack = 0;
2725 cpl->len = cpu_to_be16(txinfo->len);
2726 cpl->ctrl1 = cpu_to_be64(ctrl1);
2727
2728 /* Software descriptor */
2729 txsd = &txq->sdesc[eq->pidx];
2730 txsd->m = m;
2731 txsd->txb_used = txinfo->txb_used;
2732 txsd->hdls_used = txinfo->hdls_used;
2733 /* LINTED: E_ASSIGN_NARROW_CONV */
2734 txsd->desc_used = ndesc;
2735
2736 txq->txb_used += txinfo->txb_used / TXB_CHUNK;
2737 txq->hdl_used += txinfo->hdls_used;
2738
2739 eq->pending += ndesc;
2740 eq->avail -= ndesc;
2741 eq->pidx += ndesc;
2742 if (eq->pidx >= eq->cap)
2743 eq->pidx -= eq->cap;
2744
2745 /* SGL */
2746 dst = (void *)(cpl + 1);
2747 if (txinfo->nsegs > 0) {
2748 txq->sgl_wrs++;
2749 copy_to_txd(eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8);
2750
2751 /* Need to zero-pad to a 16 byte boundary if not on one */
2752 if ((uintptr_t)dst & 0xf)
2753 /* LINTED: E_BAD_PTR_CAST_ALIGN */
2754 *(uint64_t *)dst = 0;
2755
2756 } else {
2757 txq->imm_wrs++;
2758 #ifdef DEBUG
2759 ctrl = txinfo->len;
2760 #endif
2761 for (; m; m = m->b_cont) {
2762 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m));
2763 #ifdef DEBUG
2764 ctrl -= MBLKL(m);
2765 #endif
2766 }
2767 ASSERT(ctrl == 0);
2768 }
2769
2770 txq->txpkt_wrs++;
2771 return (0);
2772 }
2773
2774 static inline void
write_ulp_cpl_sgl(struct port_info * pi,struct sge_txq * txq,struct txpkts * txpkts,struct txinfo * txinfo)2775 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
2776 struct txpkts *txpkts, struct txinfo *txinfo)
2777 {
2778 struct ulp_txpkt *ulpmc;
2779 struct ulptx_idata *ulpsc;
2780 struct cpl_tx_pkt_core *cpl;
2781 uintptr_t flitp, start, end;
2782 uint64_t ctrl;
2783 caddr_t dst;
2784
2785 ASSERT(txpkts->npkt > 0);
2786
2787 start = (uintptr_t)txq->eq.desc;
2788 end = (uintptr_t)txq->eq.spg;
2789
2790 /* Checksum offload */
2791 switch (csum_to_ctrl(txinfo,
2792 CHELSIO_CHIP_VERSION(pi->adapter->params.chip), &ctrl)) {
2793 case COS_SUCCESS:
2794 txq->txcsum++;
2795 break;
2796 case COS_FAIL:
2797 /*
2798 * Packet will be going out with checksums which are probably
2799 * wrong but there is little we can do now.
2800 */
2801 txq->csum_failed++;
2802 break;
2803 default:
2804 break;
2805 }
2806
2807 /*
2808 * The previous packet's SGL must have ended at a 16 byte boundary (this
2809 * is required by the firmware/hardware). It follows that flitp cannot
2810 * wrap around between the ULPTX master command and ULPTX subcommand (8
2811 * bytes each), and that it can not wrap around in the middle of the
2812 * cpl_tx_pkt_core either.
2813 */
2814 flitp = (uintptr_t)txpkts->flitp;
2815 ASSERT((flitp & 0xf) == 0);
2816
2817 /* ULP master command */
2818 ulpmc = (void *)flitp;
2819 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
2820 ulpmc->len = htonl(howmany(sizeof (*ulpmc) + sizeof (*ulpsc) +
2821 sizeof (*cpl) + 8 * txinfo->nflits, 16));
2822
2823 /* ULP subcommand */
2824 ulpsc = (void *)(ulpmc + 1);
2825 ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) |
2826 F_ULP_TX_SC_MORE);
2827 ulpsc->len = cpu_to_be32(sizeof (struct cpl_tx_pkt_core));
2828
2829 flitp += sizeof (*ulpmc) + sizeof (*ulpsc);
2830 if (flitp == end)
2831 flitp = start;
2832
2833 /* CPL_TX_PKT_XT */
2834 cpl = (void *)flitp;
2835 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) |
2836 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
2837 cpl->pack = 0;
2838 cpl->len = cpu_to_be16(txinfo->len);
2839 cpl->ctrl1 = cpu_to_be64(ctrl);
2840
2841 flitp += sizeof (*cpl);
2842 if (flitp == end)
2843 flitp = start;
2844
2845 /* SGL for this frame */
2846 dst = (caddr_t)flitp;
2847 copy_to_txd(&txq->eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8);
2848 flitp = (uintptr_t)dst;
2849
2850 /* Zero pad and advance to a 16 byte boundary if not already at one. */
2851 if (flitp & 0xf) {
2852
2853 /* no matter what, flitp should be on an 8 byte boundary */
2854 ASSERT((flitp & 0x7) == 0);
2855
2856 *(uint64_t *)flitp = 0;
2857 flitp += sizeof (uint64_t);
2858 txpkts->nflits++;
2859 }
2860
2861 if (flitp == end)
2862 flitp = start;
2863
2864 txpkts->flitp = (void *)flitp;
2865 }
2866
2867 static inline void
copy_to_txd(struct sge_eq * eq,caddr_t from,caddr_t * to,int len)2868 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
2869 {
2870 if ((uintptr_t)(*to) + len <= (uintptr_t)eq->spg) {
2871 bcopy(from, *to, len);
2872 (*to) += len;
2873 } else {
2874 int portion = (uintptr_t)eq->spg - (uintptr_t)(*to);
2875
2876 bcopy(from, *to, portion);
2877 from += portion;
2878 portion = len - portion; /* remaining */
2879 bcopy(from, (void *)eq->desc, portion);
2880 (*to) = (caddr_t)eq->desc + portion;
2881 }
2882 }
2883
2884 static inline void
ring_tx_db(struct adapter * sc,struct sge_eq * eq)2885 ring_tx_db(struct adapter *sc, struct sge_eq *eq)
2886 {
2887 int val, db_mode;
2888 uint_t db = eq->doorbells;
2889
2890 if (eq->pending > 1)
2891 db &= ~DOORBELL_WCWR;
2892
2893 if (eq->pending > eq->pidx) {
2894 int offset = eq->cap - (eq->pending - eq->pidx);
2895
2896 /* pidx has wrapped around since last doorbell */
2897
2898 (void) ddi_dma_sync(eq->desc_dhdl,
2899 offset * sizeof (struct tx_desc), 0,
2900 DDI_DMA_SYNC_FORDEV);
2901 (void) ddi_dma_sync(eq->desc_dhdl,
2902 0, eq->pidx * sizeof (struct tx_desc),
2903 DDI_DMA_SYNC_FORDEV);
2904 } else if (eq->pending > 0) {
2905 (void) ddi_dma_sync(eq->desc_dhdl,
2906 (eq->pidx - eq->pending) * sizeof (struct tx_desc),
2907 eq->pending * sizeof (struct tx_desc),
2908 DDI_DMA_SYNC_FORDEV);
2909 }
2910
2911 membar_producer();
2912
2913 if (is_t4(sc->params.chip))
2914 val = V_PIDX(eq->pending);
2915 else
2916 val = V_PIDX_T5(eq->pending);
2917
2918 db_mode = (1 << (ffs(db) - 1));
2919 switch (db_mode) {
2920 case DOORBELL_UDB:
2921 *eq->udb = LE_32(V_QID(eq->udb_qid) | val);
2922 break;
2923
2924 case DOORBELL_WCWR:
2925 {
2926 volatile uint64_t *dst, *src;
2927 int i;
2928 /*
2929 * Queues whose 128B doorbell segment fits in
2930 * the page do not use relative qid
2931 * (udb_qid is always 0). Only queues with
2932 * doorbell segments can do WCWR.
2933 */
2934 ASSERT(eq->udb_qid == 0 && eq->pending == 1);
2935
2936 dst = (volatile void *)((uintptr_t)eq->udb +
2937 UDBS_WR_OFFSET - UDBS_DB_OFFSET);
2938 i = eq->pidx ? eq->pidx - 1 : eq->cap - 1;
2939 src = (void *)&eq->desc[i];
2940 while (src != (void *)&eq->desc[i + 1]) {
2941 *dst++ = *src++;
2942 }
2943 membar_producer();
2944 break;
2945 }
2946
2947 case DOORBELL_UDBWC:
2948 *eq->udb = LE_32(V_QID(eq->udb_qid) | val);
2949 membar_producer();
2950 break;
2951
2952 case DOORBELL_KDB:
2953 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
2954 V_QID(eq->cntxt_id) | val);
2955 break;
2956 }
2957
2958 eq->pending = 0;
2959 }
2960
2961 static int
reclaim_tx_descs(struct sge_txq * txq,int howmany)2962 reclaim_tx_descs(struct sge_txq *txq, int howmany)
2963 {
2964 struct tx_sdesc *txsd;
2965 uint_t cidx, can_reclaim, reclaimed, txb_freed, hdls_freed;
2966 struct sge_eq *eq = &txq->eq;
2967
2968 EQ_LOCK_ASSERT_OWNED(eq);
2969
2970 cidx = eq->spg->cidx; /* stable snapshot */
2971 cidx = be16_to_cpu(cidx);
2972
2973 if (cidx >= eq->cidx)
2974 can_reclaim = cidx - eq->cidx;
2975 else
2976 can_reclaim = cidx + eq->cap - eq->cidx;
2977
2978 if (can_reclaim == 0)
2979 return (0);
2980
2981 txb_freed = hdls_freed = reclaimed = 0;
2982 do {
2983 int ndesc;
2984
2985 txsd = &txq->sdesc[eq->cidx];
2986 ndesc = txsd->desc_used;
2987
2988 /* Firmware doesn't return "partial" credits. */
2989 ASSERT(can_reclaim >= ndesc);
2990
2991 /*
2992 * We always keep mblk around, even for immediate data. If mblk
2993 * is NULL, this has to be the software descriptor for a credit
2994 * flush work request.
2995 */
2996 if (txsd->m != NULL)
2997 freemsgchain(txsd->m);
2998 #ifdef DEBUG
2999 else {
3000 ASSERT(txsd->txb_used == 0);
3001 ASSERT(txsd->hdls_used == 0);
3002 ASSERT(ndesc == 1);
3003 }
3004 #endif
3005
3006 txb_freed += txsd->txb_used;
3007 hdls_freed += txsd->hdls_used;
3008 reclaimed += ndesc;
3009
3010 eq->cidx += ndesc;
3011 if (eq->cidx >= eq->cap)
3012 eq->cidx -= eq->cap;
3013
3014 can_reclaim -= ndesc;
3015
3016 } while (can_reclaim && reclaimed < howmany);
3017
3018 eq->avail += reclaimed;
3019 ASSERT(eq->avail < eq->cap); /* avail tops out at (cap - 1) */
3020
3021 txq->txb_avail += txb_freed;
3022
3023 txq->tx_dhdl_avail += hdls_freed;
3024 ASSERT(txq->tx_dhdl_avail <= txq->tx_dhdl_total);
3025 for (; hdls_freed; hdls_freed--) {
3026 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_cidx]);
3027 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total)
3028 txq->tx_dhdl_cidx = 0;
3029 }
3030
3031 return (reclaimed);
3032 }
3033
3034 static void
write_txqflush_wr(struct sge_txq * txq)3035 write_txqflush_wr(struct sge_txq *txq)
3036 {
3037 struct sge_eq *eq = &txq->eq;
3038 struct fw_eq_flush_wr *wr;
3039 struct tx_sdesc *txsd;
3040
3041 EQ_LOCK_ASSERT_OWNED(eq);
3042 ASSERT(eq->avail > 0);
3043
3044 wr = (void *)&eq->desc[eq->pidx];
3045 bzero(wr, sizeof (*wr));
3046 wr->opcode = FW_EQ_FLUSH_WR;
3047 wr->equiq_to_len16 = cpu_to_be32(V_FW_WR_LEN16(sizeof (*wr) / 16) |
3048 F_FW_WR_EQUEQ | F_FW_WR_EQUIQ);
3049
3050 txsd = &txq->sdesc[eq->pidx];
3051 txsd->m = NULL;
3052 txsd->txb_used = 0;
3053 txsd->hdls_used = 0;
3054 txsd->desc_used = 1;
3055
3056 eq->pending++;
3057 eq->avail--;
3058 if (++eq->pidx == eq->cap)
3059 eq->pidx = 0;
3060 }
3061
3062 static int
t4_eth_rx(struct sge_iq * iq,const struct rss_header * rss,mblk_t * m)3063 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
3064 {
3065 bool csum_ok;
3066 uint16_t err_vec;
3067 struct sge_rxq *rxq = (void *)iq;
3068 struct mblk_pair chain = {0};
3069 struct adapter *sc = iq->adapter;
3070 const struct cpl_rx_pkt *cpl = (const void *)(rss + 1);
3071
3072 iq->intr_next = iq->intr_params;
3073
3074 m->b_rptr += sc->sge.pktshift;
3075
3076 /* Compressed error vector is enabled for T6 only */
3077 if (sc->params.tp.rx_pkt_encap)
3078 /* It is enabled only in T6 config file */
3079 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec));
3080 else
3081 err_vec = ntohs(cpl->err_vec);
3082
3083 csum_ok = cpl->csum_calc && !err_vec;
3084 /* TODO: what about cpl->ip_frag? */
3085 if (csum_ok && !cpl->ip_frag) {
3086 mac_hcksum_set(m, 0, 0, 0, 0xffff,
3087 HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
3088 HCK_IPV4_HDRCKSUM_OK);
3089 rxq->rxcsum++;
3090 }
3091
3092 /* Add to the chain that we'll send up */
3093 if (chain.head != NULL)
3094 chain.tail->b_next = m;
3095 else
3096 chain.head = m;
3097 chain.tail = m;
3098
3099 t4_mac_rx(rxq->port, rxq, chain.head);
3100
3101 rxq->rxpkts++;
3102 rxq->rxbytes += be16_to_cpu(cpl->len);
3103 return (0);
3104 }
3105
3106 #define FL_HW_IDX(idx) ((idx) >> 3)
3107
3108 static inline void
ring_fl_db(struct adapter * sc,struct sge_fl * fl)3109 ring_fl_db(struct adapter *sc, struct sge_fl *fl)
3110 {
3111 int desc_start, desc_last, ndesc;
3112 uint32_t v = sc->params.arch.sge_fl_db;
3113
3114 ndesc = FL_HW_IDX(fl->pending);
3115
3116 /* Hold back one credit if pidx = cidx */
3117 if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx))
3118 ndesc--;
3119
3120 /*
3121 * There are chances of ndesc modified above (to avoid pidx = cidx).
3122 * If there is nothing to post, return.
3123 */
3124 if (ndesc <= 0)
3125 return;
3126
3127 desc_last = FL_HW_IDX(fl->pidx);
3128
3129 if (fl->pidx < fl->pending) {
3130 /* There was a wrap */
3131 desc_start = FL_HW_IDX(fl->pidx + fl->cap - fl->pending);
3132
3133 /* From desc_start to the end of list */
3134 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 0,
3135 DDI_DMA_SYNC_FORDEV);
3136
3137 /* From start of list to the desc_last */
3138 if (desc_last != 0)
3139 (void) ddi_dma_sync(fl->dhdl, 0, desc_last *
3140 RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3141 } else {
3142 /* There was no wrap, sync from start_desc to last_desc */
3143 desc_start = FL_HW_IDX(fl->pidx - fl->pending);
3144 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE,
3145 ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3146 }
3147
3148 if (is_t4(sc->params.chip))
3149 v |= V_PIDX(ndesc);
3150 else
3151 v |= V_PIDX_T5(ndesc);
3152 v |= V_QID(fl->cntxt_id) | V_PIDX(ndesc);
3153
3154 membar_producer();
3155
3156 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v);
3157
3158 /*
3159 * Update pending count:
3160 * Deduct the number of descriptors posted
3161 */
3162 fl->pending -= ndesc * 8;
3163 }
3164
3165 static void
tx_reclaim_task(void * arg)3166 tx_reclaim_task(void *arg)
3167 {
3168 struct sge_txq *txq = arg;
3169
3170 TXQ_LOCK(txq);
3171 reclaim_tx_descs(txq, txq->eq.qsize);
3172 TXQ_UNLOCK(txq);
3173 }
3174
3175 /* ARGSUSED */
3176 static int
handle_sge_egr_update(struct sge_iq * iq,const struct rss_header * rss,mblk_t * m)3177 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
3178 mblk_t *m)
3179 {
3180 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
3181 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
3182 struct adapter *sc = iq->adapter;
3183 struct sge *s = &sc->sge;
3184 struct sge_eq *eq;
3185 struct sge_txq *txq;
3186
3187 txq = (void *)s->eqmap[qid - s->eq_start];
3188 eq = &txq->eq;
3189 txq->qflush++;
3190 t4_mac_tx_update(txq->port, txq);
3191
3192 ddi_taskq_dispatch(sc->tq[eq->tx_chan], tx_reclaim_task,
3193 (void *)txq, DDI_NOSLEEP);
3194
3195 return (0);
3196 }
3197
3198 static int
handle_fw_rpl(struct sge_iq * iq,const struct rss_header * rss,mblk_t * m)3199 handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
3200 {
3201 struct adapter *sc = iq->adapter;
3202 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
3203
3204 ASSERT(m == NULL);
3205
3206 if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) {
3207 const struct rss_header *rss2;
3208
3209 rss2 = (const struct rss_header *)&cpl->data[0];
3210 return (sc->cpl_handler[rss2->opcode](iq, rss2, m));
3211 }
3212 return (sc->fw_msg_handler[cpl->type](sc, &cpl->data[0]));
3213 }
3214
3215 int
t4_alloc_tx_maps(struct adapter * sc,struct tx_maps * txmaps,int count,int flags)3216 t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count,
3217 int flags)
3218 {
3219 int i, rc;
3220
3221 txmaps->map_total = count;
3222 txmaps->map_avail = txmaps->map_cidx = txmaps->map_pidx = 0;
3223
3224 txmaps->map = kmem_zalloc(sizeof (ddi_dma_handle_t) *
3225 txmaps->map_total, flags);
3226
3227 for (i = 0; i < count; i++) {
3228 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx,
3229 DDI_DMA_SLEEP, 0, &txmaps->map[i]);
3230 if (rc != DDI_SUCCESS) {
3231 cxgb_printf(sc->dip, CE_WARN,
3232 "%s: failed to allocate DMA handle (%d)",
3233 __func__, rc);
3234 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
3235 }
3236 txmaps->map_avail++;
3237 }
3238
3239 return (0);
3240 }
3241
3242 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG)
3243 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR)
3244 #define KS_U_SET(x, y) kstatp->x.value.ul = (y)
3245 #define KS_U_FROM(x, y) kstatp->x.value.ul = (y)->x
3246 #define KS_C_SET(x, ...) \
3247 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__)
3248
3249 /*
3250 * cxgbe:X:config
3251 */
3252 struct cxgbe_port_config_kstats {
3253 kstat_named_t idx;
3254 kstat_named_t nrxq;
3255 kstat_named_t ntxq;
3256 kstat_named_t first_rxq;
3257 kstat_named_t first_txq;
3258 kstat_named_t controller;
3259 kstat_named_t factory_mac_address;
3260 };
3261
3262 /*
3263 * cxgbe:X:info
3264 */
3265 struct cxgbe_port_info_kstats {
3266 kstat_named_t transceiver;
3267 kstat_named_t rx_ovflow0;
3268 kstat_named_t rx_ovflow1;
3269 kstat_named_t rx_ovflow2;
3270 kstat_named_t rx_ovflow3;
3271 kstat_named_t rx_trunc0;
3272 kstat_named_t rx_trunc1;
3273 kstat_named_t rx_trunc2;
3274 kstat_named_t rx_trunc3;
3275 kstat_named_t tx_pause;
3276 kstat_named_t rx_pause;
3277 };
3278
3279 static kstat_t *
setup_port_config_kstats(struct port_info * pi)3280 setup_port_config_kstats(struct port_info *pi)
3281 {
3282 kstat_t *ksp;
3283 struct cxgbe_port_config_kstats *kstatp;
3284 int ndata;
3285 dev_info_t *pdip = ddi_get_parent(pi->dip);
3286 uint8_t *ma = &pi->hw_addr[0];
3287
3288 ndata = sizeof (struct cxgbe_port_config_kstats) /
3289 sizeof (kstat_named_t);
3290
3291 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "config",
3292 "net", KSTAT_TYPE_NAMED, ndata, 0);
3293 if (ksp == NULL) {
3294 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats.");
3295 return (NULL);
3296 }
3297
3298 kstatp = (struct cxgbe_port_config_kstats *)ksp->ks_data;
3299
3300 KS_UINIT(idx);
3301 KS_UINIT(nrxq);
3302 KS_UINIT(ntxq);
3303 KS_UINIT(first_rxq);
3304 KS_UINIT(first_txq);
3305 KS_CINIT(controller);
3306 KS_CINIT(factory_mac_address);
3307
3308 KS_U_SET(idx, pi->port_id);
3309 KS_U_SET(nrxq, pi->nrxq);
3310 KS_U_SET(ntxq, pi->ntxq);
3311 KS_U_SET(first_rxq, pi->first_rxq);
3312 KS_U_SET(first_txq, pi->first_txq);
3313 KS_C_SET(controller, "%s%d", ddi_driver_name(pdip),
3314 ddi_get_instance(pdip));
3315 KS_C_SET(factory_mac_address, "%02X%02X%02X%02X%02X%02X",
3316 ma[0], ma[1], ma[2], ma[3], ma[4], ma[5]);
3317
3318 /* Do NOT set ksp->ks_update. These kstats do not change. */
3319
3320 /* Install the kstat */
3321 ksp->ks_private = (void *)pi;
3322 kstat_install(ksp);
3323
3324 return (ksp);
3325 }
3326
3327 static kstat_t *
setup_port_info_kstats(struct port_info * pi)3328 setup_port_info_kstats(struct port_info *pi)
3329 {
3330 kstat_t *ksp;
3331 struct cxgbe_port_info_kstats *kstatp;
3332 int ndata;
3333
3334 ndata = sizeof (struct cxgbe_port_info_kstats) / sizeof (kstat_named_t);
3335
3336 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "info",
3337 "net", KSTAT_TYPE_NAMED, ndata, 0);
3338 if (ksp == NULL) {
3339 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats.");
3340 return (NULL);
3341 }
3342
3343 kstatp = (struct cxgbe_port_info_kstats *)ksp->ks_data;
3344
3345 KS_CINIT(transceiver);
3346 KS_UINIT(rx_ovflow0);
3347 KS_UINIT(rx_ovflow1);
3348 KS_UINIT(rx_ovflow2);
3349 KS_UINIT(rx_ovflow3);
3350 KS_UINIT(rx_trunc0);
3351 KS_UINIT(rx_trunc1);
3352 KS_UINIT(rx_trunc2);
3353 KS_UINIT(rx_trunc3);
3354 KS_UINIT(tx_pause);
3355 KS_UINIT(rx_pause);
3356
3357 /* Install the kstat */
3358 ksp->ks_update = update_port_info_kstats;
3359 ksp->ks_private = (void *)pi;
3360 kstat_install(ksp);
3361
3362 return (ksp);
3363 }
3364
3365 static int
update_port_info_kstats(kstat_t * ksp,int rw)3366 update_port_info_kstats(kstat_t *ksp, int rw)
3367 {
3368 struct cxgbe_port_info_kstats *kstatp =
3369 (struct cxgbe_port_info_kstats *)ksp->ks_data;
3370 struct port_info *pi = ksp->ks_private;
3371 static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX",
3372 "active TWINAX", "LRM" };
3373 uint32_t bgmap;
3374
3375 if (rw == KSTAT_WRITE)
3376 return (0);
3377
3378 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
3379 KS_C_SET(transceiver, "unplugged");
3380 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
3381 KS_C_SET(transceiver, "unknown");
3382 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
3383 KS_C_SET(transceiver, "unsupported");
3384 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str))
3385 KS_C_SET(transceiver, "%s", mod_str[pi->mod_type]);
3386 else
3387 KS_C_SET(transceiver, "type %d", pi->mod_type);
3388
3389 #define GET_STAT(name) t4_read_reg64(pi->adapter, \
3390 PORT_REG(pi->port_id, A_MPS_PORT_STAT_##name##_L))
3391 #define GET_STAT_COM(name) t4_read_reg64(pi->adapter, \
3392 A_MPS_STAT_##name##_L)
3393
3394 bgmap = G_NUMPORTS(t4_read_reg(pi->adapter, A_MPS_CMN_CTL));
3395 if (bgmap == 0)
3396 bgmap = (pi->port_id == 0) ? 0xf : 0;
3397 else if (bgmap == 1)
3398 bgmap = (pi->port_id < 2) ? (3 << (2 * pi->port_id)) : 0;
3399 else
3400 bgmap = 1;
3401
3402 KS_U_SET(rx_ovflow0, (bgmap & 1) ?
3403 GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0);
3404 KS_U_SET(rx_ovflow1, (bgmap & 2) ?
3405 GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0);
3406 KS_U_SET(rx_ovflow2, (bgmap & 4) ?
3407 GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0);
3408 KS_U_SET(rx_ovflow3, (bgmap & 8) ?
3409 GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0);
3410 KS_U_SET(rx_trunc0, (bgmap & 1) ?
3411 GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0);
3412 KS_U_SET(rx_trunc1, (bgmap & 2) ?
3413 GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0);
3414 KS_U_SET(rx_trunc2, (bgmap & 4) ?
3415 GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0);
3416 KS_U_SET(rx_trunc3, (bgmap & 8) ?
3417 GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0);
3418
3419 KS_U_SET(tx_pause, GET_STAT(TX_PORT_PAUSE));
3420 KS_U_SET(rx_pause, GET_STAT(RX_PORT_PAUSE));
3421
3422 return (0);
3423
3424 }
3425
3426 /*
3427 * cxgbe:X:rxqY
3428 */
3429 struct rxq_kstats {
3430 kstat_named_t rxcsum;
3431 kstat_named_t rxpkts;
3432 kstat_named_t rxbytes;
3433 kstat_named_t nomem;
3434 };
3435
3436 static kstat_t *
setup_rxq_kstats(struct port_info * pi,struct sge_rxq * rxq,int idx)3437 setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, int idx)
3438 {
3439 struct kstat *ksp;
3440 struct rxq_kstats *kstatp;
3441 int ndata;
3442 char str[16];
3443
3444 ndata = sizeof (struct rxq_kstats) / sizeof (kstat_named_t);
3445 (void) snprintf(str, sizeof (str), "rxq%u", idx);
3446
3447 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "rxq",
3448 KSTAT_TYPE_NAMED, ndata, 0);
3449 if (ksp == NULL) {
3450 cxgb_printf(pi->dip, CE_WARN,
3451 "%s: failed to initialize rxq kstats for queue %d.",
3452 __func__, idx);
3453 return (NULL);
3454 }
3455
3456 kstatp = (struct rxq_kstats *)ksp->ks_data;
3457
3458 KS_UINIT(rxcsum);
3459 KS_UINIT(rxpkts);
3460 KS_UINIT(rxbytes);
3461 KS_UINIT(nomem);
3462
3463 ksp->ks_update = update_rxq_kstats;
3464 ksp->ks_private = (void *)rxq;
3465 kstat_install(ksp);
3466
3467 return (ksp);
3468 }
3469
3470 static int
update_rxq_kstats(kstat_t * ksp,int rw)3471 update_rxq_kstats(kstat_t *ksp, int rw)
3472 {
3473 struct rxq_kstats *kstatp = (struct rxq_kstats *)ksp->ks_data;
3474 struct sge_rxq *rxq = ksp->ks_private;
3475
3476 if (rw == KSTAT_WRITE)
3477 return (0);
3478
3479 KS_U_FROM(rxcsum, rxq);
3480 KS_U_FROM(rxpkts, rxq);
3481 KS_U_FROM(rxbytes, rxq);
3482 KS_U_FROM(nomem, rxq);
3483
3484 return (0);
3485 }
3486
3487 /*
3488 * cxgbe:X:txqY
3489 */
3490 struct txq_kstats {
3491 kstat_named_t txcsum;
3492 kstat_named_t tso_wrs;
3493 kstat_named_t imm_wrs;
3494 kstat_named_t sgl_wrs;
3495 kstat_named_t txpkt_wrs;
3496 kstat_named_t txpkts_wrs;
3497 kstat_named_t txpkts_pkts;
3498 kstat_named_t txb_used;
3499 kstat_named_t hdl_used;
3500 kstat_named_t txb_full;
3501 kstat_named_t dma_hdl_failed;
3502 kstat_named_t dma_map_failed;
3503 kstat_named_t qfull;
3504 kstat_named_t qflush;
3505 kstat_named_t pullup_early;
3506 kstat_named_t pullup_late;
3507 kstat_named_t pullup_failed;
3508 kstat_named_t csum_failed;
3509 };
3510
3511 static kstat_t *
setup_txq_kstats(struct port_info * pi,struct sge_txq * txq,int idx)3512 setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, int idx)
3513 {
3514 struct kstat *ksp;
3515 struct txq_kstats *kstatp;
3516 int ndata;
3517 char str[16];
3518
3519 ndata = sizeof (struct txq_kstats) / sizeof (kstat_named_t);
3520 (void) snprintf(str, sizeof (str), "txq%u", idx);
3521
3522 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "txq",
3523 KSTAT_TYPE_NAMED, ndata, 0);
3524 if (ksp == NULL) {
3525 cxgb_printf(pi->dip, CE_WARN,
3526 "%s: failed to initialize txq kstats for queue %d.",
3527 __func__, idx);
3528 return (NULL);
3529 }
3530
3531 kstatp = (struct txq_kstats *)ksp->ks_data;
3532
3533 KS_UINIT(txcsum);
3534 KS_UINIT(tso_wrs);
3535 KS_UINIT(imm_wrs);
3536 KS_UINIT(sgl_wrs);
3537 KS_UINIT(txpkt_wrs);
3538 KS_UINIT(txpkts_wrs);
3539 KS_UINIT(txpkts_pkts);
3540 KS_UINIT(txb_used);
3541 KS_UINIT(hdl_used);
3542 KS_UINIT(txb_full);
3543 KS_UINIT(dma_hdl_failed);
3544 KS_UINIT(dma_map_failed);
3545 KS_UINIT(qfull);
3546 KS_UINIT(qflush);
3547 KS_UINIT(pullup_early);
3548 KS_UINIT(pullup_late);
3549 KS_UINIT(pullup_failed);
3550 KS_UINIT(csum_failed);
3551
3552 ksp->ks_update = update_txq_kstats;
3553 ksp->ks_private = (void *)txq;
3554 kstat_install(ksp);
3555
3556 return (ksp);
3557 }
3558
3559 static int
update_txq_kstats(kstat_t * ksp,int rw)3560 update_txq_kstats(kstat_t *ksp, int rw)
3561 {
3562 struct txq_kstats *kstatp = (struct txq_kstats *)ksp->ks_data;
3563 struct sge_txq *txq = ksp->ks_private;
3564
3565 if (rw == KSTAT_WRITE)
3566 return (0);
3567
3568 KS_U_FROM(txcsum, txq);
3569 KS_U_FROM(tso_wrs, txq);
3570 KS_U_FROM(imm_wrs, txq);
3571 KS_U_FROM(sgl_wrs, txq);
3572 KS_U_FROM(txpkt_wrs, txq);
3573 KS_U_FROM(txpkts_wrs, txq);
3574 KS_U_FROM(txpkts_pkts, txq);
3575 KS_U_FROM(txb_used, txq);
3576 KS_U_FROM(hdl_used, txq);
3577 KS_U_FROM(txb_full, txq);
3578 KS_U_FROM(dma_hdl_failed, txq);
3579 KS_U_FROM(dma_map_failed, txq);
3580 KS_U_FROM(qfull, txq);
3581 KS_U_FROM(qflush, txq);
3582 KS_U_FROM(pullup_early, txq);
3583 KS_U_FROM(pullup_late, txq);
3584 KS_U_FROM(pullup_failed, txq);
3585 KS_U_FROM(csum_failed, txq);
3586
3587 return (0);
3588 }
3589
3590 static int rxbuf_ctor(void *, void *, int);
3591 static void rxbuf_dtor(void *, void *);
3592
3593 static kmem_cache_t *
rxbuf_cache_create(struct rxbuf_cache_params * p)3594 rxbuf_cache_create(struct rxbuf_cache_params *p)
3595 {
3596 char name[32];
3597
3598 (void) snprintf(name, sizeof (name), "%s%d_rxbuf_cache",
3599 ddi_driver_name(p->dip), ddi_get_instance(p->dip));
3600
3601 return kmem_cache_create(name, sizeof (struct rxbuf), _CACHE_LINE_SIZE,
3602 rxbuf_ctor, rxbuf_dtor, NULL, p, NULL, 0);
3603 }
3604
3605 /*
3606 * If ref_cnt is more than 1 then those many calls to rxbuf_free will
3607 * have to be made before the rxb is released back to the kmem_cache.
3608 */
3609 static struct rxbuf *
rxbuf_alloc(kmem_cache_t * cache,int kmflags,uint_t ref_cnt)3610 rxbuf_alloc(kmem_cache_t *cache, int kmflags, uint_t ref_cnt)
3611 {
3612 struct rxbuf *rxb;
3613
3614 ASSERT(ref_cnt > 0);
3615
3616 rxb = kmem_cache_alloc(cache, kmflags);
3617 if (rxb != NULL) {
3618 rxb->ref_cnt = ref_cnt;
3619 rxb->cache = cache;
3620 }
3621
3622 return (rxb);
3623 }
3624
3625 /*
3626 * This is normally called via the rxb's freefunc, when an mblk referencing the
3627 * rxb is freed.
3628 */
3629 static void
rxbuf_free(struct rxbuf * rxb)3630 rxbuf_free(struct rxbuf *rxb)
3631 {
3632 if (atomic_dec_uint_nv(&rxb->ref_cnt) == 0)
3633 kmem_cache_free(rxb->cache, rxb);
3634 }
3635
3636 static int
rxbuf_ctor(void * arg1,void * arg2,int kmflag)3637 rxbuf_ctor(void *arg1, void *arg2, int kmflag)
3638 {
3639 struct rxbuf *rxb = arg1;
3640 struct rxbuf_cache_params *p = arg2;
3641 size_t real_len;
3642 ddi_dma_cookie_t cookie;
3643 uint_t ccount = 0;
3644 int (*callback)(caddr_t);
3645 int rc = ENOMEM;
3646
3647 if ((kmflag & KM_NOSLEEP) != 0)
3648 callback = DDI_DMA_DONTWAIT;
3649 else
3650 callback = DDI_DMA_SLEEP;
3651
3652 rc = ddi_dma_alloc_handle(p->dip, &p->dma_attr_rx, callback, 0,
3653 &rxb->dhdl);
3654 if (rc != DDI_SUCCESS)
3655 return (rc == DDI_DMA_BADATTR ? EINVAL : ENOMEM);
3656
3657 rc = ddi_dma_mem_alloc(rxb->dhdl, p->buf_size, &p->acc_attr_rx,
3658 DDI_DMA_STREAMING, callback, 0, &rxb->va, &real_len, &rxb->ahdl);
3659 if (rc != DDI_SUCCESS) {
3660 rc = ENOMEM;
3661 goto fail1;
3662 }
3663
3664 rc = ddi_dma_addr_bind_handle(rxb->dhdl, NULL, rxb->va, p->buf_size,
3665 DDI_DMA_READ | DDI_DMA_STREAMING, NULL, NULL, &cookie, &ccount);
3666 if (rc != DDI_DMA_MAPPED) {
3667 if (rc == DDI_DMA_INUSE)
3668 rc = EBUSY;
3669 else if (rc == DDI_DMA_TOOBIG)
3670 rc = E2BIG;
3671 else
3672 rc = ENOMEM;
3673 goto fail2;
3674 }
3675
3676 if (ccount != 1) {
3677 rc = E2BIG;
3678 goto fail3;
3679 }
3680
3681 rxb->ref_cnt = 0;
3682 rxb->buf_size = p->buf_size;
3683 rxb->freefunc.free_arg = (caddr_t)rxb;
3684 rxb->freefunc.free_func = rxbuf_free;
3685 rxb->ba = cookie.dmac_laddress;
3686
3687 return (0);
3688
3689 fail3: (void) ddi_dma_unbind_handle(rxb->dhdl);
3690 fail2: ddi_dma_mem_free(&rxb->ahdl);
3691 fail1: ddi_dma_free_handle(&rxb->dhdl);
3692 return (rc);
3693 }
3694
3695 static void
rxbuf_dtor(void * arg1,void * arg2)3696 rxbuf_dtor(void *arg1, void *arg2)
3697 {
3698 struct rxbuf *rxb = arg1;
3699
3700 (void) ddi_dma_unbind_handle(rxb->dhdl);
3701 ddi_dma_mem_free(&rxb->ahdl);
3702 ddi_dma_free_handle(&rxb->dhdl);
3703 }
3704