1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * This file is part of the Chelsio T4 support code.
14 *
15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved.
16 *
17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
20 * release for licensing terms and conditions.
21 */
22
23 /*
24 * Copyright 2025 Oxide Computer Company
25 */
26
27 #include <sys/ddi.h>
28 #include <sys/sunddi.h>
29 #include <sys/sunndi.h>
30 #include <sys/atomic.h>
31 #include <sys/dlpi.h>
32 #include <sys/pattr.h>
33 #include <sys/strsubr.h>
34 #include <sys/stream.h>
35 #include <sys/strsun.h>
36 #include <inet/ip.h>
37 #include <inet/tcp.h>
38
39 #include "common/common.h"
40 #include "common/t4_msg.h"
41 #include "common/t4_regs.h"
42 #include "common/t4_regs_values.h"
43
44 /* TODO: Tune. */
45 int rx_buf_size = 8192;
46 int tx_copy_threshold = 256;
47 uint16_t rx_copy_threshold = 256;
48
49 /* Used to track coalesced tx work request */
50 struct txpkts {
51 mblk_t *tail; /* head is in the software descriptor */
52 uint64_t *flitp; /* ptr to flit where next pkt should start */
53 uint8_t npkt; /* # of packets in this work request */
54 uint8_t nflits; /* # of flits used by this work request */
55 uint16_t plen; /* total payload (sum of all packets) */
56 };
57
58 /* All information needed to tx a frame */
59 struct txinfo {
60 uint32_t len; /* Total length of frame */
61 uint32_t flags; /* Checksum and LSO flags */
62 uint32_t mss; /* MSS for LSO */
63 uint8_t nsegs; /* # of segments in the SGL, 0 means imm. tx */
64 uint8_t nflits; /* # of flits needed for the SGL */
65 uint8_t hdls_used; /* # of DMA handles used */
66 uint32_t txb_used; /* txb_space used */
67 mac_ether_offload_info_t meoi; /* pkt hdr info for offloads */
68 struct ulptx_sgl sgl __attribute__((aligned(8)));
69 struct ulptx_sge_pair reserved[TX_SGL_SEGS / 2];
70 };
71
72 struct mblk_pair {
73 mblk_t *head, *tail;
74 };
75
76 struct rxbuf {
77 kmem_cache_t *cache; /* the kmem_cache this rxb came from */
78 ddi_dma_handle_t dhdl;
79 ddi_acc_handle_t ahdl;
80 caddr_t va; /* KVA of buffer */
81 uint64_t ba; /* bus address of buffer */
82 frtn_t freefunc;
83 uint_t buf_size;
84 volatile uint_t ref_cnt;
85 };
86
87 static int service_iq(struct sge_iq *iq, int budget);
88 static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx,
89 int8_t pktc_idx, int qsize, uint8_t esize);
90 static inline void init_fl(struct sge_fl *fl, uint16_t qsize);
91 static inline void init_eq(struct adapter *sc, struct sge_eq *eq,
92 uint16_t qsize, uint8_t tx_chan, uint16_t iqid);
93 static int alloc_iq_fl(struct port_info *pi, struct sge_iq *iq,
94 struct sge_fl *fl, int intr_idx, int cong);
95 static int free_iq_fl(struct port_info *pi, struct sge_iq *iq,
96 struct sge_fl *fl);
97 static int alloc_fwq(struct adapter *sc);
98 static int free_fwq(struct adapter *sc);
99 static int alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx,
100 int i);
101 static int free_rxq(struct port_info *pi, struct sge_rxq *rxq);
102 static int eth_eq_alloc(struct adapter *sc, struct port_info *pi,
103 struct sge_eq *eq);
104 static int alloc_eq(struct adapter *sc, struct port_info *pi,
105 struct sge_eq *eq);
106 static int free_eq(struct adapter *sc, struct sge_eq *eq);
107 static int alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx);
108 static int free_txq(struct port_info *pi, struct sge_txq *txq);
109 static int alloc_dma_memory(struct adapter *sc, size_t len, int flags,
110 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr,
111 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
112 caddr_t *pva);
113 static int free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl);
114 static int alloc_desc_ring(struct adapter *sc, size_t len, int rw,
115 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
116 caddr_t *pva);
117 static int free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl);
118 static int alloc_tx_copybuffer(struct adapter *sc, size_t len,
119 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
120 caddr_t *pva);
121 static inline bool is_new_response(const struct sge_iq *iq,
122 struct rsp_ctrl **ctrl);
123 static inline void iq_next(struct sge_iq *iq);
124 static int refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs);
125 static void refill_sfl(void *arg);
126 static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl);
127 static void free_fl_bufs(struct sge_fl *fl);
128 static mblk_t *get_fl_payload(struct adapter *sc, struct sge_fl *fl,
129 uint32_t len_newbuf, int *fl_bufs_used);
130 static int get_frame_txinfo(struct sge_txq *txq, mblk_t **fp,
131 struct txinfo *txinfo, int sgl_only);
132 static inline int fits_in_txb(struct sge_txq *txq, int len, int *waste);
133 static inline int copy_into_txb(struct sge_txq *txq, mblk_t *m, int len,
134 struct txinfo *txinfo);
135 static inline void add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len);
136 static inline int add_mblk(struct sge_txq *txq, struct txinfo *txinfo,
137 mblk_t *m, int len);
138 static void free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo);
139 static int add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m,
140 struct txinfo *txinfo);
141 static void write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts);
142 static int write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m,
143 struct txinfo *txinfo);
144 static inline void write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
145 struct txpkts *txpkts, struct txinfo *txinfo);
146 static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to,
147 int len);
148 static inline void ring_tx_db(struct adapter *sc, struct sge_eq *eq);
149 static int reclaim_tx_descs(struct sge_txq *txq, int howmany);
150 static void write_txqflush_wr(struct sge_txq *txq);
151 static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss,
152 mblk_t *m);
153 static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl);
154 static kstat_t *setup_port_config_kstats(struct port_info *pi);
155 static kstat_t *setup_port_info_kstats(struct port_info *pi);
156 static kstat_t *setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq,
157 int idx);
158 static int update_rxq_kstats(kstat_t *ksp, int rw);
159 static int update_port_info_kstats(kstat_t *ksp, int rw);
160 static kstat_t *setup_txq_kstats(struct port_info *pi, struct sge_txq *txq,
161 int idx);
162 static int update_txq_kstats(kstat_t *ksp, int rw);
163 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *,
164 mblk_t *);
165 static int t4_handle_cpl_msg(struct sge_iq *, const struct rss_header *,
166 mblk_t *);
167 static int t4_handle_fw_msg(struct sge_iq *, const struct rss_header *);
168
169 static kmem_cache_t *rxbuf_cache_create(struct rxbuf_cache_params *);
170 static struct rxbuf *rxbuf_alloc(kmem_cache_t *, int, uint_t);
171 static void rxbuf_free(struct rxbuf *);
172 static int rxbuf_ctor(void *, void *, int);
173 static void rxbuf_dtor(void *, void *);
174
175 static inline int
reclaimable(struct sge_eq * eq)176 reclaimable(struct sge_eq *eq)
177 {
178 unsigned int cidx;
179
180 cidx = eq->spg->cidx; /* stable snapshot */
181 cidx = be16_to_cpu(cidx);
182
183 if (cidx >= eq->cidx)
184 return (cidx - eq->cidx);
185 else
186 return (cidx + eq->cap - eq->cidx);
187 }
188
189 void
t4_sge_init(struct adapter * sc)190 t4_sge_init(struct adapter *sc)
191 {
192 struct driver_properties *p = &sc->props;
193 ddi_dma_attr_t *dma_attr;
194 ddi_device_acc_attr_t *acc_attr;
195 uint32_t sge_control, sge_conm_ctrl;
196 int egress_threshold;
197
198 /*
199 * Device access and DMA attributes for descriptor rings
200 */
201 acc_attr = &sc->sge.acc_attr_desc;
202 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
203 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
204 acc_attr->devacc_attr_dataorder = DDI_STRICTORDER_ACC;
205
206 dma_attr = &sc->sge.dma_attr_desc;
207 dma_attr->dma_attr_version = DMA_ATTR_V0;
208 dma_attr->dma_attr_addr_lo = 0;
209 dma_attr->dma_attr_addr_hi = UINT64_MAX;
210 dma_attr->dma_attr_count_max = UINT64_MAX;
211 dma_attr->dma_attr_align = 512;
212 dma_attr->dma_attr_burstsizes = 0xfff;
213 dma_attr->dma_attr_minxfer = 1;
214 dma_attr->dma_attr_maxxfer = UINT64_MAX;
215 dma_attr->dma_attr_seg = UINT64_MAX;
216 dma_attr->dma_attr_sgllen = 1;
217 dma_attr->dma_attr_granular = 1;
218 dma_attr->dma_attr_flags = 0;
219
220 /*
221 * Device access and DMA attributes for tx buffers
222 */
223 acc_attr = &sc->sge.acc_attr_tx;
224 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
225 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
226
227 dma_attr = &sc->sge.dma_attr_tx;
228 dma_attr->dma_attr_version = DMA_ATTR_V0;
229 dma_attr->dma_attr_addr_lo = 0;
230 dma_attr->dma_attr_addr_hi = UINT64_MAX;
231 dma_attr->dma_attr_count_max = UINT64_MAX;
232 dma_attr->dma_attr_align = 1;
233 dma_attr->dma_attr_burstsizes = 0xfff;
234 dma_attr->dma_attr_minxfer = 1;
235 dma_attr->dma_attr_maxxfer = UINT64_MAX;
236 dma_attr->dma_attr_seg = UINT64_MAX;
237 dma_attr->dma_attr_sgllen = TX_SGL_SEGS;
238 dma_attr->dma_attr_granular = 1;
239 dma_attr->dma_attr_flags = 0;
240
241 /*
242 * Ingress Padding Boundary and Egress Status Page Size are set up by
243 * t4_fixup_host_params().
244 */
245 sge_control = t4_read_reg(sc, A_SGE_CONTROL);
246 sc->sge.pktshift = G_PKTSHIFT(sge_control);
247 sc->sge.stat_len = (sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64;
248
249 /* t4_nex uses FLM packed mode */
250 sc->sge.fl_align = t4_fl_pkt_align(sc, true);
251
252 /*
253 * Device access and DMA attributes for rx buffers
254 */
255 sc->sge.rxb_params.dip = sc->dip;
256 sc->sge.rxb_params.buf_size = rx_buf_size;
257
258 acc_attr = &sc->sge.rxb_params.acc_attr_rx;
259 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
260 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
261
262 dma_attr = &sc->sge.rxb_params.dma_attr_rx;
263 dma_attr->dma_attr_version = DMA_ATTR_V0;
264 dma_attr->dma_attr_addr_lo = 0;
265 dma_attr->dma_attr_addr_hi = UINT64_MAX;
266 dma_attr->dma_attr_count_max = UINT64_MAX;
267 /*
268 * Low 4 bits of an rx buffer address have a special meaning to the SGE
269 * and an rx buf cannot have an address with any of these bits set.
270 * FL_ALIGN is >= 32 so we're sure things are ok.
271 */
272 dma_attr->dma_attr_align = sc->sge.fl_align;
273 dma_attr->dma_attr_burstsizes = 0xfff;
274 dma_attr->dma_attr_minxfer = 1;
275 dma_attr->dma_attr_maxxfer = UINT64_MAX;
276 dma_attr->dma_attr_seg = UINT64_MAX;
277 dma_attr->dma_attr_sgllen = 1;
278 dma_attr->dma_attr_granular = 1;
279 dma_attr->dma_attr_flags = 0;
280
281 sc->sge.rxbuf_cache = rxbuf_cache_create(&sc->sge.rxb_params);
282
283 /*
284 * A FL with <= fl_starve_thres buffers is starving and a periodic
285 * timer will attempt to refill it. This needs to be larger than the
286 * SGE's Egress Congestion Threshold. If it isn't, then we can get
287 * stuck waiting for new packets while the SGE is waiting for us to
288 * give it more Free List entries. (Note that the SGE's Egress
289 * Congestion Threshold is in units of 2 Free List pointers.) For T4,
290 * there was only a single field to control this. For T5 there's the
291 * original field which now only applies to Unpacked Mode Free List
292 * buffers and a new field which only applies to Packed Mode Free List
293 * buffers.
294 */
295
296 sge_conm_ctrl = t4_read_reg(sc, A_SGE_CONM_CTRL);
297 switch (CHELSIO_CHIP_VERSION(sc->params.chip)) {
298 case CHELSIO_T4:
299 egress_threshold = G_EGRTHRESHOLD(sge_conm_ctrl);
300 break;
301 case CHELSIO_T5:
302 egress_threshold = G_EGRTHRESHOLDPACKING(sge_conm_ctrl);
303 break;
304 case CHELSIO_T6:
305 default:
306 egress_threshold = G_T6_EGRTHRESHOLDPACKING(sge_conm_ctrl);
307 }
308 sc->sge.fl_starve_threshold = 2*egress_threshold + 1;
309
310 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, rx_buf_size);
311
312 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD,
313 V_THRESHOLD_0(p->counter_val[0]) |
314 V_THRESHOLD_1(p->counter_val[1]) |
315 V_THRESHOLD_2(p->counter_val[2]) |
316 V_THRESHOLD_3(p->counter_val[3]));
317
318 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1,
319 V_TIMERVALUE0(us_to_core_ticks(sc, p->timer_val[0])) |
320 V_TIMERVALUE1(us_to_core_ticks(sc, p->timer_val[1])));
321 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3,
322 V_TIMERVALUE2(us_to_core_ticks(sc, p->timer_val[2])) |
323 V_TIMERVALUE3(us_to_core_ticks(sc, p->timer_val[3])));
324 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5,
325 V_TIMERVALUE4(us_to_core_ticks(sc, p->timer_val[4])) |
326 V_TIMERVALUE5(us_to_core_ticks(sc, p->timer_val[5])));
327 }
328
329 /*
330 * Allocate and initialize the firmware event queue and the forwarded interrupt
331 * queues, if any. The adapter owns all these queues as they are not associated
332 * with any particular port.
333 *
334 * Returns errno on failure. Resources allocated up to that point may still be
335 * allocated. Caller is responsible for cleanup in case this function fails.
336 */
337 int
t4_setup_adapter_queues(struct adapter * sc)338 t4_setup_adapter_queues(struct adapter *sc)
339 {
340 int rc;
341
342 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
343
344 /*
345 * Firmware event queue
346 */
347 rc = alloc_fwq(sc);
348 if (rc != 0)
349 return (rc);
350
351 return (rc);
352 }
353
354 /*
355 * Idempotent
356 */
357 int
t4_teardown_adapter_queues(struct adapter * sc)358 t4_teardown_adapter_queues(struct adapter *sc)
359 {
360
361 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
362
363 (void) free_fwq(sc);
364
365 return (0);
366 }
367
368 static inline int
first_vector(struct port_info * pi)369 first_vector(struct port_info *pi)
370 {
371 struct adapter *sc = pi->adapter;
372 int rc = T4_EXTRA_INTR, i;
373
374 if (sc->intr_count == 1)
375 return (0);
376
377 for_each_port(sc, i) {
378 struct port_info *p = sc->port[i];
379
380 if (i == pi->port_id)
381 break;
382
383 /*
384 * Not compiled with offload support and intr_count > 1. Only
385 * NIC queues exist and they'd better be taking direct
386 * interrupts.
387 */
388 ASSERT(!(sc->flags & TAF_INTR_FWD));
389 rc += p->nrxq;
390 }
391 return (rc);
392 }
393
394 /*
395 * Given an arbitrary "index," come up with an iq that can be used by other
396 * queues (of this port) for interrupt forwarding, SGE egress updates, etc.
397 * The iq returned is guaranteed to be something that takes direct interrupts.
398 */
399 static struct sge_iq *
port_intr_iq(struct port_info * pi,int idx)400 port_intr_iq(struct port_info *pi, int idx)
401 {
402 struct adapter *sc = pi->adapter;
403 struct sge *s = &sc->sge;
404 struct sge_iq *iq = NULL;
405
406 if (sc->intr_count == 1)
407 return (&sc->sge.fwq);
408
409 /*
410 * Not compiled with offload support and intr_count > 1. Only NIC
411 * queues exist and they'd better be taking direct interrupts.
412 */
413 ASSERT(!(sc->flags & TAF_INTR_FWD));
414
415 idx %= pi->nrxq;
416 iq = &s->rxq[pi->first_rxq + idx].iq;
417
418 return (iq);
419 }
420
421 int
t4_setup_port_queues(struct port_info * pi)422 t4_setup_port_queues(struct port_info *pi)
423 {
424 int rc = 0, i, intr_idx, j;
425 struct sge_rxq *rxq;
426 struct sge_txq *txq;
427 struct adapter *sc = pi->adapter;
428 struct driver_properties *p = &sc->props;
429
430 pi->ksp_config = setup_port_config_kstats(pi);
431 pi->ksp_info = setup_port_info_kstats(pi);
432
433 /* Interrupt vector to start from (when using multiple vectors) */
434 intr_idx = first_vector(pi);
435
436 /*
437 * First pass over all rx queues (NIC and TOE):
438 * a) initialize iq and fl
439 * b) allocate queue iff it will take direct interrupts.
440 */
441
442 for_each_rxq(pi, i, rxq) {
443
444 init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, p->qsize_rxq,
445 RX_IQ_ESIZE);
446
447 init_fl(&rxq->fl, p->qsize_rxq / 8); /* 8 bufs in each entry */
448
449 if ((!(sc->flags & TAF_INTR_FWD)) ||
450 (sc->intr_count > 1 && pi->nrxq)) {
451 rxq->iq.flags |= IQ_INTR;
452 rc = alloc_rxq(pi, rxq, intr_idx, i);
453 if (rc != 0)
454 goto done;
455 intr_idx++;
456 }
457
458 }
459
460 /*
461 * Second pass over all rx queues (NIC and TOE). The queues forwarding
462 * their interrupts are allocated now.
463 */
464 j = 0;
465 for_each_rxq(pi, i, rxq) {
466 if (rxq->iq.flags & IQ_INTR)
467 continue;
468
469 intr_idx = port_intr_iq(pi, j)->abs_id;
470
471 rc = alloc_rxq(pi, rxq, intr_idx, i);
472 if (rc != 0)
473 goto done;
474 j++;
475 }
476
477 /*
478 * Now the tx queues. Only one pass needed.
479 */
480 j = 0;
481 for_each_txq(pi, i, txq) {
482 uint16_t iqid;
483
484 iqid = port_intr_iq(pi, j)->cntxt_id;
485 init_eq(sc, &txq->eq, p->qsize_txq, pi->tx_chan, iqid);
486 rc = alloc_txq(pi, txq, i);
487 if (rc != 0)
488 goto done;
489 }
490
491 done:
492 if (rc != 0)
493 (void) t4_teardown_port_queues(pi);
494
495 return (rc);
496 }
497
498 /*
499 * Idempotent
500 */
501 int
t4_teardown_port_queues(struct port_info * pi)502 t4_teardown_port_queues(struct port_info *pi)
503 {
504 int i;
505 struct sge_rxq *rxq;
506 struct sge_txq *txq;
507
508 if (pi->ksp_config != NULL) {
509 kstat_delete(pi->ksp_config);
510 pi->ksp_config = NULL;
511 }
512 if (pi->ksp_info != NULL) {
513 kstat_delete(pi->ksp_info);
514 pi->ksp_info = NULL;
515 }
516
517 for_each_txq(pi, i, txq) {
518 (void) free_txq(pi, txq);
519 }
520
521 for_each_rxq(pi, i, rxq) {
522 if ((rxq->iq.flags & IQ_INTR) == 0)
523 (void) free_rxq(pi, rxq);
524 }
525
526 /*
527 * Then take down the rx queues that take direct interrupts.
528 */
529
530 for_each_rxq(pi, i, rxq) {
531 if (rxq->iq.flags & IQ_INTR)
532 (void) free_rxq(pi, rxq);
533 }
534
535 return (0);
536 }
537
538 /* Deals with errors and forwarded interrupts */
539 uint_t
t4_intr_all(caddr_t arg1,caddr_t arg2)540 t4_intr_all(caddr_t arg1, caddr_t arg2)
541 {
542
543 (void) t4_intr_err(arg1, arg2);
544 (void) t4_intr(arg1, arg2);
545
546 return (DDI_INTR_CLAIMED);
547 }
548
549 static void
t4_intr_rx_work(struct sge_iq * iq)550 t4_intr_rx_work(struct sge_iq *iq)
551 {
552 mblk_t *mp = NULL;
553 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
554 RXQ_LOCK(rxq);
555 if (!iq->polling) {
556 mp = t4_ring_rx(rxq, iq->qsize/8);
557 t4_write_reg(iq->adapter, MYPF_REG(A_SGE_PF_GTS),
558 V_INGRESSQID((u32)iq->cntxt_id) |
559 V_SEINTARM(iq->intr_next));
560 }
561 RXQ_UNLOCK(rxq);
562 if (mp != NULL) {
563 mac_rx_ring(rxq->port->mh, rxq->ring_handle, mp,
564 rxq->ring_gen_num);
565 }
566 }
567
568 /* Deals with interrupts on the given ingress queue */
569 /* ARGSUSED */
570 uint_t
t4_intr(caddr_t arg1,caddr_t arg2)571 t4_intr(caddr_t arg1, caddr_t arg2)
572 {
573 struct sge_iq *iq = (struct sge_iq *)arg2;
574 int state;
575
576 /*
577 * Right now receive polling is only enabled for MSI-X and
578 * when we have enough msi-x vectors i.e no interrupt forwarding.
579 */
580 if (iq->adapter->props.multi_rings) {
581 t4_intr_rx_work(iq);
582 } else {
583 state = atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY);
584 if (state == IQS_IDLE) {
585 (void) service_iq(iq, 0);
586 (void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE);
587 }
588 }
589 return (DDI_INTR_CLAIMED);
590 }
591
592 /* Deals with error interrupts */
593 /* ARGSUSED */
594 uint_t
t4_intr_err(caddr_t arg1,caddr_t arg2)595 t4_intr_err(caddr_t arg1, caddr_t arg2)
596 {
597 /* LINTED: E_BAD_PTR_CAST_ALIGN */
598 struct adapter *sc = (struct adapter *)arg1;
599
600 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0);
601 (void) t4_slow_intr_handler(sc);
602
603 return (DDI_INTR_CLAIMED);
604 }
605
606 /*
607 * t4_ring_rx - Process responses from an SGE response queue.
608 *
609 * This function processes responses from an SGE response queue up to the
610 * supplied budget. Responses include received packets as well as control
611 * messages from FW or HW.
612 *
613 * It returns a chain of mblks containing the received data, to be
614 * passed up to mac_rx_ring().
615 */
616 mblk_t *
t4_ring_rx(struct sge_rxq * rxq,int budget)617 t4_ring_rx(struct sge_rxq *rxq, int budget)
618 {
619 struct sge_iq *iq = &rxq->iq;
620 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
621 struct adapter *sc = iq->adapter;
622 struct rsp_ctrl *ctrl;
623 const struct rss_header *rss;
624 int ndescs = 0, fl_bufs_used = 0;
625 int rsp_type;
626 uint32_t lq;
627 mblk_t *mblk_head = NULL, **mblk_tail, *m;
628 struct cpl_rx_pkt *cpl;
629 uint32_t received_bytes = 0, pkt_len = 0;
630 bool csum_ok;
631 uint16_t err_vec;
632
633 mblk_tail = &mblk_head;
634
635 while (is_new_response(iq, &ctrl)) {
636
637 membar_consumer();
638
639 m = NULL;
640 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
641 lq = be32_to_cpu(ctrl->pldbuflen_qid);
642 rss = (const void *)iq->cdesc;
643
644 switch (rsp_type) {
645 case X_RSPD_TYPE_FLBUF:
646
647 ASSERT(iq->flags & IQ_HAS_FL);
648
649 if (CPL_RX_PKT == rss->opcode) {
650 cpl = (void *)(rss + 1);
651 pkt_len = be16_to_cpu(cpl->len);
652
653 if (iq->polling &&
654 ((received_bytes + pkt_len) > budget))
655 goto done;
656
657 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
658 if (m == NULL)
659 goto done;
660
661 iq->intr_next = iq->intr_params;
662 m->b_rptr += sc->sge.pktshift;
663 if (sc->params.tp.rx_pkt_encap) {
664 /* Enabled only in T6 config file */
665 err_vec = G_T6_COMPR_RXERR_VEC(
666 ntohs(cpl->err_vec));
667 } else {
668 err_vec = ntohs(cpl->err_vec);
669 }
670
671 csum_ok = cpl->csum_calc && !err_vec;
672
673 /* TODO: what about cpl->ip_frag? */
674 if (csum_ok && !cpl->ip_frag) {
675 mac_hcksum_set(m, 0, 0, 0, 0xffff,
676 HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
677 HCK_IPV4_HDRCKSUM_OK);
678 rxq->rxcsum++;
679 }
680 rxq->rxpkts++;
681 rxq->rxbytes += pkt_len;
682 received_bytes += pkt_len;
683
684 *mblk_tail = m;
685 mblk_tail = &m->b_next;
686
687 break;
688 }
689
690 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
691 if (m == NULL)
692 goto done;
693 /* FALLTHROUGH */
694
695 case X_RSPD_TYPE_CPL:
696 (void) t4_handle_cpl_msg(iq, rss, m);
697 break;
698
699 default:
700 break;
701 }
702 iq_next(iq);
703 ++ndescs;
704 if (!iq->polling && (ndescs == budget))
705 break;
706 }
707
708 done:
709
710 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
711 V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) |
712 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
713
714 if ((fl_bufs_used > 0) || (iq->flags & IQ_HAS_FL)) {
715 int starved;
716 FL_LOCK(fl);
717 fl->needed += fl_bufs_used;
718 starved = refill_fl(sc, fl, fl->cap / 8);
719 FL_UNLOCK(fl);
720 if (starved)
721 add_fl_to_sfl(sc, fl);
722 }
723 return (mblk_head);
724 }
725
726 /*
727 * Deals with anything and everything on the given ingress queue.
728 */
729 static int
service_iq(struct sge_iq * iq,int budget)730 service_iq(struct sge_iq *iq, int budget)
731 {
732 struct sge_iq *q;
733 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
734 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
735 struct adapter *sc = iq->adapter;
736 struct rsp_ctrl *ctrl;
737 const struct rss_header *rss;
738 int ndescs = 0, limit, fl_bufs_used = 0;
739 int rsp_type;
740 uint32_t lq;
741 int starved;
742 mblk_t *m;
743 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
744
745 limit = budget ? budget : iq->qsize / 8;
746
747 /*
748 * We always come back and check the descriptor ring for new indirect
749 * interrupts and other responses after running a single handler.
750 */
751 for (;;) {
752 while (is_new_response(iq, &ctrl)) {
753
754 membar_consumer();
755
756 m = NULL;
757 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
758 lq = be32_to_cpu(ctrl->pldbuflen_qid);
759 rss = (const void *)iq->cdesc;
760
761 switch (rsp_type) {
762 case X_RSPD_TYPE_FLBUF:
763
764 ASSERT(iq->flags & IQ_HAS_FL);
765
766 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
767 if (m == NULL) {
768 /*
769 * Rearm the iq with a
770 * longer-than-default timer
771 */
772 const uint32_t timer_idx =
773 V_QINTR_TIMER_IDX(SGE_NTIMERS-1);
774 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
775 V_CIDXINC(ndescs) |
776 V_INGRESSQID((u32)iq->cntxt_id) |
777 V_SEINTARM(timer_idx));
778 if (fl_bufs_used > 0) {
779 ASSERT(iq->flags & IQ_HAS_FL);
780 FL_LOCK(fl);
781 fl->needed += fl_bufs_used;
782 starved = refill_fl(sc, fl,
783 fl->cap / 8);
784 FL_UNLOCK(fl);
785 if (starved)
786 add_fl_to_sfl(sc, fl);
787 }
788 return (0);
789 }
790
791 /* FALLTHRU */
792 case X_RSPD_TYPE_CPL:
793 (void) t4_handle_cpl_msg(iq, rss, m);
794 break;
795
796 case X_RSPD_TYPE_INTR:
797
798 /*
799 * Interrupts should be forwarded only to queues
800 * that are not forwarding their interrupts.
801 * This means service_iq can recurse but only 1
802 * level deep.
803 */
804 ASSERT(budget == 0);
805
806 q = sc->sge.iqmap[lq - sc->sge.iq_start];
807 if (atomic_cas_uint(&q->state, IQS_IDLE,
808 IQS_BUSY) == IQS_IDLE) {
809 if (service_iq(q, q->qsize / 8) == 0) {
810 (void) atomic_cas_uint(
811 &q->state, IQS_BUSY,
812 IQS_IDLE);
813 } else {
814 STAILQ_INSERT_TAIL(&iql, q,
815 link);
816 }
817 }
818 break;
819
820 default:
821 break;
822 }
823
824 iq_next(iq);
825 if (++ndescs == limit) {
826 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
827 V_CIDXINC(ndescs) |
828 V_INGRESSQID(iq->cntxt_id) |
829 V_SEINTARM(V_QINTR_TIMER_IDX(
830 X_TIMERREG_UPDATE_CIDX)));
831 ndescs = 0;
832
833 if (fl_bufs_used > 0) {
834 ASSERT(iq->flags & IQ_HAS_FL);
835 FL_LOCK(fl);
836 fl->needed += fl_bufs_used;
837 (void) refill_fl(sc, fl, fl->cap / 8);
838 FL_UNLOCK(fl);
839 fl_bufs_used = 0;
840 }
841
842 if (budget != 0)
843 return (EINPROGRESS);
844 }
845 }
846
847 if (STAILQ_EMPTY(&iql) != 0)
848 break;
849
850 /*
851 * Process the head only, and send it to the back of the list if
852 * it's still not done.
853 */
854 q = STAILQ_FIRST(&iql);
855 STAILQ_REMOVE_HEAD(&iql, link);
856 if (service_iq(q, q->qsize / 8) == 0)
857 (void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE);
858 else
859 STAILQ_INSERT_TAIL(&iql, q, link);
860 }
861
862 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
863 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
864
865 if (iq->flags & IQ_HAS_FL) {
866
867 FL_LOCK(fl);
868 fl->needed += fl_bufs_used;
869 starved = refill_fl(sc, fl, fl->cap / 4);
870 FL_UNLOCK(fl);
871 if (starved != 0)
872 add_fl_to_sfl(sc, fl);
873 }
874
875 return (0);
876 }
877
878 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */
879 #define TXPKTS_PKT_HDR ((\
880 sizeof (struct ulp_txpkt) + \
881 sizeof (struct ulptx_idata) + \
882 sizeof (struct cpl_tx_pkt_core)) / 8)
883
884 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */
885 #define TXPKTS_WR_HDR (\
886 sizeof (struct fw_eth_tx_pkts_wr) / 8 + \
887 TXPKTS_PKT_HDR)
888
889 /* Header of a tx WR, before SGL of first packet (in flits) */
890 #define TXPKT_WR_HDR ((\
891 sizeof (struct fw_eth_tx_pkt_wr) + \
892 sizeof (struct cpl_tx_pkt_core)) / 8)
893
894 /* Header of a tx LSO WR, before SGL of first packet (in flits) */
895 #define TXPKT_LSO_WR_HDR ((\
896 sizeof (struct fw_eth_tx_pkt_wr) + \
897 sizeof (struct cpl_tx_pkt_lso_core) + \
898 sizeof (struct cpl_tx_pkt_core)) / 8)
899
900 mblk_t *
t4_eth_tx(void * arg,mblk_t * frame)901 t4_eth_tx(void *arg, mblk_t *frame)
902 {
903 struct sge_txq *txq = (struct sge_txq *)arg;
904 struct port_info *pi = txq->port;
905 struct adapter *sc = pi->adapter;
906 struct sge_eq *eq = &txq->eq;
907 mblk_t *next_frame;
908 int rc, coalescing;
909 struct txpkts txpkts;
910 struct txinfo txinfo;
911
912 txpkts.npkt = 0; /* indicates there's nothing in txpkts */
913 coalescing = 0;
914
915 TXQ_LOCK(txq);
916 if (eq->avail < 8)
917 (void) reclaim_tx_descs(txq, 8);
918 for (; frame; frame = next_frame) {
919
920 if (eq->avail < 8)
921 break;
922
923 next_frame = frame->b_next;
924 frame->b_next = NULL;
925
926 if (next_frame != NULL)
927 coalescing = 1;
928
929 rc = get_frame_txinfo(txq, &frame, &txinfo, coalescing);
930 if (rc != 0) {
931 if (rc == ENOMEM) {
932
933 /* Short of resources, suspend tx */
934
935 frame->b_next = next_frame;
936 break;
937 }
938
939 /*
940 * Unrecoverable error for this frame, throw it
941 * away and move on to the next.
942 */
943
944 freemsg(frame);
945 continue;
946 }
947
948 if (coalescing != 0 &&
949 add_to_txpkts(txq, &txpkts, frame, &txinfo) == 0) {
950
951 /* Successfully absorbed into txpkts */
952
953 write_ulp_cpl_sgl(pi, txq, &txpkts, &txinfo);
954 goto doorbell;
955 }
956
957 /*
958 * We weren't coalescing to begin with, or current frame could
959 * not be coalesced (add_to_txpkts flushes txpkts if a frame
960 * given to it can't be coalesced). Either way there should be
961 * nothing in txpkts.
962 */
963 ASSERT(txpkts.npkt == 0);
964
965 /* We're sending out individual frames now */
966 coalescing = 0;
967
968 if (eq->avail < 8)
969 (void) reclaim_tx_descs(txq, 8);
970 rc = write_txpkt_wr(pi, txq, frame, &txinfo);
971 if (rc != 0) {
972
973 /* Short of hardware descriptors, suspend tx */
974
975 /*
976 * This is an unlikely but expensive failure. We've
977 * done all the hard work (DMA bindings etc.) and now we
978 * can't send out the frame. What's worse, we have to
979 * spend even more time freeing up everything in txinfo.
980 */
981 txq->qfull++;
982 free_txinfo_resources(txq, &txinfo);
983
984 frame->b_next = next_frame;
985 break;
986 }
987
988 doorbell:
989 /* Fewer and fewer doorbells as the queue fills up */
990 if (eq->pending >= (1 << (fls(eq->qsize - eq->avail) / 2))) {
991 txq->txbytes += txinfo.len;
992 txq->txpkts++;
993 ring_tx_db(sc, eq);
994 }
995 (void) reclaim_tx_descs(txq, 32);
996 }
997
998 if (txpkts.npkt > 0)
999 write_txpkts_wr(txq, &txpkts);
1000
1001 /*
1002 * frame not NULL means there was an error but we haven't thrown it
1003 * away. This can happen when we're short of tx descriptors (qfull) or
1004 * maybe even DMA handles (dma_hdl_failed). Either way, a credit flush
1005 * and reclaim will get things going again.
1006 *
1007 * If eq->avail is already 0 we know a credit flush was requested in the
1008 * WR that reduced it to 0 so we don't need another flush (we don't have
1009 * any descriptor for a flush WR anyway, duh).
1010 */
1011 if (frame && eq->avail > 0)
1012 write_txqflush_wr(txq);
1013
1014 if (eq->pending != 0)
1015 ring_tx_db(sc, eq);
1016
1017 (void) reclaim_tx_descs(txq, eq->qsize);
1018 TXQ_UNLOCK(txq);
1019
1020 return (frame);
1021 }
1022
1023 static inline void
init_iq(struct sge_iq * iq,struct adapter * sc,int tmr_idx,int8_t pktc_idx,int qsize,uint8_t esize)1024 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int8_t pktc_idx,
1025 int qsize, uint8_t esize)
1026 {
1027 ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS);
1028 ASSERT(pktc_idx < SGE_NCOUNTERS); /* -ve is ok, means don't use */
1029
1030 iq->flags = 0;
1031 iq->adapter = sc;
1032 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
1033 iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
1034 if (pktc_idx >= 0) {
1035 iq->intr_params |= F_QINTR_CNT_EN;
1036 iq->intr_pktc_idx = pktc_idx;
1037 }
1038 iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */
1039 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */
1040 }
1041
1042 static inline void
init_fl(struct sge_fl * fl,uint16_t qsize)1043 init_fl(struct sge_fl *fl, uint16_t qsize)
1044 {
1045
1046 fl->qsize = qsize;
1047 fl->allocb_fail = 0;
1048 }
1049
1050 static inline void
init_eq(struct adapter * sc,struct sge_eq * eq,uint16_t qsize,uint8_t tx_chan,uint16_t iqid)1051 init_eq(struct adapter *sc, struct sge_eq *eq, uint16_t qsize, uint8_t tx_chan,
1052 uint16_t iqid)
1053 {
1054 struct sge *s = &sc->sge;
1055 uint32_t r;
1056
1057 ASSERT(tx_chan < NCHAN);
1058
1059 if (is_t5(sc->params.chip)) {
1060 r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF);
1061 r >>= S_QUEUESPERPAGEPF0 +
1062 (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf;
1063 s->s_qpp = r & M_QUEUESPERPAGEPF0;
1064 }
1065
1066 eq->flags = 0;
1067 eq->tx_chan = tx_chan;
1068 eq->iqid = iqid;
1069 eq->qsize = qsize;
1070 }
1071
1072 /*
1073 * Allocates the ring for an ingress queue and an optional freelist. If the
1074 * freelist is specified it will be allocated and then associated with the
1075 * ingress queue.
1076 *
1077 * Returns errno on failure. Resources allocated up to that point may still be
1078 * allocated. Caller is responsible for cleanup in case this function fails.
1079 *
1080 * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then
1081 * the intr_idx specifies the vector, starting from 0. Otherwise it specifies
1082 * the index of the queue to which its interrupts will be forwarded.
1083 */
1084 static int
alloc_iq_fl(struct port_info * pi,struct sge_iq * iq,struct sge_fl * fl,int intr_idx,int cong)1085 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
1086 int intr_idx, int cong)
1087 {
1088 int rc, i, cntxt_id;
1089 size_t len;
1090 struct fw_iq_cmd c;
1091 struct adapter *sc = iq->adapter;
1092 uint32_t v = 0;
1093
1094 len = iq->qsize * iq->esize;
1095 rc = alloc_desc_ring(sc, len, DDI_DMA_READ, &iq->dhdl, &iq->ahdl,
1096 &iq->ba, (caddr_t *)&iq->desc);
1097 if (rc != 0)
1098 return (rc);
1099
1100 bzero(&c, sizeof (c));
1101 c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
1102 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
1103 V_FW_IQ_CMD_VFN(0));
1104
1105 c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
1106 FW_LEN16(c));
1107
1108 /* Special handling for firmware event queue */
1109 if (iq == &sc->sge.fwq)
1110 v |= F_FW_IQ_CMD_IQASYNCH;
1111
1112 if (iq->flags & IQ_INTR)
1113 ASSERT(intr_idx < sc->intr_count);
1114 else
1115 v |= F_FW_IQ_CMD_IQANDST;
1116 v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx);
1117
1118 c.type_to_iqandstindex = cpu_to_be32(v |
1119 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
1120 V_FW_IQ_CMD_VIID(pi->viid) |
1121 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
1122 c.iqdroprss_to_iqesize = cpu_to_be16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
1123 F_FW_IQ_CMD_IQGTSMODE |
1124 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) |
1125 V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4));
1126 c.iqsize = cpu_to_be16(iq->qsize);
1127 c.iqaddr = cpu_to_be64(iq->ba);
1128 if (cong >= 0) {
1129 const uint32_t iq_type =
1130 cong ? FW_IQ_IQTYPE_NIC : FW_IQ_IQTYPE_OFLD;
1131 c.iqns_to_fl0congen = BE_32(F_FW_IQ_CMD_IQFLINTCONGEN |
1132 V_FW_IQ_CMD_IQTYPE(iq_type));
1133 }
1134
1135 if (fl != NULL) {
1136 unsigned int chip_ver = CHELSIO_CHIP_VERSION(sc->params.chip);
1137
1138 mutex_init(&fl->lock, NULL, MUTEX_DRIVER,
1139 DDI_INTR_PRI(sc->intr_pri));
1140 fl->flags |= FL_MTX;
1141
1142 len = fl->qsize * RX_FL_ESIZE;
1143 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &fl->dhdl,
1144 &fl->ahdl, &fl->ba, (caddr_t *)&fl->desc);
1145 if (rc != 0)
1146 return (rc);
1147
1148 /* Allocate space for one software descriptor per buffer. */
1149 fl->cap = (fl->qsize - sc->sge.stat_len / RX_FL_ESIZE) * 8;
1150 fl->sdesc = kmem_zalloc(sizeof (struct fl_sdesc) * fl->cap,
1151 KM_SLEEP);
1152 fl->needed = fl->cap;
1153 fl->lowat = roundup(sc->sge.fl_starve_threshold, 8);
1154
1155 c.iqns_to_fl0congen |=
1156 cpu_to_be32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
1157 F_FW_IQ_CMD_FL0PACKEN | F_FW_IQ_CMD_FL0PADEN);
1158 if (cong >= 0) {
1159 c.iqns_to_fl0congen |=
1160 BE_32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) |
1161 F_FW_IQ_CMD_FL0CONGCIF |
1162 F_FW_IQ_CMD_FL0CONGEN);
1163 }
1164
1165 /*
1166 * In T6, for egress queue type FL there is internal overhead
1167 * of 16B for header going into FLM module. Hence the maximum
1168 * allowed burst size is 448 bytes. For T4/T5, the hardware
1169 * doesn't coalesce fetch requests if more than 64 bytes of
1170 * Free List pointers are provided, so we use a 128-byte Fetch
1171 * Burst Minimum there (T6 implements coalescing so we can use
1172 * the smaller 64-byte value there).
1173 */
1174
1175 c.fl0dcaen_to_fl0cidxfthresh = cpu_to_be16(
1176 V_FW_IQ_CMD_FL0FBMIN(chip_ver <= CHELSIO_T5 ?
1177 X_FETCHBURSTMIN_128B : X_FETCHBURSTMIN_64B) |
1178 V_FW_IQ_CMD_FL0FBMAX(chip_ver <= CHELSIO_T5 ?
1179 X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B));
1180 c.fl0size = cpu_to_be16(fl->qsize);
1181 c.fl0addr = cpu_to_be64(fl->ba);
1182 }
1183
1184 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1185 if (rc != 0) {
1186 cxgb_printf(sc->dip, CE_WARN,
1187 "failed to create ingress queue: %d", rc);
1188 return (rc);
1189 }
1190
1191 iq->cdesc = iq->desc;
1192 iq->cidx = 0;
1193 iq->gen = 1;
1194 iq->intr_next = iq->intr_params;
1195 iq->adapter = sc;
1196 iq->cntxt_id = be16_to_cpu(c.iqid);
1197 iq->abs_id = be16_to_cpu(c.physiqid);
1198 iq->flags |= IQ_ALLOCATED;
1199 mutex_init(&iq->lock, NULL, MUTEX_DRIVER,
1200 DDI_INTR_PRI(DDI_INTR_PRI(sc->intr_pri)));
1201 iq->polling = 0;
1202
1203 cntxt_id = iq->cntxt_id - sc->sge.iq_start;
1204 if (cntxt_id >= sc->sge.iqmap_sz) {
1205 panic("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
1206 cntxt_id, sc->sge.iqmap_sz - 1);
1207 }
1208 sc->sge.iqmap[cntxt_id] = iq;
1209
1210 if (fl != NULL) {
1211 fl->cntxt_id = be16_to_cpu(c.fl0id);
1212 fl->pidx = fl->cidx = 0;
1213 fl->copy_threshold = rx_copy_threshold;
1214
1215 cntxt_id = fl->cntxt_id - sc->sge.eq_start;
1216 if (cntxt_id >= sc->sge.eqmap_sz) {
1217 panic("%s: fl->cntxt_id (%d) more than the max (%d)",
1218 __func__, cntxt_id, sc->sge.eqmap_sz - 1);
1219 }
1220 sc->sge.eqmap[cntxt_id] = (void *)fl;
1221
1222 FL_LOCK(fl);
1223 (void) refill_fl(sc, fl, fl->lowat);
1224 FL_UNLOCK(fl);
1225
1226 iq->flags |= IQ_HAS_FL;
1227 }
1228
1229 if (is_t5(sc->params.chip) && cong >= 0) {
1230 uint32_t param, val;
1231
1232 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1233 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
1234 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id);
1235 if (cong == 0)
1236 val = 1 << 19;
1237 else {
1238 val = 2 << 19;
1239 for (i = 0; i < 4; i++) {
1240 if (cong & (1 << i))
1241 val |= 1 << (i << 2);
1242 }
1243 }
1244
1245 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
1246 if (rc != 0) {
1247 /* report error but carry on */
1248 cxgb_printf(sc->dip, CE_WARN,
1249 "failed to set congestion manager context for "
1250 "ingress queue %d: %d", iq->cntxt_id, rc);
1251 }
1252 }
1253
1254 /* Enable IQ interrupts */
1255 iq->state = IQS_IDLE;
1256 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) |
1257 V_INGRESSQID(iq->cntxt_id));
1258
1259 return (0);
1260 }
1261
1262 static int
free_iq_fl(struct port_info * pi,struct sge_iq * iq,struct sge_fl * fl)1263 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl)
1264 {
1265 int rc;
1266
1267 if (iq != NULL) {
1268 struct adapter *sc = iq->adapter;
1269 dev_info_t *dip;
1270
1271 dip = pi ? pi->dip : sc->dip;
1272 if (iq->flags & IQ_ALLOCATED) {
1273 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0,
1274 FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id,
1275 fl ? fl->cntxt_id : 0xffff, 0xffff);
1276 if (rc != 0) {
1277 cxgb_printf(dip, CE_WARN,
1278 "failed to free queue %p: %d", iq, rc);
1279 return (rc);
1280 }
1281 mutex_destroy(&iq->lock);
1282 iq->flags &= ~IQ_ALLOCATED;
1283 }
1284
1285 if (iq->desc != NULL) {
1286 (void) free_desc_ring(&iq->dhdl, &iq->ahdl);
1287 iq->desc = NULL;
1288 }
1289
1290 bzero(iq, sizeof (*iq));
1291 }
1292
1293 if (fl != NULL) {
1294 if (fl->sdesc != NULL) {
1295 FL_LOCK(fl);
1296 free_fl_bufs(fl);
1297 FL_UNLOCK(fl);
1298
1299 kmem_free(fl->sdesc, sizeof (struct fl_sdesc) *
1300 fl->cap);
1301 fl->sdesc = NULL;
1302 }
1303
1304 if (fl->desc != NULL) {
1305 (void) free_desc_ring(&fl->dhdl, &fl->ahdl);
1306 fl->desc = NULL;
1307 }
1308
1309 if (fl->flags & FL_MTX) {
1310 mutex_destroy(&fl->lock);
1311 fl->flags &= ~FL_MTX;
1312 }
1313
1314 bzero(fl, sizeof (struct sge_fl));
1315 }
1316
1317 return (0);
1318 }
1319
1320 static int
alloc_fwq(struct adapter * sc)1321 alloc_fwq(struct adapter *sc)
1322 {
1323 int rc, intr_idx;
1324 struct sge_iq *fwq = &sc->sge.fwq;
1325
1326 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE);
1327 fwq->flags |= IQ_INTR; /* always */
1328 intr_idx = sc->intr_count > 1 ? 1 : 0;
1329 rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1);
1330 if (rc != 0) {
1331 cxgb_printf(sc->dip, CE_WARN,
1332 "failed to create firmware event queue: %d.", rc);
1333 return (rc);
1334 }
1335
1336 return (0);
1337 }
1338
1339 static int
free_fwq(struct adapter * sc)1340 free_fwq(struct adapter *sc)
1341 {
1342
1343 return (free_iq_fl(NULL, &sc->sge.fwq, NULL));
1344 }
1345
1346 static int
alloc_rxq(struct port_info * pi,struct sge_rxq * rxq,int intr_idx,int i)1347 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int i)
1348 {
1349 int rc;
1350
1351 rxq->port = pi;
1352 rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx,
1353 t4_get_tp_ch_map(pi->adapter, pi->tx_chan));
1354 if (rc != 0)
1355 return (rc);
1356
1357 rxq->ksp = setup_rxq_kstats(pi, rxq, i);
1358
1359 return (rc);
1360 }
1361
1362 static int
free_rxq(struct port_info * pi,struct sge_rxq * rxq)1363 free_rxq(struct port_info *pi, struct sge_rxq *rxq)
1364 {
1365 int rc;
1366
1367 if (rxq->ksp != NULL) {
1368 kstat_delete(rxq->ksp);
1369 rxq->ksp = NULL;
1370 }
1371
1372 rc = free_iq_fl(pi, &rxq->iq, &rxq->fl);
1373 if (rc == 0)
1374 bzero(&rxq->fl, sizeof (*rxq) - offsetof(struct sge_rxq, fl));
1375
1376 return (rc);
1377 }
1378
1379 static int
eth_eq_alloc(struct adapter * sc,struct port_info * pi,struct sge_eq * eq)1380 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1381 {
1382 int rc, cntxt_id;
1383 struct fw_eq_eth_cmd c;
1384
1385 bzero(&c, sizeof (c));
1386
1387 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
1388 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
1389 V_FW_EQ_ETH_CMD_VFN(0));
1390 c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC |
1391 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
1392 c.autoequiqe_to_viid = BE_32(F_FW_EQ_ETH_CMD_AUTOEQUIQE |
1393 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(pi->viid));
1394 c.fetchszm_to_iqid =
1395 BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1396 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
1397 V_FW_EQ_ETH_CMD_IQID(eq->iqid));
1398 c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1399 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1400 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1401 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize));
1402 c.eqaddr = BE_64(eq->ba);
1403
1404 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1405 if (rc != 0) {
1406 cxgb_printf(pi->dip, CE_WARN,
1407 "failed to create Ethernet egress queue: %d", rc);
1408 return (rc);
1409 }
1410 eq->flags |= EQ_ALLOCATED;
1411
1412 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd));
1413 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1414 if (cntxt_id >= sc->sge.eqmap_sz)
1415 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1416 cntxt_id, sc->sge.eqmap_sz - 1);
1417 sc->sge.eqmap[cntxt_id] = eq;
1418
1419 return (rc);
1420 }
1421
1422 static int
alloc_eq(struct adapter * sc,struct port_info * pi,struct sge_eq * eq)1423 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1424 {
1425 int rc;
1426 size_t len;
1427
1428 mutex_init(&eq->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(sc->intr_pri));
1429 eq->flags |= EQ_MTX;
1430
1431 len = eq->qsize * EQ_ESIZE;
1432 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &eq->desc_dhdl,
1433 &eq->desc_ahdl, &eq->ba, (caddr_t *)&eq->desc);
1434 if (rc != 0)
1435 return (rc);
1436
1437 eq->cap = eq->qsize - sc->sge.stat_len / EQ_ESIZE;
1438 eq->spg = (void *)&eq->desc[eq->cap];
1439 eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */
1440 eq->pidx = eq->cidx = 0;
1441 eq->doorbells = sc->doorbells;
1442
1443 rc = eth_eq_alloc(sc, pi, eq);
1444
1445 if (eq->doorbells & (DOORBELL_UDB | DOORBELL_UDBWC | DOORBELL_WCWR)) {
1446 uint32_t s_qpp = sc->sge.s_qpp;
1447 uint32_t mask = (1 << s_qpp) - 1;
1448 volatile uint8_t *udb;
1449
1450 udb = (volatile uint8_t *)sc->reg1p + UDBS_DB_OFFSET;
1451 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */
1452 eq->udb_qid = eq->cntxt_id & mask; /* id in page */
1453 if (eq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE)
1454 eq->doorbells &= ~DOORBELL_WCWR;
1455 else {
1456 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */
1457 eq->udb_qid = 0;
1458 }
1459 eq->udb = (volatile void *)udb;
1460 }
1461
1462 if (rc != 0) {
1463 cxgb_printf(sc->dip, CE_WARN,
1464 "failed to allocate egress queue: %d", rc);
1465 }
1466
1467 return (rc);
1468 }
1469
1470 static int
free_eq(struct adapter * sc,struct sge_eq * eq)1471 free_eq(struct adapter *sc, struct sge_eq *eq)
1472 {
1473 int rc;
1474
1475 if (eq->flags & EQ_ALLOCATED) {
1476 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id);
1477 if (rc != 0) {
1478 cxgb_printf(sc->dip, CE_WARN,
1479 "failed to free egress queue: %d", rc);
1480 return (rc);
1481 }
1482 eq->flags &= ~EQ_ALLOCATED;
1483 }
1484
1485 if (eq->desc != NULL) {
1486 (void) free_desc_ring(&eq->desc_dhdl, &eq->desc_ahdl);
1487 eq->desc = NULL;
1488 }
1489
1490 if (eq->flags & EQ_MTX)
1491 mutex_destroy(&eq->lock);
1492
1493 bzero(eq, sizeof (*eq));
1494 return (0);
1495 }
1496
1497 static int
alloc_txq(struct port_info * pi,struct sge_txq * txq,int idx)1498 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx)
1499 {
1500 int rc, i;
1501 struct adapter *sc = pi->adapter;
1502 struct sge_eq *eq = &txq->eq;
1503
1504 rc = alloc_eq(sc, pi, eq);
1505 if (rc != 0)
1506 return (rc);
1507
1508 txq->port = pi;
1509 txq->sdesc = kmem_zalloc(sizeof (struct tx_sdesc) * eq->cap, KM_SLEEP);
1510 txq->txb_size = eq->qsize * tx_copy_threshold;
1511 rc = alloc_tx_copybuffer(sc, txq->txb_size, &txq->txb_dhdl,
1512 &txq->txb_ahdl, &txq->txb_ba, &txq->txb_va);
1513 if (rc == 0)
1514 txq->txb_avail = txq->txb_size;
1515 else
1516 txq->txb_avail = txq->txb_size = 0;
1517
1518 /*
1519 * TODO: is this too low? Worst case would need around 4 times qsize
1520 * (all tx descriptors filled to the brim with SGLs, with each entry in
1521 * the SGL coming from a distinct DMA handle). Increase tx_dhdl_total
1522 * if you see too many dma_hdl_failed.
1523 */
1524 txq->tx_dhdl_total = eq->qsize * 2;
1525 txq->tx_dhdl = kmem_zalloc(sizeof (ddi_dma_handle_t) *
1526 txq->tx_dhdl_total, KM_SLEEP);
1527 for (i = 0; i < txq->tx_dhdl_total; i++) {
1528 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx,
1529 DDI_DMA_SLEEP, 0, &txq->tx_dhdl[i]);
1530 if (rc != DDI_SUCCESS) {
1531 cxgb_printf(sc->dip, CE_WARN,
1532 "%s: failed to allocate DMA handle (%d)",
1533 __func__, rc);
1534 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
1535 }
1536 txq->tx_dhdl_avail++;
1537 }
1538
1539 txq->ksp = setup_txq_kstats(pi, txq, idx);
1540
1541 return (rc);
1542 }
1543
1544 static int
free_txq(struct port_info * pi,struct sge_txq * txq)1545 free_txq(struct port_info *pi, struct sge_txq *txq)
1546 {
1547 int i;
1548 struct adapter *sc = pi->adapter;
1549 struct sge_eq *eq = &txq->eq;
1550
1551 if (txq->ksp != NULL) {
1552 kstat_delete(txq->ksp);
1553 txq->ksp = NULL;
1554 }
1555
1556 if (txq->txb_va != NULL) {
1557 (void) free_desc_ring(&txq->txb_dhdl, &txq->txb_ahdl);
1558 txq->txb_va = NULL;
1559 }
1560
1561 if (txq->sdesc != NULL) {
1562 struct tx_sdesc *sd;
1563 ddi_dma_handle_t hdl;
1564
1565 TXQ_LOCK(txq);
1566 while (eq->cidx != eq->pidx) {
1567 sd = &txq->sdesc[eq->cidx];
1568
1569 for (i = sd->hdls_used; i; i--) {
1570 hdl = txq->tx_dhdl[txq->tx_dhdl_cidx];
1571 (void) ddi_dma_unbind_handle(hdl);
1572 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total)
1573 txq->tx_dhdl_cidx = 0;
1574 }
1575
1576 ASSERT(sd->m);
1577 freemsgchain(sd->m);
1578
1579 eq->cidx += sd->desc_used;
1580 if (eq->cidx >= eq->cap)
1581 eq->cidx -= eq->cap;
1582
1583 txq->txb_avail += txq->txb_used;
1584 }
1585 ASSERT(txq->tx_dhdl_cidx == txq->tx_dhdl_pidx);
1586 ASSERT(txq->txb_avail == txq->txb_size);
1587 TXQ_UNLOCK(txq);
1588
1589 kmem_free(txq->sdesc, sizeof (struct tx_sdesc) * eq->cap);
1590 txq->sdesc = NULL;
1591 }
1592
1593 if (txq->tx_dhdl != NULL) {
1594 for (i = 0; i < txq->tx_dhdl_total; i++) {
1595 if (txq->tx_dhdl[i] != NULL)
1596 ddi_dma_free_handle(&txq->tx_dhdl[i]);
1597 }
1598 kmem_free(txq->tx_dhdl,
1599 sizeof (ddi_dma_handle_t) * txq->tx_dhdl_total);
1600 txq->tx_dhdl = NULL;
1601 }
1602
1603 (void) free_eq(sc, &txq->eq);
1604
1605 bzero(txq, sizeof (*txq));
1606 return (0);
1607 }
1608
1609 /*
1610 * Allocates a block of contiguous memory for DMA. Can be used to allocate
1611 * memory for descriptor rings or for tx/rx copy buffers.
1612 *
1613 * Caller does not have to clean up anything if this function fails, it cleans
1614 * up after itself.
1615 *
1616 * Caller provides the following:
1617 * len length of the block of memory to allocate.
1618 * flags DDI_DMA_* flags to use (CONSISTENT/STREAMING, READ/WRITE/RDWR)
1619 * acc_attr device access attributes for the allocation.
1620 * dma_attr DMA attributes for the allocation
1621 *
1622 * If the function is successful it fills up this information:
1623 * dma_hdl DMA handle for the allocated memory
1624 * acc_hdl access handle for the allocated memory
1625 * ba bus address of the allocated memory
1626 * va KVA of the allocated memory.
1627 */
1628 static int
alloc_dma_memory(struct adapter * sc,size_t len,int flags,ddi_device_acc_attr_t * acc_attr,ddi_dma_attr_t * dma_attr,ddi_dma_handle_t * dma_hdl,ddi_acc_handle_t * acc_hdl,uint64_t * pba,caddr_t * pva)1629 alloc_dma_memory(struct adapter *sc, size_t len, int flags,
1630 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr,
1631 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
1632 uint64_t *pba, caddr_t *pva)
1633 {
1634 int rc;
1635 ddi_dma_handle_t dhdl;
1636 ddi_acc_handle_t ahdl;
1637 ddi_dma_cookie_t cookie;
1638 uint_t ccount;
1639 caddr_t va;
1640 size_t real_len;
1641
1642 *pva = NULL;
1643
1644 /*
1645 * DMA handle.
1646 */
1647 rc = ddi_dma_alloc_handle(sc->dip, dma_attr, DDI_DMA_SLEEP, 0, &dhdl);
1648 if (rc != DDI_SUCCESS) {
1649 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
1650 }
1651
1652 /*
1653 * Memory suitable for DMA.
1654 */
1655 rc = ddi_dma_mem_alloc(dhdl, len, acc_attr,
1656 flags & DDI_DMA_CONSISTENT ? DDI_DMA_CONSISTENT : DDI_DMA_STREAMING,
1657 DDI_DMA_SLEEP, 0, &va, &real_len, &ahdl);
1658 if (rc != DDI_SUCCESS) {
1659 ddi_dma_free_handle(&dhdl);
1660 return (ENOMEM);
1661 }
1662
1663 /*
1664 * DMA bindings.
1665 */
1666 rc = ddi_dma_addr_bind_handle(dhdl, NULL, va, real_len, flags, NULL,
1667 NULL, &cookie, &ccount);
1668 if (rc != DDI_DMA_MAPPED) {
1669 ddi_dma_mem_free(&ahdl);
1670 ddi_dma_free_handle(&dhdl);
1671 return (ENOMEM);
1672 }
1673 if (ccount != 1) {
1674 /* unusable DMA mapping */
1675 (void) free_desc_ring(&dhdl, &ahdl);
1676 return (ENOMEM);
1677 }
1678
1679 bzero(va, real_len);
1680 *dma_hdl = dhdl;
1681 *acc_hdl = ahdl;
1682 *pba = cookie.dmac_laddress;
1683 *pva = va;
1684
1685 return (0);
1686 }
1687
1688 static int
free_dma_memory(ddi_dma_handle_t * dhdl,ddi_acc_handle_t * ahdl)1689 free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl)
1690 {
1691 (void) ddi_dma_unbind_handle(*dhdl);
1692 ddi_dma_mem_free(ahdl);
1693 ddi_dma_free_handle(dhdl);
1694
1695 return (0);
1696 }
1697
1698 static int
alloc_desc_ring(struct adapter * sc,size_t len,int rw,ddi_dma_handle_t * dma_hdl,ddi_acc_handle_t * acc_hdl,uint64_t * pba,caddr_t * pva)1699 alloc_desc_ring(struct adapter *sc, size_t len, int rw,
1700 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
1701 uint64_t *pba, caddr_t *pva)
1702 {
1703 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_desc;
1704 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc;
1705
1706 return (alloc_dma_memory(sc, len, DDI_DMA_CONSISTENT | rw, acc_attr,
1707 dma_attr, dma_hdl, acc_hdl, pba, pva));
1708 }
1709
1710 static int
free_desc_ring(ddi_dma_handle_t * dhdl,ddi_acc_handle_t * ahdl)1711 free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl)
1712 {
1713 return (free_dma_memory(dhdl, ahdl));
1714 }
1715
1716 static int
alloc_tx_copybuffer(struct adapter * sc,size_t len,ddi_dma_handle_t * dma_hdl,ddi_acc_handle_t * acc_hdl,uint64_t * pba,caddr_t * pva)1717 alloc_tx_copybuffer(struct adapter *sc, size_t len,
1718 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
1719 uint64_t *pba, caddr_t *pva)
1720 {
1721 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_tx;
1722 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; /* NOT dma_attr_tx */
1723
1724 return (alloc_dma_memory(sc, len, DDI_DMA_STREAMING | DDI_DMA_WRITE,
1725 acc_attr, dma_attr, dma_hdl, acc_hdl, pba, pva));
1726 }
1727
1728 static inline bool
is_new_response(const struct sge_iq * iq,struct rsp_ctrl ** ctrl)1729 is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl)
1730 {
1731 (void) ddi_dma_sync(iq->dhdl, (uintptr_t)iq->cdesc -
1732 (uintptr_t)iq->desc, iq->esize, DDI_DMA_SYNC_FORKERNEL);
1733
1734 *ctrl = (void *)((uintptr_t)iq->cdesc +
1735 (iq->esize - sizeof (struct rsp_ctrl)));
1736
1737 return ((((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen));
1738 }
1739
1740 static inline void
iq_next(struct sge_iq * iq)1741 iq_next(struct sge_iq *iq)
1742 {
1743 iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize);
1744 if (++iq->cidx == iq->qsize - 1) {
1745 iq->cidx = 0;
1746 iq->gen ^= 1;
1747 iq->cdesc = iq->desc;
1748 }
1749 }
1750
1751 /*
1752 * Fill up the freelist by upto nbufs and maybe ring its doorbell.
1753 *
1754 * Returns non-zero to indicate that it should be added to the list of starving
1755 * freelists.
1756 */
1757 static int
refill_fl(struct adapter * sc,struct sge_fl * fl,int nbufs)1758 refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
1759 {
1760 uint64_t *d = &fl->desc[fl->pidx];
1761 struct fl_sdesc *sd = &fl->sdesc[fl->pidx];
1762
1763 FL_LOCK_ASSERT_OWNED(fl);
1764 ASSERT(nbufs >= 0);
1765
1766 if (nbufs > fl->needed)
1767 nbufs = fl->needed;
1768
1769 while (nbufs--) {
1770 if (sd->rxb != NULL) {
1771 if (sd->rxb->ref_cnt == 1) {
1772 /*
1773 * Buffer is available for recycling. Two ways
1774 * this can happen:
1775 *
1776 * a) All the packets DMA'd into it last time
1777 * around were within the rx_copy_threshold
1778 * and no part of the buffer was ever passed
1779 * up (ref_cnt never went over 1).
1780 *
1781 * b) Packets DMA'd into the buffer were passed
1782 * up but have all been freed by the upper
1783 * layers by now (ref_cnt went over 1 but is
1784 * now back to 1).
1785 *
1786 * Either way the bus address in the descriptor
1787 * ring is already valid.
1788 */
1789 ASSERT(*d == cpu_to_be64(sd->rxb->ba));
1790 d++;
1791 goto recycled;
1792 } else {
1793 /*
1794 * Buffer still in use and we need a
1795 * replacement. But first release our reference
1796 * on the existing buffer.
1797 */
1798 rxbuf_free(sd->rxb);
1799 }
1800 }
1801
1802 sd->rxb = rxbuf_alloc(sc->sge.rxbuf_cache, KM_NOSLEEP, 1);
1803 if (sd->rxb == NULL)
1804 break;
1805 *d++ = cpu_to_be64(sd->rxb->ba);
1806
1807 recycled: fl->pending++;
1808 sd++;
1809 fl->needed--;
1810 if (++fl->pidx == fl->cap) {
1811 fl->pidx = 0;
1812 sd = fl->sdesc;
1813 d = fl->desc;
1814 }
1815 }
1816
1817 if (fl->pending >= 8)
1818 ring_fl_db(sc, fl);
1819
1820 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
1821 }
1822
1823 #ifndef TAILQ_FOREACH_SAFE
1824 #define TAILQ_FOREACH_SAFE(var, head, field, tvar) \
1825 for ((var) = TAILQ_FIRST((head)); \
1826 (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \
1827 (var) = (tvar))
1828 #endif
1829
1830 /*
1831 * Attempt to refill all starving freelists.
1832 */
1833 static void
refill_sfl(void * arg)1834 refill_sfl(void *arg)
1835 {
1836 struct adapter *sc = arg;
1837 struct sge_fl *fl, *fl_temp;
1838
1839 mutex_enter(&sc->sfl_lock);
1840 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) {
1841 FL_LOCK(fl);
1842 (void) refill_fl(sc, fl, 64);
1843 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) {
1844 TAILQ_REMOVE(&sc->sfl, fl, link);
1845 fl->flags &= ~FL_STARVING;
1846 }
1847 FL_UNLOCK(fl);
1848 }
1849
1850 if (!TAILQ_EMPTY(&sc->sfl) != 0)
1851 sc->sfl_timer = timeout(refill_sfl, sc, drv_usectohz(100000));
1852 mutex_exit(&sc->sfl_lock);
1853 }
1854
1855 static void
add_fl_to_sfl(struct adapter * sc,struct sge_fl * fl)1856 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl)
1857 {
1858 mutex_enter(&sc->sfl_lock);
1859 FL_LOCK(fl);
1860 if ((fl->flags & FL_DOOMED) == 0) {
1861 if (TAILQ_EMPTY(&sc->sfl) != 0) {
1862 sc->sfl_timer = timeout(refill_sfl, sc,
1863 drv_usectohz(100000));
1864 }
1865 fl->flags |= FL_STARVING;
1866 TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
1867 }
1868 FL_UNLOCK(fl);
1869 mutex_exit(&sc->sfl_lock);
1870 }
1871
1872 static void
free_fl_bufs(struct sge_fl * fl)1873 free_fl_bufs(struct sge_fl *fl)
1874 {
1875 struct fl_sdesc *sd;
1876 unsigned int i;
1877
1878 FL_LOCK_ASSERT_OWNED(fl);
1879
1880 for (i = 0; i < fl->cap; i++) {
1881 sd = &fl->sdesc[i];
1882
1883 if (sd->rxb != NULL) {
1884 rxbuf_free(sd->rxb);
1885 sd->rxb = NULL;
1886 }
1887 }
1888 }
1889
1890 /*
1891 * Note that fl->cidx and fl->offset are left unchanged in case of failure.
1892 */
1893 static mblk_t *
get_fl_payload(struct adapter * sc,struct sge_fl * fl,uint32_t len_newbuf,int * fl_bufs_used)1894 get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf,
1895 int *fl_bufs_used)
1896 {
1897 struct mblk_pair frame = {0};
1898 struct rxbuf *rxb;
1899 mblk_t *m = NULL;
1900 uint_t nbuf = 0, len, copy, n;
1901 uint32_t cidx, offset, rcidx, roffset;
1902
1903 /*
1904 * The SGE won't pack a new frame into the current buffer if the entire
1905 * payload doesn't fit in the remaining space. Move on to the next buf
1906 * in that case.
1907 */
1908 rcidx = fl->cidx;
1909 roffset = fl->offset;
1910 if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) {
1911 fl->offset = 0;
1912 if (++fl->cidx == fl->cap)
1913 fl->cidx = 0;
1914 nbuf++;
1915 }
1916 cidx = fl->cidx;
1917 offset = fl->offset;
1918
1919 len = G_RSPD_LEN(len_newbuf); /* pktshift + payload length */
1920 copy = (len <= fl->copy_threshold);
1921 if (copy != 0) {
1922 frame.head = m = allocb(len, BPRI_HI);
1923 if (m == NULL) {
1924 fl->allocb_fail++;
1925 DTRACE_PROBE1(t4__fl_alloc_fail, struct sge_fl *, fl);
1926 fl->cidx = rcidx;
1927 fl->offset = roffset;
1928 return (NULL);
1929 }
1930 }
1931
1932 while (len) {
1933 rxb = fl->sdesc[cidx].rxb;
1934 n = min(len, rxb->buf_size - offset);
1935
1936 (void) ddi_dma_sync(rxb->dhdl, offset, n,
1937 DDI_DMA_SYNC_FORKERNEL);
1938
1939 if (copy != 0)
1940 bcopy(rxb->va + offset, m->b_wptr, n);
1941 else {
1942 m = desballoc((unsigned char *)rxb->va + offset, n,
1943 BPRI_HI, &rxb->freefunc);
1944 if (m == NULL) {
1945 fl->allocb_fail++;
1946 DTRACE_PROBE1(t4__fl_alloc_fail,
1947 struct sge_fl *, fl);
1948 if (frame.head)
1949 freemsgchain(frame.head);
1950 fl->cidx = rcidx;
1951 fl->offset = roffset;
1952 return (NULL);
1953 }
1954 atomic_inc_uint(&rxb->ref_cnt);
1955 if (frame.head != NULL)
1956 frame.tail->b_cont = m;
1957 else
1958 frame.head = m;
1959 frame.tail = m;
1960 }
1961 m->b_wptr += n;
1962 len -= n;
1963 offset += roundup(n, sc->sge.fl_align);
1964 ASSERT(offset <= rxb->buf_size);
1965 if (offset == rxb->buf_size) {
1966 offset = 0;
1967 if (++cidx == fl->cap)
1968 cidx = 0;
1969 nbuf++;
1970 }
1971 }
1972
1973 fl->cidx = cidx;
1974 fl->offset = offset;
1975 (*fl_bufs_used) += nbuf;
1976
1977 ASSERT(frame.head != NULL);
1978 return (frame.head);
1979 }
1980
1981 /*
1982 * We'll do immediate data tx for non-LSO, but only when not coalescing. We're
1983 * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes
1984 * of immediate data.
1985 */
1986 #define IMM_LEN ( \
1987 2 * EQ_ESIZE \
1988 - sizeof (struct fw_eth_tx_pkt_wr) \
1989 - sizeof (struct cpl_tx_pkt_core))
1990
1991 /*
1992 * Returns non-zero on failure, no need to cleanup anything in that case.
1993 *
1994 * Note 1: We always try to pull up the mblk if required and return E2BIG only
1995 * if this fails.
1996 *
1997 * Note 2: We'll also pullup incoming mblk if HW_LSO is set and the first mblk
1998 * does not have the TCP header in it.
1999 */
2000 static int
get_frame_txinfo(struct sge_txq * txq,mblk_t ** fp,struct txinfo * txinfo,int sgl_only)2001 get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, struct txinfo *txinfo,
2002 int sgl_only)
2003 {
2004 uint32_t flags = 0, len, n;
2005 mblk_t *m = *fp;
2006 int rc;
2007
2008 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb and dma_hdls */
2009
2010 mac_hcksum_get(m, NULL, NULL, NULL, NULL, &flags);
2011 txinfo->flags = (flags & HCK_TX_FLAGS);
2012
2013 mac_lso_get(m, &txinfo->mss, &flags);
2014 txinfo->flags |= (flags & HW_LSO_FLAGS);
2015
2016 if (flags & HW_LSO)
2017 sgl_only = 1; /* Do not allow immediate data with LSO */
2018
2019 /*
2020 * If checksum or segmentation offloads are requested, gather
2021 * information about the sizes and types of headers in the packet.
2022 */
2023 if (txinfo->flags != 0) {
2024 mac_ether_offload_info(m, &txinfo->meoi);
2025 } else {
2026 bzero(&txinfo->meoi, sizeof (txinfo->meoi));
2027 }
2028
2029 start: txinfo->nsegs = 0;
2030 txinfo->hdls_used = 0;
2031 txinfo->txb_used = 0;
2032 txinfo->len = 0;
2033
2034 /* total length and a rough estimate of # of segments */
2035 n = 0;
2036 for (; m; m = m->b_cont) {
2037 len = MBLKL(m);
2038 n += (len / PAGE_SIZE) + 1;
2039 txinfo->len += len;
2040 }
2041 m = *fp;
2042
2043 if (n >= TX_SGL_SEGS || (flags & HW_LSO && MBLKL(m) < 50)) {
2044 txq->pullup_early++;
2045 m = msgpullup(*fp, -1);
2046 if (m == NULL) {
2047 txq->pullup_failed++;
2048 return (E2BIG); /* (*fp) left as it was */
2049 }
2050 freemsg(*fp);
2051 *fp = m;
2052 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags);
2053 }
2054
2055 if (txinfo->len <= IMM_LEN && !sgl_only)
2056 return (0); /* nsegs = 0 tells caller to use imm. tx */
2057
2058 if (txinfo->len <= txq->copy_threshold &&
2059 copy_into_txb(txq, m, txinfo->len, txinfo) == 0)
2060 goto done;
2061
2062 for (; m; m = m->b_cont) {
2063
2064 len = MBLKL(m);
2065
2066 /* Use tx copy buffer if this mblk is small enough */
2067 if (len <= txq->copy_threshold &&
2068 copy_into_txb(txq, m, len, txinfo) == 0)
2069 continue;
2070
2071 /* Add DMA bindings for this mblk to the SGL */
2072 rc = add_mblk(txq, txinfo, m, len);
2073
2074 if (rc == E2BIG ||
2075 (txinfo->nsegs == TX_SGL_SEGS && m->b_cont)) {
2076
2077 txq->pullup_late++;
2078 m = msgpullup(*fp, -1);
2079 if (m != NULL) {
2080 free_txinfo_resources(txq, txinfo);
2081 freemsg(*fp);
2082 *fp = m;
2083 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags);
2084 goto start;
2085 }
2086
2087 txq->pullup_failed++;
2088 rc = E2BIG;
2089 }
2090
2091 if (rc != 0) {
2092 free_txinfo_resources(txq, txinfo);
2093 return (rc);
2094 }
2095 }
2096
2097 ASSERT(txinfo->nsegs > 0 && txinfo->nsegs <= TX_SGL_SEGS);
2098
2099 done:
2100
2101 /*
2102 * Store the # of flits required to hold this frame's SGL in nflits. An
2103 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by
2104 * multiple (len0 + len1, addr0, addr1) tuples. If addr1 is not used
2105 * then len1 must be set to 0.
2106 */
2107 n = txinfo->nsegs - 1;
2108 txinfo->nflits = (3 * n) / 2 + (n & 1) + 2;
2109 if (n & 1)
2110 txinfo->sgl.sge[n / 2].len[1] = cpu_to_be32(0);
2111
2112 txinfo->sgl.cmd_nsge = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_DSGL) |
2113 V_ULPTX_NSGE(txinfo->nsegs));
2114
2115 return (0);
2116 }
2117
2118 static inline int
fits_in_txb(struct sge_txq * txq,int len,int * waste)2119 fits_in_txb(struct sge_txq *txq, int len, int *waste)
2120 {
2121 if (txq->txb_avail < len)
2122 return (0);
2123
2124 if (txq->txb_next + len <= txq->txb_size) {
2125 *waste = 0;
2126 return (1);
2127 }
2128
2129 *waste = txq->txb_size - txq->txb_next;
2130
2131 return (txq->txb_avail - *waste < len ? 0 : 1);
2132 }
2133
2134 #define TXB_CHUNK 64
2135
2136 /*
2137 * Copies the specified # of bytes into txq's tx copy buffer and updates txinfo
2138 * and txq to indicate resources used. Caller has to make sure that those many
2139 * bytes are available in the mblk chain (b_cont linked).
2140 */
2141 static inline int
copy_into_txb(struct sge_txq * txq,mblk_t * m,int len,struct txinfo * txinfo)2142 copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, struct txinfo *txinfo)
2143 {
2144 int waste, n;
2145
2146 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb */
2147
2148 if (!fits_in_txb(txq, len, &waste)) {
2149 txq->txb_full++;
2150 return (ENOMEM);
2151 }
2152
2153 if (waste != 0) {
2154 ASSERT((waste & (TXB_CHUNK - 1)) == 0);
2155 txinfo->txb_used += waste;
2156 txq->txb_avail -= waste;
2157 txq->txb_next = 0;
2158 }
2159
2160 for (n = 0; n < len; m = m->b_cont) {
2161 bcopy(m->b_rptr, txq->txb_va + txq->txb_next + n, MBLKL(m));
2162 n += MBLKL(m);
2163 }
2164
2165 add_seg(txinfo, txq->txb_ba + txq->txb_next, len);
2166
2167 n = roundup(len, TXB_CHUNK);
2168 txinfo->txb_used += n;
2169 txq->txb_avail -= n;
2170 txq->txb_next += n;
2171 ASSERT(txq->txb_next <= txq->txb_size);
2172 if (txq->txb_next == txq->txb_size)
2173 txq->txb_next = 0;
2174
2175 return (0);
2176 }
2177
2178 static inline void
add_seg(struct txinfo * txinfo,uint64_t ba,uint32_t len)2179 add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len)
2180 {
2181 ASSERT(txinfo->nsegs < TX_SGL_SEGS); /* must have room */
2182
2183 if (txinfo->nsegs != 0) {
2184 int idx = txinfo->nsegs - 1;
2185 txinfo->sgl.sge[idx / 2].len[idx & 1] = cpu_to_be32(len);
2186 txinfo->sgl.sge[idx / 2].addr[idx & 1] = cpu_to_be64(ba);
2187 } else {
2188 txinfo->sgl.len0 = cpu_to_be32(len);
2189 txinfo->sgl.addr0 = cpu_to_be64(ba);
2190 }
2191 txinfo->nsegs++;
2192 }
2193
2194 /*
2195 * This function cleans up any partially allocated resources when it fails so
2196 * there's nothing for the caller to clean up in that case.
2197 *
2198 * EIO indicates permanent failure. Caller should drop the frame containing
2199 * this mblk and continue.
2200 *
2201 * E2BIG indicates that the SGL length for this mblk exceeds the hardware
2202 * limit. Caller should pull up the frame before trying to send it out.
2203 * (This error means our pullup_early heuristic did not work for this frame)
2204 *
2205 * ENOMEM indicates a temporary shortage of resources (DMA handles, other DMA
2206 * resources, etc.). Caller should suspend the tx queue and wait for reclaim to
2207 * free up resources.
2208 */
2209 static inline int
add_mblk(struct sge_txq * txq,struct txinfo * txinfo,mblk_t * m,int len)2210 add_mblk(struct sge_txq *txq, struct txinfo *txinfo, mblk_t *m, int len)
2211 {
2212 ddi_dma_handle_t dhdl;
2213 ddi_dma_cookie_t cookie;
2214 uint_t ccount = 0;
2215 int rc;
2216
2217 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate dhdls */
2218
2219 if (txq->tx_dhdl_avail == 0) {
2220 txq->dma_hdl_failed++;
2221 return (ENOMEM);
2222 }
2223
2224 dhdl = txq->tx_dhdl[txq->tx_dhdl_pidx];
2225 rc = ddi_dma_addr_bind_handle(dhdl, NULL, (caddr_t)m->b_rptr, len,
2226 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie,
2227 &ccount);
2228 if (rc != DDI_DMA_MAPPED) {
2229 txq->dma_map_failed++;
2230
2231 ASSERT(rc != DDI_DMA_INUSE && rc != DDI_DMA_PARTIAL_MAP);
2232
2233 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EIO);
2234 }
2235
2236 if (ccount + txinfo->nsegs > TX_SGL_SEGS) {
2237 (void) ddi_dma_unbind_handle(dhdl);
2238 return (E2BIG);
2239 }
2240
2241 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size);
2242 while (--ccount) {
2243 ddi_dma_nextcookie(dhdl, &cookie);
2244 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size);
2245 }
2246
2247 if (++txq->tx_dhdl_pidx == txq->tx_dhdl_total)
2248 txq->tx_dhdl_pidx = 0;
2249 txq->tx_dhdl_avail--;
2250 txinfo->hdls_used++;
2251
2252 return (0);
2253 }
2254
2255 /*
2256 * Releases all the txq resources used up in the specified txinfo.
2257 */
2258 static void
free_txinfo_resources(struct sge_txq * txq,struct txinfo * txinfo)2259 free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo)
2260 {
2261 int n;
2262
2263 TXQ_LOCK_ASSERT_OWNED(txq); /* dhdls, txb */
2264
2265 n = txinfo->txb_used;
2266 if (n > 0) {
2267 txq->txb_avail += n;
2268 if (n <= txq->txb_next)
2269 txq->txb_next -= n;
2270 else {
2271 n -= txq->txb_next;
2272 txq->txb_next = txq->txb_size - n;
2273 }
2274 }
2275
2276 for (n = txinfo->hdls_used; n > 0; n--) {
2277 if (txq->tx_dhdl_pidx > 0)
2278 txq->tx_dhdl_pidx--;
2279 else
2280 txq->tx_dhdl_pidx = txq->tx_dhdl_total - 1;
2281 txq->tx_dhdl_avail++;
2282 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_pidx]);
2283 }
2284 }
2285
2286 /*
2287 * Returns 0 to indicate that m has been accepted into a coalesced tx work
2288 * request. It has either been folded into txpkts or txpkts was flushed and m
2289 * has started a new coalesced work request (as the first frame in a fresh
2290 * txpkts).
2291 *
2292 * Returns non-zero to indicate a failure - caller is responsible for
2293 * transmitting m, if there was anything in txpkts it has been flushed.
2294 */
2295 static int
add_to_txpkts(struct sge_txq * txq,struct txpkts * txpkts,mblk_t * m,struct txinfo * txinfo)2296 add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m,
2297 struct txinfo *txinfo)
2298 {
2299 struct sge_eq *eq = &txq->eq;
2300 int can_coalesce;
2301 struct tx_sdesc *txsd;
2302 uint8_t flits;
2303
2304 TXQ_LOCK_ASSERT_OWNED(txq);
2305
2306 if (txpkts->npkt > 0) {
2307 flits = TXPKTS_PKT_HDR + txinfo->nflits;
2308 can_coalesce = (txinfo->flags & HW_LSO) == 0 &&
2309 txpkts->nflits + flits <= TX_WR_FLITS &&
2310 txpkts->nflits + flits <= eq->avail * 8 &&
2311 txpkts->plen + txinfo->len < 65536;
2312
2313 if (can_coalesce != 0) {
2314 txpkts->tail->b_next = m;
2315 txpkts->tail = m;
2316 txpkts->npkt++;
2317 txpkts->nflits += flits;
2318 txpkts->plen += txinfo->len;
2319
2320 txsd = &txq->sdesc[eq->pidx];
2321 txsd->txb_used += txinfo->txb_used;
2322 txsd->hdls_used += txinfo->hdls_used;
2323
2324 return (0);
2325 }
2326
2327 /*
2328 * Couldn't coalesce m into txpkts. The first order of business
2329 * is to send txpkts on its way. Then we'll revisit m.
2330 */
2331 write_txpkts_wr(txq, txpkts);
2332 }
2333
2334 /*
2335 * Check if we can start a new coalesced tx work request with m as
2336 * the first packet in it.
2337 */
2338
2339 ASSERT(txpkts->npkt == 0);
2340 ASSERT(txinfo->len < 65536);
2341
2342 flits = TXPKTS_WR_HDR + txinfo->nflits;
2343 can_coalesce = (txinfo->flags & HW_LSO) == 0 &&
2344 flits <= eq->avail * 8 && flits <= TX_WR_FLITS;
2345
2346 if (can_coalesce == 0)
2347 return (EINVAL);
2348
2349 /*
2350 * Start a fresh coalesced tx WR with m as the first frame in it.
2351 */
2352 txpkts->tail = m;
2353 txpkts->npkt = 1;
2354 txpkts->nflits = flits;
2355 txpkts->flitp = &eq->desc[eq->pidx].flit[2];
2356 txpkts->plen = txinfo->len;
2357
2358 txsd = &txq->sdesc[eq->pidx];
2359 txsd->m = m;
2360 txsd->txb_used = txinfo->txb_used;
2361 txsd->hdls_used = txinfo->hdls_used;
2362
2363 return (0);
2364 }
2365
2366 /*
2367 * Note that write_txpkts_wr can never run out of hardware descriptors (but
2368 * write_txpkt_wr can). add_to_txpkts ensures that a frame is accepted for
2369 * coalescing only if sufficient hardware descriptors are available.
2370 */
2371 static void
write_txpkts_wr(struct sge_txq * txq,struct txpkts * txpkts)2372 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts)
2373 {
2374 struct sge_eq *eq = &txq->eq;
2375 struct fw_eth_tx_pkts_wr *wr;
2376 struct tx_sdesc *txsd;
2377 uint32_t ctrl;
2378 uint16_t ndesc;
2379
2380 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */
2381
2382 ndesc = howmany(txpkts->nflits, 8);
2383
2384 wr = (void *)&eq->desc[eq->pidx];
2385 wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR) |
2386 V_FW_WR_IMMDLEN(0)); /* immdlen does not matter in this WR */
2387 ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2));
2388 if (eq->avail == ndesc)
2389 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
2390 wr->equiq_to_len16 = cpu_to_be32(ctrl);
2391 wr->plen = cpu_to_be16(txpkts->plen);
2392 wr->npkt = txpkts->npkt;
2393 wr->r3 = wr->type = 0;
2394
2395 /* Everything else already written */
2396
2397 txsd = &txq->sdesc[eq->pidx];
2398 txsd->desc_used = ndesc;
2399
2400 txq->txb_used += txsd->txb_used / TXB_CHUNK;
2401 txq->hdl_used += txsd->hdls_used;
2402
2403 ASSERT(eq->avail >= ndesc);
2404
2405 eq->pending += ndesc;
2406 eq->avail -= ndesc;
2407 eq->pidx += ndesc;
2408 if (eq->pidx >= eq->cap)
2409 eq->pidx -= eq->cap;
2410
2411 txq->txpkts_pkts += txpkts->npkt;
2412 txq->txpkts_wrs++;
2413 txpkts->npkt = 0; /* emptied */
2414 }
2415
2416 typedef enum {
2417 COS_SUCCESS, /* ctrl flit contains proper bits for csum offload */
2418 COS_IGNORE, /* no csum offload requested */
2419 COS_FAIL, /* csum offload requested, but pkt data missing */
2420 } csum_offload_status_t;
2421 /*
2422 * Build a ctrl1 flit for checksum offload in CPL_TX_PKT_XT command
2423 */
2424 static csum_offload_status_t
csum_to_ctrl(const struct txinfo * txinfo,uint32_t chip_version,uint64_t * ctrlp)2425 csum_to_ctrl(const struct txinfo *txinfo, uint32_t chip_version,
2426 uint64_t *ctrlp)
2427 {
2428 const mac_ether_offload_info_t *meoi = &txinfo->meoi;
2429 const uint32_t tx_flags = txinfo->flags;
2430 const boolean_t needs_l3_csum = (tx_flags & HW_LSO) != 0 ||
2431 (tx_flags & HCK_IPV4_HDRCKSUM) != 0;
2432 const boolean_t needs_l4_csum = (tx_flags & HW_LSO) != 0 ||
2433 (tx_flags & (HCK_FULLCKSUM | HCK_PARTIALCKSUM)) != 0;
2434
2435 /*
2436 * Default to disabling any checksumming both for cases where it is not
2437 * requested, but also if we cannot appropriately interrogate the
2438 * required information from the packet.
2439 */
2440 uint64_t ctrl = F_TXPKT_L4CSUM_DIS | F_TXPKT_IPCSUM_DIS;
2441 if (!needs_l3_csum && !needs_l4_csum) {
2442 *ctrlp = ctrl;
2443 return (COS_IGNORE);
2444 }
2445
2446 if (needs_l3_csum) {
2447 /* Only IPv4 checksums are supported (for L3) */
2448 if ((meoi->meoi_flags & MEOI_L3INFO_SET) == 0 ||
2449 meoi->meoi_l3proto != ETHERTYPE_IP) {
2450 *ctrlp = ctrl;
2451 return (COS_FAIL);
2452 }
2453 ctrl &= ~F_TXPKT_IPCSUM_DIS;
2454 }
2455
2456 if (needs_l4_csum) {
2457 /*
2458 * We need at least all of the L3 header to make decisions about
2459 * the contained L4 protocol. If not all of the L4 information
2460 * is present, we will leave it to the NIC to checksum all it is
2461 * able to.
2462 */
2463 if ((meoi->meoi_flags & MEOI_L3INFO_SET) == 0) {
2464 *ctrlp = ctrl;
2465 return (COS_FAIL);
2466 }
2467
2468 /*
2469 * Since we are parsing the packet anyways, make the checksum
2470 * decision based on the L4 protocol, rather than using the
2471 * Generic TCP/UDP checksum using start & end offsets in the
2472 * packet (like requested with PARTIALCKSUM).
2473 */
2474 int csum_type = -1;
2475 if (meoi->meoi_l3proto == ETHERTYPE_IP &&
2476 meoi->meoi_l4proto == IPPROTO_TCP) {
2477 csum_type = TX_CSUM_TCPIP;
2478 } else if (meoi->meoi_l3proto == ETHERTYPE_IPV6 &&
2479 meoi->meoi_l4proto == IPPROTO_TCP) {
2480 csum_type = TX_CSUM_TCPIP6;
2481 } else if (meoi->meoi_l3proto == ETHERTYPE_IP &&
2482 meoi->meoi_l4proto == IPPROTO_UDP) {
2483 csum_type = TX_CSUM_UDPIP;
2484 } else if (meoi->meoi_l3proto == ETHERTYPE_IPV6 &&
2485 meoi->meoi_l4proto == IPPROTO_UDP) {
2486 csum_type = TX_CSUM_UDPIP6;
2487 } else {
2488 *ctrlp = ctrl;
2489 return (COS_FAIL);
2490 }
2491
2492 ASSERT(csum_type != -1);
2493 ctrl &= ~F_TXPKT_L4CSUM_DIS;
2494 ctrl |= V_TXPKT_CSUM_TYPE(csum_type);
2495 }
2496
2497 if ((ctrl & F_TXPKT_IPCSUM_DIS) == 0 &&
2498 (ctrl & F_TXPKT_L4CSUM_DIS) != 0) {
2499 /*
2500 * If only the IPv4 checksum is requested, we need to set an
2501 * appropriate type in the command for it.
2502 */
2503 ctrl |= V_TXPKT_CSUM_TYPE(TX_CSUM_IP);
2504 }
2505
2506 ASSERT(ctrl != (F_TXPKT_L4CSUM_DIS | F_TXPKT_IPCSUM_DIS));
2507
2508 /*
2509 * Fill in the requisite L2/L3 header length data.
2510 *
2511 * The Ethernet header length is recorded as 'size - 14 bytes'
2512 */
2513 const uint8_t eth_len = meoi->meoi_l2hlen - 14;
2514 if (chip_version >= CHELSIO_T6) {
2515 ctrl |= V_T6_TXPKT_ETHHDR_LEN(eth_len);
2516 } else {
2517 ctrl |= V_TXPKT_ETHHDR_LEN(eth_len);
2518 }
2519 ctrl |= V_TXPKT_IPHDR_LEN(meoi->meoi_l3hlen);
2520
2521 *ctrlp = ctrl;
2522 return (COS_SUCCESS);
2523 }
2524
2525 static int
write_txpkt_wr(struct port_info * pi,struct sge_txq * txq,mblk_t * m,struct txinfo * txinfo)2526 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m,
2527 struct txinfo *txinfo)
2528 {
2529 struct sge_eq *eq = &txq->eq;
2530 struct fw_eth_tx_pkt_wr *wr;
2531 struct cpl_tx_pkt_core *cpl;
2532 uint32_t ctrl; /* used in many unrelated places */
2533 uint64_t ctrl1;
2534 int nflits, ndesc;
2535 struct tx_sdesc *txsd;
2536 caddr_t dst;
2537 const mac_ether_offload_info_t *meoi = &txinfo->meoi;
2538
2539 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */
2540
2541 /*
2542 * Do we have enough flits to send this frame out?
2543 */
2544 ctrl = sizeof (struct cpl_tx_pkt_core);
2545 if (txinfo->flags & HW_LSO) {
2546 nflits = TXPKT_LSO_WR_HDR;
2547 ctrl += sizeof (struct cpl_tx_pkt_lso_core);
2548 } else {
2549 nflits = TXPKT_WR_HDR;
2550 }
2551 if (txinfo->nsegs > 0)
2552 nflits += txinfo->nflits;
2553 else {
2554 nflits += howmany(txinfo->len, 8);
2555 ctrl += txinfo->len;
2556 }
2557 ndesc = howmany(nflits, 8);
2558 if (ndesc > eq->avail)
2559 return (ENOMEM);
2560
2561 /* Firmware work request header */
2562 wr = (void *)&eq->desc[eq->pidx];
2563 wr->op_immdlen = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
2564 V_FW_WR_IMMDLEN(ctrl));
2565 ctrl = V_FW_WR_LEN16(howmany(nflits, 2));
2566 if (eq->avail == ndesc)
2567 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
2568 wr->equiq_to_len16 = cpu_to_be32(ctrl);
2569 wr->r3 = 0;
2570
2571 if (txinfo->flags & HW_LSO &&
2572 (meoi->meoi_flags & MEOI_L4INFO_SET) != 0 &&
2573 meoi->meoi_l4proto == IPPROTO_TCP) {
2574 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
2575
2576 ctrl = V_LSO_OPCODE((u32)CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE |
2577 F_LSO_LAST_SLICE;
2578
2579 if (meoi->meoi_l2hlen > sizeof (struct ether_header)) {
2580 /*
2581 * This presently assumes a standard VLAN header,
2582 * without support for Q-in-Q.
2583 */
2584 ctrl |= V_LSO_ETHHDR_LEN(1);
2585 }
2586
2587 switch (meoi->meoi_l3proto) {
2588 case ETHERTYPE_IPV6:
2589 ctrl |= F_LSO_IPV6;
2590 /* FALLTHROUGH */
2591 case ETHERTYPE_IP:
2592 ctrl |= V_LSO_IPHDR_LEN(meoi->meoi_l3hlen / 4);
2593 break;
2594 default:
2595 break;
2596 }
2597
2598 ctrl |= V_LSO_TCPHDR_LEN(meoi->meoi_l4hlen / 4);
2599
2600 lso->lso_ctrl = cpu_to_be32(ctrl);
2601 lso->ipid_ofst = cpu_to_be16(0);
2602 lso->mss = cpu_to_be16(txinfo->mss);
2603 lso->seqno_offset = cpu_to_be32(0);
2604 if (is_t4(pi->adapter->params.chip))
2605 lso->len = cpu_to_be32(txinfo->len);
2606 else
2607 lso->len = cpu_to_be32(V_LSO_T5_XFER_SIZE(txinfo->len));
2608
2609 cpl = (void *)(lso + 1);
2610
2611 txq->tso_wrs++;
2612 } else {
2613 cpl = (void *)(wr + 1);
2614 }
2615
2616 /* Checksum offload */
2617 switch (csum_to_ctrl(txinfo,
2618 CHELSIO_CHIP_VERSION(pi->adapter->params.chip), &ctrl1)) {
2619 case COS_SUCCESS:
2620 txq->txcsum++;
2621 break;
2622 case COS_FAIL:
2623 /*
2624 * Packet will be going out with checksums which are probably
2625 * wrong but there is little we can do now.
2626 */
2627 txq->csum_failed++;
2628 break;
2629 default:
2630 break;
2631 }
2632
2633 /* CPL header */
2634 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) |
2635 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
2636 cpl->pack = 0;
2637 cpl->len = cpu_to_be16(txinfo->len);
2638 cpl->ctrl1 = cpu_to_be64(ctrl1);
2639
2640 /* Software descriptor */
2641 txsd = &txq->sdesc[eq->pidx];
2642 txsd->m = m;
2643 txsd->txb_used = txinfo->txb_used;
2644 txsd->hdls_used = txinfo->hdls_used;
2645 /* LINTED: E_ASSIGN_NARROW_CONV */
2646 txsd->desc_used = ndesc;
2647
2648 txq->txb_used += txinfo->txb_used / TXB_CHUNK;
2649 txq->hdl_used += txinfo->hdls_used;
2650
2651 eq->pending += ndesc;
2652 eq->avail -= ndesc;
2653 eq->pidx += ndesc;
2654 if (eq->pidx >= eq->cap)
2655 eq->pidx -= eq->cap;
2656
2657 /* SGL */
2658 dst = (void *)(cpl + 1);
2659 if (txinfo->nsegs > 0) {
2660 txq->sgl_wrs++;
2661 copy_to_txd(eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8);
2662
2663 /* Need to zero-pad to a 16 byte boundary if not on one */
2664 if ((uintptr_t)dst & 0xf)
2665 /* LINTED: E_BAD_PTR_CAST_ALIGN */
2666 *(uint64_t *)dst = 0;
2667
2668 } else {
2669 txq->imm_wrs++;
2670 #ifdef DEBUG
2671 ctrl = txinfo->len;
2672 #endif
2673 for (; m; m = m->b_cont) {
2674 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m));
2675 #ifdef DEBUG
2676 ctrl -= MBLKL(m);
2677 #endif
2678 }
2679 ASSERT(ctrl == 0);
2680 }
2681
2682 txq->txpkt_wrs++;
2683 return (0);
2684 }
2685
2686 static inline void
write_ulp_cpl_sgl(struct port_info * pi,struct sge_txq * txq,struct txpkts * txpkts,struct txinfo * txinfo)2687 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
2688 struct txpkts *txpkts, struct txinfo *txinfo)
2689 {
2690 struct ulp_txpkt *ulpmc;
2691 struct ulptx_idata *ulpsc;
2692 struct cpl_tx_pkt_core *cpl;
2693 uintptr_t flitp, start, end;
2694 uint64_t ctrl;
2695 caddr_t dst;
2696
2697 ASSERT(txpkts->npkt > 0);
2698
2699 start = (uintptr_t)txq->eq.desc;
2700 end = (uintptr_t)txq->eq.spg;
2701
2702 /* Checksum offload */
2703 switch (csum_to_ctrl(txinfo,
2704 CHELSIO_CHIP_VERSION(pi->adapter->params.chip), &ctrl)) {
2705 case COS_SUCCESS:
2706 txq->txcsum++;
2707 break;
2708 case COS_FAIL:
2709 /*
2710 * Packet will be going out with checksums which are probably
2711 * wrong but there is little we can do now.
2712 */
2713 txq->csum_failed++;
2714 break;
2715 default:
2716 break;
2717 }
2718
2719 /*
2720 * The previous packet's SGL must have ended at a 16 byte boundary (this
2721 * is required by the firmware/hardware). It follows that flitp cannot
2722 * wrap around between the ULPTX master command and ULPTX subcommand (8
2723 * bytes each), and that it can not wrap around in the middle of the
2724 * cpl_tx_pkt_core either.
2725 */
2726 flitp = (uintptr_t)txpkts->flitp;
2727 ASSERT((flitp & 0xf) == 0);
2728
2729 /* ULP master command */
2730 ulpmc = (void *)flitp;
2731 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
2732 ulpmc->len = htonl(howmany(sizeof (*ulpmc) + sizeof (*ulpsc) +
2733 sizeof (*cpl) + 8 * txinfo->nflits, 16));
2734
2735 /* ULP subcommand */
2736 ulpsc = (void *)(ulpmc + 1);
2737 ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) |
2738 F_ULP_TX_SC_MORE);
2739 ulpsc->len = cpu_to_be32(sizeof (struct cpl_tx_pkt_core));
2740
2741 flitp += sizeof (*ulpmc) + sizeof (*ulpsc);
2742 if (flitp == end)
2743 flitp = start;
2744
2745 /* CPL_TX_PKT_XT */
2746 cpl = (void *)flitp;
2747 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) |
2748 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
2749 cpl->pack = 0;
2750 cpl->len = cpu_to_be16(txinfo->len);
2751 cpl->ctrl1 = cpu_to_be64(ctrl);
2752
2753 flitp += sizeof (*cpl);
2754 if (flitp == end)
2755 flitp = start;
2756
2757 /* SGL for this frame */
2758 dst = (caddr_t)flitp;
2759 copy_to_txd(&txq->eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8);
2760 flitp = (uintptr_t)dst;
2761
2762 /* Zero pad and advance to a 16 byte boundary if not already at one. */
2763 if (flitp & 0xf) {
2764
2765 /* no matter what, flitp should be on an 8 byte boundary */
2766 ASSERT((flitp & 0x7) == 0);
2767
2768 *(uint64_t *)flitp = 0;
2769 flitp += sizeof (uint64_t);
2770 txpkts->nflits++;
2771 }
2772
2773 if (flitp == end)
2774 flitp = start;
2775
2776 txpkts->flitp = (void *)flitp;
2777 }
2778
2779 static inline void
copy_to_txd(struct sge_eq * eq,caddr_t from,caddr_t * to,int len)2780 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
2781 {
2782 if ((uintptr_t)(*to) + len <= (uintptr_t)eq->spg) {
2783 bcopy(from, *to, len);
2784 (*to) += len;
2785 } else {
2786 int portion = (uintptr_t)eq->spg - (uintptr_t)(*to);
2787
2788 bcopy(from, *to, portion);
2789 from += portion;
2790 portion = len - portion; /* remaining */
2791 bcopy(from, (void *)eq->desc, portion);
2792 (*to) = (caddr_t)eq->desc + portion;
2793 }
2794 }
2795
2796 static inline void
ring_tx_db(struct adapter * sc,struct sge_eq * eq)2797 ring_tx_db(struct adapter *sc, struct sge_eq *eq)
2798 {
2799 int val, db_mode;
2800 t4_doorbells_t db = eq->doorbells;
2801
2802 if (eq->pending > 1)
2803 db &= ~DOORBELL_WCWR;
2804
2805 if (eq->pending > eq->pidx) {
2806 int offset = eq->cap - (eq->pending - eq->pidx);
2807
2808 /* pidx has wrapped around since last doorbell */
2809
2810 (void) ddi_dma_sync(eq->desc_dhdl,
2811 offset * sizeof (struct tx_desc), 0,
2812 DDI_DMA_SYNC_FORDEV);
2813 (void) ddi_dma_sync(eq->desc_dhdl,
2814 0, eq->pidx * sizeof (struct tx_desc),
2815 DDI_DMA_SYNC_FORDEV);
2816 } else if (eq->pending > 0) {
2817 (void) ddi_dma_sync(eq->desc_dhdl,
2818 (eq->pidx - eq->pending) * sizeof (struct tx_desc),
2819 eq->pending * sizeof (struct tx_desc),
2820 DDI_DMA_SYNC_FORDEV);
2821 }
2822
2823 membar_producer();
2824
2825 if (is_t4(sc->params.chip))
2826 val = V_PIDX(eq->pending);
2827 else
2828 val = V_PIDX_T5(eq->pending);
2829
2830 db_mode = (1 << (ffs(db) - 1));
2831 switch (db_mode) {
2832 case DOORBELL_UDB:
2833 *eq->udb = LE_32(V_QID(eq->udb_qid) | val);
2834 break;
2835
2836 case DOORBELL_WCWR:
2837 {
2838 volatile uint64_t *dst, *src;
2839 int i;
2840 /*
2841 * Queues whose 128B doorbell segment fits in
2842 * the page do not use relative qid
2843 * (udb_qid is always 0). Only queues with
2844 * doorbell segments can do WCWR.
2845 */
2846 ASSERT(eq->udb_qid == 0 && eq->pending == 1);
2847
2848 dst = (volatile void *)((uintptr_t)eq->udb +
2849 UDBS_WR_OFFSET - UDBS_DB_OFFSET);
2850 i = eq->pidx ? eq->pidx - 1 : eq->cap - 1;
2851 src = (void *)&eq->desc[i];
2852 while (src != (void *)&eq->desc[i + 1]) {
2853 *dst++ = *src++;
2854 }
2855 membar_producer();
2856 break;
2857 }
2858
2859 case DOORBELL_UDBWC:
2860 *eq->udb = LE_32(V_QID(eq->udb_qid) | val);
2861 membar_producer();
2862 break;
2863
2864 case DOORBELL_KDB:
2865 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
2866 V_QID(eq->cntxt_id) | val);
2867 break;
2868 }
2869
2870 eq->pending = 0;
2871 }
2872
2873 static int
reclaim_tx_descs(struct sge_txq * txq,int howmany)2874 reclaim_tx_descs(struct sge_txq *txq, int howmany)
2875 {
2876 struct tx_sdesc *txsd;
2877 uint_t cidx, can_reclaim, reclaimed, txb_freed, hdls_freed;
2878 struct sge_eq *eq = &txq->eq;
2879
2880 EQ_LOCK_ASSERT_OWNED(eq);
2881
2882 cidx = eq->spg->cidx; /* stable snapshot */
2883 cidx = be16_to_cpu(cidx);
2884
2885 if (cidx >= eq->cidx)
2886 can_reclaim = cidx - eq->cidx;
2887 else
2888 can_reclaim = cidx + eq->cap - eq->cidx;
2889
2890 if (can_reclaim == 0)
2891 return (0);
2892
2893 txb_freed = hdls_freed = reclaimed = 0;
2894 do {
2895 int ndesc;
2896
2897 txsd = &txq->sdesc[eq->cidx];
2898 ndesc = txsd->desc_used;
2899
2900 /* Firmware doesn't return "partial" credits. */
2901 ASSERT(can_reclaim >= ndesc);
2902
2903 /*
2904 * We always keep mblk around, even for immediate data. If mblk
2905 * is NULL, this has to be the software descriptor for a credit
2906 * flush work request.
2907 */
2908 if (txsd->m != NULL)
2909 freemsgchain(txsd->m);
2910 #ifdef DEBUG
2911 else {
2912 ASSERT(txsd->txb_used == 0);
2913 ASSERT(txsd->hdls_used == 0);
2914 ASSERT(ndesc == 1);
2915 }
2916 #endif
2917
2918 txb_freed += txsd->txb_used;
2919 hdls_freed += txsd->hdls_used;
2920 reclaimed += ndesc;
2921
2922 eq->cidx += ndesc;
2923 if (eq->cidx >= eq->cap)
2924 eq->cidx -= eq->cap;
2925
2926 can_reclaim -= ndesc;
2927
2928 } while (can_reclaim && reclaimed < howmany);
2929
2930 eq->avail += reclaimed;
2931 ASSERT(eq->avail < eq->cap); /* avail tops out at (cap - 1) */
2932
2933 txq->txb_avail += txb_freed;
2934
2935 txq->tx_dhdl_avail += hdls_freed;
2936 ASSERT(txq->tx_dhdl_avail <= txq->tx_dhdl_total);
2937 for (; hdls_freed; hdls_freed--) {
2938 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_cidx]);
2939 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total)
2940 txq->tx_dhdl_cidx = 0;
2941 }
2942
2943 return (reclaimed);
2944 }
2945
2946 static void
write_txqflush_wr(struct sge_txq * txq)2947 write_txqflush_wr(struct sge_txq *txq)
2948 {
2949 struct sge_eq *eq = &txq->eq;
2950 struct fw_eq_flush_wr *wr;
2951 struct tx_sdesc *txsd;
2952
2953 EQ_LOCK_ASSERT_OWNED(eq);
2954 ASSERT(eq->avail > 0);
2955
2956 wr = (void *)&eq->desc[eq->pidx];
2957 bzero(wr, sizeof (*wr));
2958 wr->opcode = FW_EQ_FLUSH_WR;
2959 wr->equiq_to_len16 = cpu_to_be32(V_FW_WR_LEN16(sizeof (*wr) / 16) |
2960 F_FW_WR_EQUEQ | F_FW_WR_EQUIQ);
2961
2962 txsd = &txq->sdesc[eq->pidx];
2963 txsd->m = NULL;
2964 txsd->txb_used = 0;
2965 txsd->hdls_used = 0;
2966 txsd->desc_used = 1;
2967
2968 eq->pending++;
2969 eq->avail--;
2970 if (++eq->pidx == eq->cap)
2971 eq->pidx = 0;
2972 }
2973
2974 static int
t4_handle_cpl_msg(struct sge_iq * iq,const struct rss_header * rss,mblk_t * mp)2975 t4_handle_cpl_msg(struct sge_iq *iq, const struct rss_header *rss, mblk_t *mp)
2976 {
2977 const uint8_t opcode = rss->opcode;
2978
2979 DTRACE_PROBE4(t4__cpl_msg, struct sge_iq *, iq, uint8_t, opcode,
2980 const struct rss_header *, rss, mblk_t *, mp);
2981
2982 switch (opcode) {
2983 case CPL_FW4_MSG:
2984 case CPL_FW6_MSG:
2985 ASSERT3P(mp, ==, NULL);
2986 return (t4_handle_fw_msg(iq, rss));
2987 case CPL_SGE_EGR_UPDATE:
2988 return (handle_sge_egr_update(iq, rss, mp));
2989 case CPL_RX_PKT:
2990 return (t4_eth_rx(iq, rss, mp));
2991 default:
2992 cxgb_printf(iq->adapter->dip, CE_WARN,
2993 "unhandled CPL opcode 0x%02x", opcode);
2994 if (mp != NULL) {
2995 freemsg(mp);
2996 }
2997 return (0);
2998 }
2999 }
3000
3001 static int
t4_handle_fw_msg(struct sge_iq * iq,const struct rss_header * rss)3002 t4_handle_fw_msg(struct sge_iq *iq, const struct rss_header *rss)
3003 {
3004 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
3005 const uint8_t msg_type = cpl->type;
3006 const struct rss_header *rss2;
3007 struct adapter *sc = iq->adapter;
3008
3009 DTRACE_PROBE3(t4__fw_msg, struct sge_iq *, iq, uint8_t, msg_type,
3010 const struct rss_header *, rss);
3011
3012 switch (msg_type) {
3013 case FW_TYPE_RSSCPL: /* also synonym for FW6_TYPE_RSSCPL */
3014 rss2 = (const struct rss_header *)&cpl->data[0];
3015 return (t4_handle_cpl_msg(iq, rss2, NULL));
3016 case FW6_TYPE_CMD_RPL:
3017 return (t4_handle_fw_rpl(sc, &cpl->data[0]));
3018 default:
3019 cxgb_printf(sc->dip, CE_WARN,
3020 "unhandled fw_msg type 0x%02x", msg_type);
3021 return (0);
3022 }
3023 }
3024
3025 static int
t4_eth_rx(struct sge_iq * iq,const struct rss_header * rss,mblk_t * m)3026 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
3027 {
3028 bool csum_ok;
3029 uint16_t err_vec;
3030 struct sge_rxq *rxq = (void *)iq;
3031 struct mblk_pair chain = {0};
3032 struct adapter *sc = iq->adapter;
3033 const struct cpl_rx_pkt *cpl = (const void *)(rss + 1);
3034
3035 iq->intr_next = iq->intr_params;
3036
3037 m->b_rptr += sc->sge.pktshift;
3038
3039 /* Compressed error vector is enabled for T6 only */
3040 if (sc->params.tp.rx_pkt_encap)
3041 /* It is enabled only in T6 config file */
3042 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec));
3043 else
3044 err_vec = ntohs(cpl->err_vec);
3045
3046 csum_ok = cpl->csum_calc && !err_vec;
3047 /* TODO: what about cpl->ip_frag? */
3048 if (csum_ok && !cpl->ip_frag) {
3049 mac_hcksum_set(m, 0, 0, 0, 0xffff,
3050 HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
3051 HCK_IPV4_HDRCKSUM_OK);
3052 rxq->rxcsum++;
3053 }
3054
3055 /* Add to the chain that we'll send up */
3056 if (chain.head != NULL)
3057 chain.tail->b_next = m;
3058 else
3059 chain.head = m;
3060 chain.tail = m;
3061
3062 t4_mac_rx(rxq->port, rxq, chain.head);
3063
3064 rxq->rxpkts++;
3065 rxq->rxbytes += be16_to_cpu(cpl->len);
3066 return (0);
3067 }
3068
3069 #define FL_HW_IDX(idx) ((idx) >> 3)
3070
3071 static inline void
ring_fl_db(struct adapter * sc,struct sge_fl * fl)3072 ring_fl_db(struct adapter *sc, struct sge_fl *fl)
3073 {
3074 int desc_start, desc_last, ndesc;
3075 uint32_t v = sc->params.arch.sge_fl_db;
3076
3077 ndesc = FL_HW_IDX(fl->pending);
3078
3079 /* Hold back one credit if pidx = cidx */
3080 if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx))
3081 ndesc--;
3082
3083 /*
3084 * There are chances of ndesc modified above (to avoid pidx = cidx).
3085 * If there is nothing to post, return.
3086 */
3087 if (ndesc <= 0)
3088 return;
3089
3090 desc_last = FL_HW_IDX(fl->pidx);
3091
3092 if (fl->pidx < fl->pending) {
3093 /* There was a wrap */
3094 desc_start = FL_HW_IDX(fl->pidx + fl->cap - fl->pending);
3095
3096 /* From desc_start to the end of list */
3097 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 0,
3098 DDI_DMA_SYNC_FORDEV);
3099
3100 /* From start of list to the desc_last */
3101 if (desc_last != 0)
3102 (void) ddi_dma_sync(fl->dhdl, 0, desc_last *
3103 RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3104 } else {
3105 /* There was no wrap, sync from start_desc to last_desc */
3106 desc_start = FL_HW_IDX(fl->pidx - fl->pending);
3107 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE,
3108 ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3109 }
3110
3111 if (is_t4(sc->params.chip))
3112 v |= V_PIDX(ndesc);
3113 else
3114 v |= V_PIDX_T5(ndesc);
3115 v |= V_QID(fl->cntxt_id) | V_PIDX(ndesc);
3116
3117 membar_producer();
3118
3119 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v);
3120
3121 /*
3122 * Update pending count:
3123 * Deduct the number of descriptors posted
3124 */
3125 fl->pending -= ndesc * 8;
3126 }
3127
3128 static void
tx_reclaim_task(void * arg)3129 tx_reclaim_task(void *arg)
3130 {
3131 struct sge_txq *txq = arg;
3132
3133 TXQ_LOCK(txq);
3134 reclaim_tx_descs(txq, txq->eq.qsize);
3135 TXQ_UNLOCK(txq);
3136 }
3137
3138 /* ARGSUSED */
3139 static int
handle_sge_egr_update(struct sge_iq * iq,const struct rss_header * rss,mblk_t * m)3140 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
3141 mblk_t *m)
3142 {
3143 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
3144 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
3145 struct adapter *sc = iq->adapter;
3146 struct sge *s = &sc->sge;
3147 struct sge_eq *eq;
3148 struct sge_txq *txq;
3149
3150 txq = (void *)s->eqmap[qid - s->eq_start];
3151 eq = &txq->eq;
3152 txq->qflush++;
3153 t4_mac_tx_update(txq->port, txq);
3154
3155 ddi_taskq_dispatch(sc->tq[eq->tx_chan], tx_reclaim_task,
3156 (void *)txq, DDI_NOSLEEP);
3157
3158 return (0);
3159 }
3160
3161 int
t4_alloc_tx_maps(struct adapter * sc,struct tx_maps * txmaps,int count,int flags)3162 t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count,
3163 int flags)
3164 {
3165 int i, rc;
3166
3167 txmaps->map_total = count;
3168 txmaps->map_avail = txmaps->map_cidx = txmaps->map_pidx = 0;
3169
3170 txmaps->map = kmem_zalloc(sizeof (ddi_dma_handle_t) *
3171 txmaps->map_total, flags);
3172
3173 for (i = 0; i < count; i++) {
3174 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx,
3175 DDI_DMA_SLEEP, 0, &txmaps->map[i]);
3176 if (rc != DDI_SUCCESS) {
3177 cxgb_printf(sc->dip, CE_WARN,
3178 "%s: failed to allocate DMA handle (%d)",
3179 __func__, rc);
3180 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
3181 }
3182 txmaps->map_avail++;
3183 }
3184
3185 return (0);
3186 }
3187
3188 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG)
3189 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR)
3190 #define KS_U_SET(x, y) kstatp->x.value.ul = (y)
3191 #define KS_U_FROM(x, y) kstatp->x.value.ul = (y)->x
3192 #define KS_C_SET(x, ...) \
3193 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__)
3194
3195 /*
3196 * cxgbe:X:config
3197 */
3198 struct cxgbe_port_config_kstats {
3199 kstat_named_t idx;
3200 kstat_named_t nrxq;
3201 kstat_named_t ntxq;
3202 kstat_named_t first_rxq;
3203 kstat_named_t first_txq;
3204 kstat_named_t controller;
3205 kstat_named_t factory_mac_address;
3206 };
3207
3208 /*
3209 * cxgbe:X:info
3210 */
3211 struct cxgbe_port_info_kstats {
3212 kstat_named_t transceiver;
3213 kstat_named_t rx_ovflow0;
3214 kstat_named_t rx_ovflow1;
3215 kstat_named_t rx_ovflow2;
3216 kstat_named_t rx_ovflow3;
3217 kstat_named_t rx_trunc0;
3218 kstat_named_t rx_trunc1;
3219 kstat_named_t rx_trunc2;
3220 kstat_named_t rx_trunc3;
3221 kstat_named_t tx_pause;
3222 kstat_named_t rx_pause;
3223 };
3224
3225 static kstat_t *
setup_port_config_kstats(struct port_info * pi)3226 setup_port_config_kstats(struct port_info *pi)
3227 {
3228 kstat_t *ksp;
3229 struct cxgbe_port_config_kstats *kstatp;
3230 int ndata;
3231 dev_info_t *pdip = ddi_get_parent(pi->dip);
3232 uint8_t *ma = &pi->hw_addr[0];
3233
3234 ndata = sizeof (struct cxgbe_port_config_kstats) /
3235 sizeof (kstat_named_t);
3236
3237 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "config",
3238 "net", KSTAT_TYPE_NAMED, ndata, 0);
3239 if (ksp == NULL) {
3240 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats.");
3241 return (NULL);
3242 }
3243
3244 kstatp = (struct cxgbe_port_config_kstats *)ksp->ks_data;
3245
3246 KS_UINIT(idx);
3247 KS_UINIT(nrxq);
3248 KS_UINIT(ntxq);
3249 KS_UINIT(first_rxq);
3250 KS_UINIT(first_txq);
3251 KS_CINIT(controller);
3252 KS_CINIT(factory_mac_address);
3253
3254 KS_U_SET(idx, pi->port_id);
3255 KS_U_SET(nrxq, pi->nrxq);
3256 KS_U_SET(ntxq, pi->ntxq);
3257 KS_U_SET(first_rxq, pi->first_rxq);
3258 KS_U_SET(first_txq, pi->first_txq);
3259 KS_C_SET(controller, "%s%d", ddi_driver_name(pdip),
3260 ddi_get_instance(pdip));
3261 KS_C_SET(factory_mac_address, "%02X%02X%02X%02X%02X%02X",
3262 ma[0], ma[1], ma[2], ma[3], ma[4], ma[5]);
3263
3264 /* Do NOT set ksp->ks_update. These kstats do not change. */
3265
3266 /* Install the kstat */
3267 ksp->ks_private = (void *)pi;
3268 kstat_install(ksp);
3269
3270 return (ksp);
3271 }
3272
3273 static kstat_t *
setup_port_info_kstats(struct port_info * pi)3274 setup_port_info_kstats(struct port_info *pi)
3275 {
3276 kstat_t *ksp;
3277 struct cxgbe_port_info_kstats *kstatp;
3278 int ndata;
3279
3280 ndata = sizeof (struct cxgbe_port_info_kstats) / sizeof (kstat_named_t);
3281
3282 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "info",
3283 "net", KSTAT_TYPE_NAMED, ndata, 0);
3284 if (ksp == NULL) {
3285 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats.");
3286 return (NULL);
3287 }
3288
3289 kstatp = (struct cxgbe_port_info_kstats *)ksp->ks_data;
3290
3291 KS_CINIT(transceiver);
3292 KS_UINIT(rx_ovflow0);
3293 KS_UINIT(rx_ovflow1);
3294 KS_UINIT(rx_ovflow2);
3295 KS_UINIT(rx_ovflow3);
3296 KS_UINIT(rx_trunc0);
3297 KS_UINIT(rx_trunc1);
3298 KS_UINIT(rx_trunc2);
3299 KS_UINIT(rx_trunc3);
3300 KS_UINIT(tx_pause);
3301 KS_UINIT(rx_pause);
3302
3303 /* Install the kstat */
3304 ksp->ks_update = update_port_info_kstats;
3305 ksp->ks_private = (void *)pi;
3306 kstat_install(ksp);
3307
3308 return (ksp);
3309 }
3310
3311 static int
update_port_info_kstats(kstat_t * ksp,int rw)3312 update_port_info_kstats(kstat_t *ksp, int rw)
3313 {
3314 struct cxgbe_port_info_kstats *kstatp =
3315 (struct cxgbe_port_info_kstats *)ksp->ks_data;
3316 struct port_info *pi = ksp->ks_private;
3317 static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX",
3318 "active TWINAX", "LRM" };
3319 uint32_t bgmap;
3320
3321 if (rw == KSTAT_WRITE)
3322 return (0);
3323
3324 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
3325 KS_C_SET(transceiver, "unplugged");
3326 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
3327 KS_C_SET(transceiver, "unknown");
3328 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
3329 KS_C_SET(transceiver, "unsupported");
3330 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str))
3331 KS_C_SET(transceiver, "%s", mod_str[pi->mod_type]);
3332 else
3333 KS_C_SET(transceiver, "type %d", pi->mod_type);
3334
3335 #define GET_STAT(name) t4_read_reg64(pi->adapter, \
3336 PORT_REG(pi->port_id, A_MPS_PORT_STAT_##name##_L))
3337 #define GET_STAT_COM(name) t4_read_reg64(pi->adapter, \
3338 A_MPS_STAT_##name##_L)
3339
3340 bgmap = G_NUMPORTS(t4_read_reg(pi->adapter, A_MPS_CMN_CTL));
3341 if (bgmap == 0)
3342 bgmap = (pi->port_id == 0) ? 0xf : 0;
3343 else if (bgmap == 1)
3344 bgmap = (pi->port_id < 2) ? (3 << (2 * pi->port_id)) : 0;
3345 else
3346 bgmap = 1;
3347
3348 KS_U_SET(rx_ovflow0, (bgmap & 1) ?
3349 GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0);
3350 KS_U_SET(rx_ovflow1, (bgmap & 2) ?
3351 GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0);
3352 KS_U_SET(rx_ovflow2, (bgmap & 4) ?
3353 GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0);
3354 KS_U_SET(rx_ovflow3, (bgmap & 8) ?
3355 GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0);
3356 KS_U_SET(rx_trunc0, (bgmap & 1) ?
3357 GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0);
3358 KS_U_SET(rx_trunc1, (bgmap & 2) ?
3359 GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0);
3360 KS_U_SET(rx_trunc2, (bgmap & 4) ?
3361 GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0);
3362 KS_U_SET(rx_trunc3, (bgmap & 8) ?
3363 GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0);
3364
3365 KS_U_SET(tx_pause, GET_STAT(TX_PORT_PAUSE));
3366 KS_U_SET(rx_pause, GET_STAT(RX_PORT_PAUSE));
3367
3368 return (0);
3369
3370 }
3371
3372 /*
3373 * cxgbe:X:rxqY
3374 */
3375 struct rxq_kstats {
3376 kstat_named_t rxcsum;
3377 kstat_named_t rxpkts;
3378 kstat_named_t rxbytes;
3379 kstat_named_t nomem;
3380 };
3381
3382 static kstat_t *
setup_rxq_kstats(struct port_info * pi,struct sge_rxq * rxq,int idx)3383 setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, int idx)
3384 {
3385 struct kstat *ksp;
3386 struct rxq_kstats *kstatp;
3387 int ndata;
3388 char str[16];
3389
3390 ndata = sizeof (struct rxq_kstats) / sizeof (kstat_named_t);
3391 (void) snprintf(str, sizeof (str), "rxq%u", idx);
3392
3393 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "rxq",
3394 KSTAT_TYPE_NAMED, ndata, 0);
3395 if (ksp == NULL) {
3396 cxgb_printf(pi->dip, CE_WARN,
3397 "%s: failed to initialize rxq kstats for queue %d.",
3398 __func__, idx);
3399 return (NULL);
3400 }
3401
3402 kstatp = (struct rxq_kstats *)ksp->ks_data;
3403
3404 KS_UINIT(rxcsum);
3405 KS_UINIT(rxpkts);
3406 KS_UINIT(rxbytes);
3407 KS_UINIT(nomem);
3408
3409 ksp->ks_update = update_rxq_kstats;
3410 ksp->ks_private = (void *)rxq;
3411 kstat_install(ksp);
3412
3413 return (ksp);
3414 }
3415
3416 static int
update_rxq_kstats(kstat_t * ksp,int rw)3417 update_rxq_kstats(kstat_t *ksp, int rw)
3418 {
3419 struct rxq_kstats *kstatp = (struct rxq_kstats *)ksp->ks_data;
3420 struct sge_rxq *rxq = ksp->ks_private;
3421
3422 if (rw == KSTAT_WRITE)
3423 return (0);
3424
3425 KS_U_FROM(rxcsum, rxq);
3426 KS_U_FROM(rxpkts, rxq);
3427 KS_U_FROM(rxbytes, rxq);
3428 KS_U_FROM(nomem, rxq);
3429
3430 return (0);
3431 }
3432
3433 /*
3434 * cxgbe:X:txqY
3435 */
3436 struct txq_kstats {
3437 kstat_named_t txcsum;
3438 kstat_named_t tso_wrs;
3439 kstat_named_t imm_wrs;
3440 kstat_named_t sgl_wrs;
3441 kstat_named_t txpkt_wrs;
3442 kstat_named_t txpkts_wrs;
3443 kstat_named_t txpkts_pkts;
3444 kstat_named_t txb_used;
3445 kstat_named_t hdl_used;
3446 kstat_named_t txb_full;
3447 kstat_named_t dma_hdl_failed;
3448 kstat_named_t dma_map_failed;
3449 kstat_named_t qfull;
3450 kstat_named_t qflush;
3451 kstat_named_t pullup_early;
3452 kstat_named_t pullup_late;
3453 kstat_named_t pullup_failed;
3454 kstat_named_t csum_failed;
3455 };
3456
3457 static kstat_t *
setup_txq_kstats(struct port_info * pi,struct sge_txq * txq,int idx)3458 setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, int idx)
3459 {
3460 struct kstat *ksp;
3461 struct txq_kstats *kstatp;
3462 int ndata;
3463 char str[16];
3464
3465 ndata = sizeof (struct txq_kstats) / sizeof (kstat_named_t);
3466 (void) snprintf(str, sizeof (str), "txq%u", idx);
3467
3468 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "txq",
3469 KSTAT_TYPE_NAMED, ndata, 0);
3470 if (ksp == NULL) {
3471 cxgb_printf(pi->dip, CE_WARN,
3472 "%s: failed to initialize txq kstats for queue %d.",
3473 __func__, idx);
3474 return (NULL);
3475 }
3476
3477 kstatp = (struct txq_kstats *)ksp->ks_data;
3478
3479 KS_UINIT(txcsum);
3480 KS_UINIT(tso_wrs);
3481 KS_UINIT(imm_wrs);
3482 KS_UINIT(sgl_wrs);
3483 KS_UINIT(txpkt_wrs);
3484 KS_UINIT(txpkts_wrs);
3485 KS_UINIT(txpkts_pkts);
3486 KS_UINIT(txb_used);
3487 KS_UINIT(hdl_used);
3488 KS_UINIT(txb_full);
3489 KS_UINIT(dma_hdl_failed);
3490 KS_UINIT(dma_map_failed);
3491 KS_UINIT(qfull);
3492 KS_UINIT(qflush);
3493 KS_UINIT(pullup_early);
3494 KS_UINIT(pullup_late);
3495 KS_UINIT(pullup_failed);
3496 KS_UINIT(csum_failed);
3497
3498 ksp->ks_update = update_txq_kstats;
3499 ksp->ks_private = (void *)txq;
3500 kstat_install(ksp);
3501
3502 return (ksp);
3503 }
3504
3505 static int
update_txq_kstats(kstat_t * ksp,int rw)3506 update_txq_kstats(kstat_t *ksp, int rw)
3507 {
3508 struct txq_kstats *kstatp = (struct txq_kstats *)ksp->ks_data;
3509 struct sge_txq *txq = ksp->ks_private;
3510
3511 if (rw == KSTAT_WRITE)
3512 return (0);
3513
3514 KS_U_FROM(txcsum, txq);
3515 KS_U_FROM(tso_wrs, txq);
3516 KS_U_FROM(imm_wrs, txq);
3517 KS_U_FROM(sgl_wrs, txq);
3518 KS_U_FROM(txpkt_wrs, txq);
3519 KS_U_FROM(txpkts_wrs, txq);
3520 KS_U_FROM(txpkts_pkts, txq);
3521 KS_U_FROM(txb_used, txq);
3522 KS_U_FROM(hdl_used, txq);
3523 KS_U_FROM(txb_full, txq);
3524 KS_U_FROM(dma_hdl_failed, txq);
3525 KS_U_FROM(dma_map_failed, txq);
3526 KS_U_FROM(qfull, txq);
3527 KS_U_FROM(qflush, txq);
3528 KS_U_FROM(pullup_early, txq);
3529 KS_U_FROM(pullup_late, txq);
3530 KS_U_FROM(pullup_failed, txq);
3531 KS_U_FROM(csum_failed, txq);
3532
3533 return (0);
3534 }
3535
3536 static int rxbuf_ctor(void *, void *, int);
3537 static void rxbuf_dtor(void *, void *);
3538
3539 static kmem_cache_t *
rxbuf_cache_create(struct rxbuf_cache_params * p)3540 rxbuf_cache_create(struct rxbuf_cache_params *p)
3541 {
3542 char name[32];
3543
3544 (void) snprintf(name, sizeof (name), "%s%d_rxbuf_cache",
3545 ddi_driver_name(p->dip), ddi_get_instance(p->dip));
3546
3547 return kmem_cache_create(name, sizeof (struct rxbuf), _CACHE_LINE_SIZE,
3548 rxbuf_ctor, rxbuf_dtor, NULL, p, NULL, 0);
3549 }
3550
3551 /*
3552 * If ref_cnt is more than 1 then those many calls to rxbuf_free will
3553 * have to be made before the rxb is released back to the kmem_cache.
3554 */
3555 static struct rxbuf *
rxbuf_alloc(kmem_cache_t * cache,int kmflags,uint_t ref_cnt)3556 rxbuf_alloc(kmem_cache_t *cache, int kmflags, uint_t ref_cnt)
3557 {
3558 struct rxbuf *rxb;
3559
3560 ASSERT(ref_cnt > 0);
3561
3562 rxb = kmem_cache_alloc(cache, kmflags);
3563 if (rxb != NULL) {
3564 rxb->ref_cnt = ref_cnt;
3565 rxb->cache = cache;
3566 }
3567
3568 return (rxb);
3569 }
3570
3571 /*
3572 * This is normally called via the rxb's freefunc, when an mblk referencing the
3573 * rxb is freed.
3574 */
3575 static void
rxbuf_free(struct rxbuf * rxb)3576 rxbuf_free(struct rxbuf *rxb)
3577 {
3578 if (atomic_dec_uint_nv(&rxb->ref_cnt) == 0)
3579 kmem_cache_free(rxb->cache, rxb);
3580 }
3581
3582 static int
rxbuf_ctor(void * arg1,void * arg2,int kmflag)3583 rxbuf_ctor(void *arg1, void *arg2, int kmflag)
3584 {
3585 struct rxbuf *rxb = arg1;
3586 struct rxbuf_cache_params *p = arg2;
3587 size_t real_len;
3588 ddi_dma_cookie_t cookie;
3589 uint_t ccount = 0;
3590 int (*callback)(caddr_t);
3591 int rc = ENOMEM;
3592
3593 if ((kmflag & KM_NOSLEEP) != 0)
3594 callback = DDI_DMA_DONTWAIT;
3595 else
3596 callback = DDI_DMA_SLEEP;
3597
3598 rc = ddi_dma_alloc_handle(p->dip, &p->dma_attr_rx, callback, 0,
3599 &rxb->dhdl);
3600 if (rc != DDI_SUCCESS)
3601 return (rc == DDI_DMA_BADATTR ? EINVAL : ENOMEM);
3602
3603 rc = ddi_dma_mem_alloc(rxb->dhdl, p->buf_size, &p->acc_attr_rx,
3604 DDI_DMA_STREAMING, callback, 0, &rxb->va, &real_len, &rxb->ahdl);
3605 if (rc != DDI_SUCCESS) {
3606 rc = ENOMEM;
3607 goto fail1;
3608 }
3609
3610 rc = ddi_dma_addr_bind_handle(rxb->dhdl, NULL, rxb->va, p->buf_size,
3611 DDI_DMA_READ | DDI_DMA_STREAMING, NULL, NULL, &cookie, &ccount);
3612 if (rc != DDI_DMA_MAPPED) {
3613 if (rc == DDI_DMA_INUSE)
3614 rc = EBUSY;
3615 else if (rc == DDI_DMA_TOOBIG)
3616 rc = E2BIG;
3617 else
3618 rc = ENOMEM;
3619 goto fail2;
3620 }
3621
3622 if (ccount != 1) {
3623 rc = E2BIG;
3624 goto fail3;
3625 }
3626
3627 rxb->ref_cnt = 0;
3628 rxb->buf_size = p->buf_size;
3629 rxb->freefunc.free_arg = (caddr_t)rxb;
3630 rxb->freefunc.free_func = rxbuf_free;
3631 rxb->ba = cookie.dmac_laddress;
3632
3633 return (0);
3634
3635 fail3: (void) ddi_dma_unbind_handle(rxb->dhdl);
3636 fail2: ddi_dma_mem_free(&rxb->ahdl);
3637 fail1: ddi_dma_free_handle(&rxb->dhdl);
3638 return (rc);
3639 }
3640
3641 static void
rxbuf_dtor(void * arg1,void * arg2)3642 rxbuf_dtor(void *arg1, void *arg2)
3643 {
3644 struct rxbuf *rxb = arg1;
3645
3646 (void) ddi_dma_unbind_handle(rxb->dhdl);
3647 ddi_dma_mem_free(&rxb->ahdl);
3648 ddi_dma_free_handle(&rxb->dhdl);
3649 }
3650