1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * This file is part of the Chelsio T4 support code.
14 *
15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved.
16 *
17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
20 * release for licensing terms and conditions.
21 */
22
23 /*
24 * Copyright 2021 Oxide Computer Company
25 */
26
27 #include <sys/ddi.h>
28 #include <sys/sunddi.h>
29 #include <sys/sunndi.h>
30 #include <sys/atomic.h>
31 #include <sys/dlpi.h>
32 #include <sys/pattr.h>
33 #include <sys/strsubr.h>
34 #include <sys/stream.h>
35 #include <sys/strsun.h>
36 #include <inet/ip.h>
37 #include <inet/tcp.h>
38
39 #include "version.h"
40 #include "common/common.h"
41 #include "common/t4_msg.h"
42 #include "common/t4_regs.h"
43 #include "common/t4_regs_values.h"
44
45 /* TODO: Tune. */
46 int rx_buf_size = 8192;
47 int tx_copy_threshold = 256;
48 uint16_t rx_copy_threshold = 256;
49
50 /* Used to track coalesced tx work request */
51 struct txpkts {
52 mblk_t *tail; /* head is in the software descriptor */
53 uint64_t *flitp; /* ptr to flit where next pkt should start */
54 uint8_t npkt; /* # of packets in this work request */
55 uint8_t nflits; /* # of flits used by this work request */
56 uint16_t plen; /* total payload (sum of all packets) */
57 };
58
59 /* All information needed to tx a frame */
60 struct txinfo {
61 uint32_t len; /* Total length of frame */
62 uint32_t flags; /* Checksum and LSO flags */
63 uint32_t mss; /* MSS for LSO */
64 uint8_t nsegs; /* # of segments in the SGL, 0 means imm. tx */
65 uint8_t nflits; /* # of flits needed for the SGL */
66 uint8_t hdls_used; /* # of DMA handles used */
67 uint32_t txb_used; /* txb_space used */
68 struct ulptx_sgl sgl __attribute__((aligned(8)));
69 struct ulptx_sge_pair reserved[TX_SGL_SEGS / 2];
70 };
71
72 static int service_iq(struct sge_iq *iq, int budget);
73 static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx,
74 int8_t pktc_idx, int qsize, uint8_t esize);
75 static inline void init_fl(struct sge_fl *fl, uint16_t qsize);
76 static inline void init_eq(struct adapter *sc, struct sge_eq *eq,
77 uint16_t eqtype, uint16_t qsize,uint8_t tx_chan, uint16_t iqid);
78 static int alloc_iq_fl(struct port_info *pi, struct sge_iq *iq,
79 struct sge_fl *fl, int intr_idx, int cong);
80 static int free_iq_fl(struct port_info *pi, struct sge_iq *iq,
81 struct sge_fl *fl);
82 static int alloc_fwq(struct adapter *sc);
83 static int free_fwq(struct adapter *sc);
84 #ifdef TCP_OFFLOAD_ENABLE
85 static int alloc_mgmtq(struct adapter *sc);
86 #endif
87 static int alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx,
88 int i);
89 static int free_rxq(struct port_info *pi, struct sge_rxq *rxq);
90 #ifdef TCP_OFFLOAD_ENABLE
91 static int alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq,
92 int intr_idx);
93 static int free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq);
94 #endif
95 static int ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq);
96 static int eth_eq_alloc(struct adapter *sc, struct port_info *pi,
97 struct sge_eq *eq);
98 #ifdef TCP_OFFLOAD_ENABLE
99 static int ofld_eq_alloc(struct adapter *sc, struct port_info *pi,
100 struct sge_eq *eq);
101 #endif
102 static int alloc_eq(struct adapter *sc, struct port_info *pi,
103 struct sge_eq *eq);
104 static int free_eq(struct adapter *sc, struct sge_eq *eq);
105 #ifdef TCP_OFFLOAD_ENABLE
106 static int alloc_wrq(struct adapter *sc, struct port_info *pi,
107 struct sge_wrq *wrq, int idx);
108 static int free_wrq(struct adapter *sc, struct sge_wrq *wrq);
109 #endif
110 static int alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx);
111 static int free_txq(struct port_info *pi, struct sge_txq *txq);
112 static int alloc_dma_memory(struct adapter *sc, size_t len, int flags,
113 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr,
114 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
115 caddr_t *pva);
116 static int free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl);
117 static int alloc_desc_ring(struct adapter *sc, size_t len, int rw,
118 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
119 caddr_t *pva);
120 static int free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl);
121 static int alloc_tx_copybuffer(struct adapter *sc, size_t len,
122 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
123 caddr_t *pva);
124 static inline bool is_new_response(const struct sge_iq *iq,
125 struct rsp_ctrl **ctrl);
126 static inline void iq_next(struct sge_iq *iq);
127 static int refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs);
128 static void refill_sfl(void *arg);
129 static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl);
130 static void free_fl_bufs(struct sge_fl *fl);
131 static mblk_t *get_fl_payload(struct adapter *sc, struct sge_fl *fl,
132 uint32_t len_newbuf, int *fl_bufs_used);
133 static int get_frame_txinfo(struct sge_txq *txq, mblk_t **fp,
134 struct txinfo *txinfo, int sgl_only);
135 static inline int fits_in_txb(struct sge_txq *txq, int len, int *waste);
136 static inline int copy_into_txb(struct sge_txq *txq, mblk_t *m, int len,
137 struct txinfo *txinfo);
138 static inline void add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len);
139 static inline int add_mblk(struct sge_txq *txq, struct txinfo *txinfo,
140 mblk_t *m, int len);
141 static void free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo);
142 static int add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m,
143 struct txinfo *txinfo);
144 static void write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts);
145 static int write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m,
146 struct txinfo *txinfo);
147 static inline void write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
148 struct txpkts *txpkts, struct txinfo *txinfo);
149 static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to,
150 int len);
151 static inline void ring_tx_db(struct adapter *sc, struct sge_eq *eq);
152 static int reclaim_tx_descs(struct sge_txq *txq, int howmany);
153 static void write_txqflush_wr(struct sge_txq *txq);
154 static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss,
155 mblk_t *m);
156 static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl);
157 static kstat_t *setup_port_config_kstats(struct port_info *pi);
158 static kstat_t *setup_port_info_kstats(struct port_info *pi);
159 static kstat_t *setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq,
160 int idx);
161 static int update_rxq_kstats(kstat_t *ksp, int rw);
162 static int update_port_info_kstats(kstat_t *ksp, int rw);
163 static kstat_t *setup_txq_kstats(struct port_info *pi, struct sge_txq *txq,
164 int idx);
165 static int update_txq_kstats(kstat_t *ksp, int rw);
166 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *,
167 mblk_t *);
168 static int handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss,
169 mblk_t *m);
170
171 static inline int
reclaimable(struct sge_eq * eq)172 reclaimable(struct sge_eq *eq)
173 {
174 unsigned int cidx;
175
176 cidx = eq->spg->cidx; /* stable snapshot */
177 cidx = be16_to_cpu(cidx);
178
179 if (cidx >= eq->cidx)
180 return (cidx - eq->cidx);
181 else
182 return (cidx + eq->cap - eq->cidx);
183 }
184
185 void
t4_sge_init(struct adapter * sc)186 t4_sge_init(struct adapter *sc)
187 {
188 struct driver_properties *p = &sc->props;
189 ddi_dma_attr_t *dma_attr;
190 ddi_device_acc_attr_t *acc_attr;
191 uint32_t sge_control, sge_conm_ctrl;
192 int egress_threshold;
193
194 /*
195 * Device access and DMA attributes for descriptor rings
196 */
197 acc_attr = &sc->sge.acc_attr_desc;
198 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
199 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
200 acc_attr->devacc_attr_dataorder = DDI_STRICTORDER_ACC;
201
202 dma_attr = &sc->sge.dma_attr_desc;
203 dma_attr->dma_attr_version = DMA_ATTR_V0;
204 dma_attr->dma_attr_addr_lo = 0;
205 dma_attr->dma_attr_addr_hi = UINT64_MAX;
206 dma_attr->dma_attr_count_max = UINT64_MAX;
207 dma_attr->dma_attr_align = 512;
208 dma_attr->dma_attr_burstsizes = 0xfff;
209 dma_attr->dma_attr_minxfer = 1;
210 dma_attr->dma_attr_maxxfer = UINT64_MAX;
211 dma_attr->dma_attr_seg = UINT64_MAX;
212 dma_attr->dma_attr_sgllen = 1;
213 dma_attr->dma_attr_granular = 1;
214 dma_attr->dma_attr_flags = 0;
215
216 /*
217 * Device access and DMA attributes for tx buffers
218 */
219 acc_attr = &sc->sge.acc_attr_tx;
220 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
221 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
222
223 dma_attr = &sc->sge.dma_attr_tx;
224 dma_attr->dma_attr_version = DMA_ATTR_V0;
225 dma_attr->dma_attr_addr_lo = 0;
226 dma_attr->dma_attr_addr_hi = UINT64_MAX;
227 dma_attr->dma_attr_count_max = UINT64_MAX;
228 dma_attr->dma_attr_align = 1;
229 dma_attr->dma_attr_burstsizes = 0xfff;
230 dma_attr->dma_attr_minxfer = 1;
231 dma_attr->dma_attr_maxxfer = UINT64_MAX;
232 dma_attr->dma_attr_seg = UINT64_MAX;
233 dma_attr->dma_attr_sgllen = TX_SGL_SEGS;
234 dma_attr->dma_attr_granular = 1;
235 dma_attr->dma_attr_flags = 0;
236
237 /*
238 * Ingress Padding Boundary and Egress Status Page Size are set up by
239 * t4_fixup_host_params().
240 */
241 sge_control = t4_read_reg(sc, A_SGE_CONTROL);
242 sc->sge.pktshift = G_PKTSHIFT(sge_control);
243 sc->sge.stat_len = (sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64;
244
245 /* t4_nex uses FLM packed mode */
246 sc->sge.fl_align = t4_fl_pkt_align(sc, true);
247
248 /*
249 * Device access and DMA attributes for rx buffers
250 */
251 sc->sge.rxb_params.dip = sc->dip;
252 sc->sge.rxb_params.buf_size = rx_buf_size;
253
254 acc_attr = &sc->sge.rxb_params.acc_attr_rx;
255 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
256 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
257
258 dma_attr = &sc->sge.rxb_params.dma_attr_rx;
259 dma_attr->dma_attr_version = DMA_ATTR_V0;
260 dma_attr->dma_attr_addr_lo = 0;
261 dma_attr->dma_attr_addr_hi = UINT64_MAX;
262 dma_attr->dma_attr_count_max = UINT64_MAX;
263 /*
264 * Low 4 bits of an rx buffer address have a special meaning to the SGE
265 * and an rx buf cannot have an address with any of these bits set.
266 * FL_ALIGN is >= 32 so we're sure things are ok.
267 */
268 dma_attr->dma_attr_align = sc->sge.fl_align;
269 dma_attr->dma_attr_burstsizes = 0xfff;
270 dma_attr->dma_attr_minxfer = 1;
271 dma_attr->dma_attr_maxxfer = UINT64_MAX;
272 dma_attr->dma_attr_seg = UINT64_MAX;
273 dma_attr->dma_attr_sgllen = 1;
274 dma_attr->dma_attr_granular = 1;
275 dma_attr->dma_attr_flags = 0;
276
277 sc->sge.rxbuf_cache = rxbuf_cache_create(&sc->sge.rxb_params);
278
279 /*
280 * A FL with <= fl_starve_thres buffers is starving and a periodic
281 * timer will attempt to refill it. This needs to be larger than the
282 * SGE's Egress Congestion Threshold. If it isn't, then we can get
283 * stuck waiting for new packets while the SGE is waiting for us to
284 * give it more Free List entries. (Note that the SGE's Egress
285 * Congestion Threshold is in units of 2 Free List pointers.) For T4,
286 * there was only a single field to control this. For T5 there's the
287 * original field which now only applies to Unpacked Mode Free List
288 * buffers and a new field which only applies to Packed Mode Free List
289 * buffers.
290 */
291
292 sge_conm_ctrl = t4_read_reg(sc, A_SGE_CONM_CTRL);
293 switch (CHELSIO_CHIP_VERSION(sc->params.chip)) {
294 case CHELSIO_T4:
295 egress_threshold = G_EGRTHRESHOLD(sge_conm_ctrl);
296 break;
297 case CHELSIO_T5:
298 egress_threshold = G_EGRTHRESHOLDPACKING(sge_conm_ctrl);
299 break;
300 case CHELSIO_T6:
301 default:
302 egress_threshold = G_T6_EGRTHRESHOLDPACKING(sge_conm_ctrl);
303 }
304 sc->sge.fl_starve_threshold = 2*egress_threshold + 1;
305
306 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, rx_buf_size);
307
308 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD,
309 V_THRESHOLD_0(p->counter_val[0]) |
310 V_THRESHOLD_1(p->counter_val[1]) |
311 V_THRESHOLD_2(p->counter_val[2]) |
312 V_THRESHOLD_3(p->counter_val[3]));
313
314 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1,
315 V_TIMERVALUE0(us_to_core_ticks(sc, p->timer_val[0])) |
316 V_TIMERVALUE1(us_to_core_ticks(sc, p->timer_val[1])));
317 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3,
318 V_TIMERVALUE2(us_to_core_ticks(sc, p->timer_val[2])) |
319 V_TIMERVALUE3(us_to_core_ticks(sc, p->timer_val[3])));
320 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5,
321 V_TIMERVALUE4(us_to_core_ticks(sc, p->timer_val[4])) |
322 V_TIMERVALUE5(us_to_core_ticks(sc, p->timer_val[5])));
323
324 (void) t4_register_cpl_handler(sc, CPL_FW4_MSG, handle_fw_rpl);
325 (void) t4_register_cpl_handler(sc, CPL_FW6_MSG, handle_fw_rpl);
326 (void) t4_register_cpl_handler(sc, CPL_SGE_EGR_UPDATE, handle_sge_egr_update);
327 (void) t4_register_cpl_handler(sc, CPL_RX_PKT, t4_eth_rx);
328 (void) t4_register_fw_msg_handler(sc, FW6_TYPE_CMD_RPL,
329 t4_handle_fw_rpl);
330 }
331
332 /*
333 * Allocate and initialize the firmware event queue and the forwarded interrupt
334 * queues, if any. The adapter owns all these queues as they are not associated
335 * with any particular port.
336 *
337 * Returns errno on failure. Resources allocated up to that point may still be
338 * allocated. Caller is responsible for cleanup in case this function fails.
339 */
340 int
t4_setup_adapter_queues(struct adapter * sc)341 t4_setup_adapter_queues(struct adapter *sc)
342 {
343 int rc;
344
345 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
346
347 /*
348 * Firmware event queue
349 */
350 rc = alloc_fwq(sc);
351 if (rc != 0)
352 return (rc);
353
354 #ifdef TCP_OFFLOAD_ENABLE
355 /*
356 * Management queue. This is just a control queue that uses the fwq as
357 * its associated iq.
358 */
359 rc = alloc_mgmtq(sc);
360 #endif
361
362 return (rc);
363 }
364
365 /*
366 * Idempotent
367 */
368 int
t4_teardown_adapter_queues(struct adapter * sc)369 t4_teardown_adapter_queues(struct adapter *sc)
370 {
371
372 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
373
374 (void) free_fwq(sc);
375
376 return (0);
377 }
378
379 static inline int
first_vector(struct port_info * pi)380 first_vector(struct port_info *pi)
381 {
382 struct adapter *sc = pi->adapter;
383 int rc = T4_EXTRA_INTR, i;
384
385 if (sc->intr_count == 1)
386 return (0);
387
388 for_each_port(sc, i) {
389 struct port_info *p = sc->port[i];
390
391 if (i == pi->port_id)
392 break;
393
394 #ifdef TCP_OFFLOAD_ENABLE
395 if (!(sc->flags & INTR_FWD))
396 rc += p->nrxq + p->nofldrxq;
397 else
398 rc += max(p->nrxq, p->nofldrxq);
399 #else
400 /*
401 * Not compiled with offload support and intr_count > 1. Only
402 * NIC queues exist and they'd better be taking direct
403 * interrupts.
404 */
405 ASSERT(!(sc->flags & INTR_FWD));
406 rc += p->nrxq;
407 #endif
408 }
409 return (rc);
410 }
411
412 /*
413 * Given an arbitrary "index," come up with an iq that can be used by other
414 * queues (of this port) for interrupt forwarding, SGE egress updates, etc.
415 * The iq returned is guaranteed to be something that takes direct interrupts.
416 */
417 static struct sge_iq *
port_intr_iq(struct port_info * pi,int idx)418 port_intr_iq(struct port_info *pi, int idx)
419 {
420 struct adapter *sc = pi->adapter;
421 struct sge *s = &sc->sge;
422 struct sge_iq *iq = NULL;
423
424 if (sc->intr_count == 1)
425 return (&sc->sge.fwq);
426
427 #ifdef TCP_OFFLOAD_ENABLE
428 if (!(sc->flags & INTR_FWD)) {
429 idx %= pi->nrxq + pi->nofldrxq;
430
431 if (idx >= pi->nrxq) {
432 idx -= pi->nrxq;
433 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
434 } else
435 iq = &s->rxq[pi->first_rxq + idx].iq;
436
437 } else {
438 idx %= max(pi->nrxq, pi->nofldrxq);
439
440 if (pi->nrxq >= pi->nofldrxq)
441 iq = &s->rxq[pi->first_rxq + idx].iq;
442 else
443 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
444 }
445 #else
446 /*
447 * Not compiled with offload support and intr_count > 1. Only NIC
448 * queues exist and they'd better be taking direct interrupts.
449 */
450 ASSERT(!(sc->flags & INTR_FWD));
451
452 idx %= pi->nrxq;
453 iq = &s->rxq[pi->first_rxq + idx].iq;
454 #endif
455
456 return (iq);
457 }
458
459 int
t4_setup_port_queues(struct port_info * pi)460 t4_setup_port_queues(struct port_info *pi)
461 {
462 int rc = 0, i, intr_idx, j;
463 struct sge_rxq *rxq;
464 struct sge_txq *txq;
465 #ifdef TCP_OFFLOAD_ENABLE
466 int iqid;
467 struct sge_wrq *ctrlq;
468 struct sge_ofld_rxq *ofld_rxq;
469 struct sge_wrq *ofld_txq;
470 #endif
471 struct adapter *sc = pi->adapter;
472 struct driver_properties *p = &sc->props;
473
474 pi->ksp_config = setup_port_config_kstats(pi);
475 pi->ksp_info = setup_port_info_kstats(pi);
476
477 /* Interrupt vector to start from (when using multiple vectors) */
478 intr_idx = first_vector(pi);
479
480 /*
481 * First pass over all rx queues (NIC and TOE):
482 * a) initialize iq and fl
483 * b) allocate queue iff it will take direct interrupts.
484 */
485
486 for_each_rxq(pi, i, rxq) {
487
488 init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, p->qsize_rxq,
489 RX_IQ_ESIZE);
490
491 init_fl(&rxq->fl, p->qsize_rxq / 8); /* 8 bufs in each entry */
492
493 if ((!(sc->flags & INTR_FWD))
494 #ifdef TCP_OFFLOAD_ENABLE
495 || (sc->intr_count > 1 && pi->nrxq >= pi->nofldrxq)
496 #else
497 || (sc->intr_count > 1 && pi->nrxq)
498 #endif
499 ) {
500 rxq->iq.flags |= IQ_INTR;
501 rc = alloc_rxq(pi, rxq, intr_idx, i);
502 if (rc != 0)
503 goto done;
504 intr_idx++;
505 }
506
507 }
508
509 #ifdef TCP_OFFLOAD_ENABLE
510 for_each_ofld_rxq(pi, i, ofld_rxq) {
511
512 init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
513 p->qsize_rxq, RX_IQ_ESIZE);
514
515 init_fl(&ofld_rxq->fl, p->qsize_rxq / 8);
516
517 if (!(sc->flags & INTR_FWD) ||
518 (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) {
519 ofld_rxq->iq.flags = IQ_INTR;
520 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx);
521 if (rc != 0)
522 goto done;
523
524 intr_idx++;
525 }
526 }
527 #endif
528
529 /*
530 * Second pass over all rx queues (NIC and TOE). The queues forwarding
531 * their interrupts are allocated now.
532 */
533 j = 0;
534 for_each_rxq(pi, i, rxq) {
535 if (rxq->iq.flags & IQ_INTR)
536 continue;
537
538 intr_idx = port_intr_iq(pi, j)->abs_id;
539
540 rc = alloc_rxq(pi, rxq, intr_idx, i);
541 if (rc != 0)
542 goto done;
543 j++;
544 }
545
546 #ifdef TCP_OFFLOAD_ENABLE
547 for_each_ofld_rxq(pi, i, ofld_rxq) {
548 if (ofld_rxq->iq.flags & IQ_INTR)
549 continue;
550
551 intr_idx = port_intr_iq(pi, j)->abs_id;
552 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx);
553 if (rc != 0)
554 goto done;
555 j++;
556 }
557 #endif
558 /*
559 * Now the tx queues. Only one pass needed.
560 */
561 j = 0;
562 for_each_txq(pi, i, txq) {
563 uint16_t iqid;
564
565 iqid = port_intr_iq(pi, j)->cntxt_id;
566 init_eq(sc, &txq->eq, EQ_ETH, p->qsize_txq, pi->tx_chan, iqid);
567 rc = alloc_txq(pi, txq, i);
568 if (rc != 0)
569 goto done;
570 }
571
572 #ifdef TCP_OFFLOAD_ENABLE
573 for_each_ofld_txq(pi, i, ofld_txq) {
574 uint16_t iqid;
575
576 iqid = port_intr_iq(pi, j)->cntxt_id;
577 init_eq(sc, &ofld_txq->eq, EQ_OFLD, p->qsize_txq, pi->tx_chan,
578 iqid);
579 rc = alloc_wrq(sc, pi, ofld_txq, i);
580 if (rc != 0)
581 goto done;
582 }
583
584 /*
585 * Finally, the control queue.
586 */
587 ctrlq = &sc->sge.ctrlq[pi->port_id];
588 iqid = port_intr_iq(pi, 0)->cntxt_id;
589 init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, iqid);
590 rc = alloc_wrq(sc, pi, ctrlq, 0);
591 #endif
592
593 done:
594 if (rc != 0)
595 (void) t4_teardown_port_queues(pi);
596
597 return (rc);
598 }
599
600 /*
601 * Idempotent
602 */
603 int
t4_teardown_port_queues(struct port_info * pi)604 t4_teardown_port_queues(struct port_info *pi)
605 {
606 int i;
607 struct sge_rxq *rxq;
608 struct sge_txq *txq;
609 #ifdef TCP_OFFLOAD_ENABLE
610 struct adapter *sc = pi->adapter;
611 struct sge_ofld_rxq *ofld_rxq;
612 struct sge_wrq *ofld_txq;
613 #endif
614
615 if (pi->ksp_config != NULL) {
616 kstat_delete(pi->ksp_config);
617 pi->ksp_config = NULL;
618 }
619 if (pi->ksp_info != NULL) {
620 kstat_delete(pi->ksp_info);
621 pi->ksp_info = NULL;
622 }
623
624 #ifdef TCP_OFFLOAD_ENABLE
625 (void) free_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
626 #endif
627
628 for_each_txq(pi, i, txq) {
629 (void) free_txq(pi, txq);
630 }
631
632 #ifdef TCP_OFFLOAD_ENABLE
633 for_each_ofld_txq(pi, i, ofld_txq) {
634 (void) free_wrq(sc, ofld_txq);
635 }
636
637 for_each_ofld_rxq(pi, i, ofld_rxq) {
638 if ((ofld_rxq->iq.flags & IQ_INTR) == 0)
639 (void) free_ofld_rxq(pi, ofld_rxq);
640 }
641 #endif
642
643 for_each_rxq(pi, i, rxq) {
644 if ((rxq->iq.flags & IQ_INTR) == 0)
645 (void) free_rxq(pi, rxq);
646 }
647
648 /*
649 * Then take down the rx queues that take direct interrupts.
650 */
651
652 for_each_rxq(pi, i, rxq) {
653 if (rxq->iq.flags & IQ_INTR)
654 (void) free_rxq(pi, rxq);
655 }
656
657 #ifdef TCP_OFFLOAD_ENABLE
658 for_each_ofld_rxq(pi, i, ofld_rxq) {
659 if (ofld_rxq->iq.flags & IQ_INTR)
660 (void) free_ofld_rxq(pi, ofld_rxq);
661 }
662 #endif
663
664 return (0);
665 }
666
667 /* Deals with errors and forwarded interrupts */
668 uint_t
t4_intr_all(caddr_t arg1,caddr_t arg2)669 t4_intr_all(caddr_t arg1, caddr_t arg2)
670 {
671
672 (void) t4_intr_err(arg1, arg2);
673 (void) t4_intr(arg1, arg2);
674
675 return (DDI_INTR_CLAIMED);
676 }
677
678 static void
t4_intr_rx_work(struct sge_iq * iq)679 t4_intr_rx_work(struct sge_iq *iq)
680 {
681 mblk_t *mp = NULL;
682 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
683 RXQ_LOCK(rxq);
684 if (!iq->polling) {
685 mp = t4_ring_rx(rxq, iq->qsize/8);
686 t4_write_reg(iq->adapter, MYPF_REG(A_SGE_PF_GTS),
687 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
688 }
689 RXQ_UNLOCK(rxq);
690 if (mp != NULL)
691 mac_rx_ring(rxq->port->mh, rxq->ring_handle, mp,
692 rxq->ring_gen_num);
693 }
694
695 /* Deals with interrupts on the given ingress queue */
696 /* ARGSUSED */
697 uint_t
t4_intr(caddr_t arg1,caddr_t arg2)698 t4_intr(caddr_t arg1, caddr_t arg2)
699 {
700 struct sge_iq *iq = (struct sge_iq *)arg2;
701 int state;
702
703 /* Right now receive polling is only enabled for MSI-X and
704 * when we have enough msi-x vectors i.e no interrupt forwarding.
705 */
706 if (iq->adapter->props.multi_rings) {
707 t4_intr_rx_work(iq);
708 } else {
709 state = atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY);
710 if (state == IQS_IDLE) {
711 (void) service_iq(iq, 0);
712 (void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE);
713 }
714 }
715 return (DDI_INTR_CLAIMED);
716 }
717
718 /* Deals with error interrupts */
719 /* ARGSUSED */
720 uint_t
t4_intr_err(caddr_t arg1,caddr_t arg2)721 t4_intr_err(caddr_t arg1, caddr_t arg2)
722 {
723 /* LINTED: E_BAD_PTR_CAST_ALIGN */
724 struct adapter *sc = (struct adapter *)arg1;
725
726 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0);
727 (void) t4_slow_intr_handler(sc);
728
729 return (DDI_INTR_CLAIMED);
730 }
731
732 /*
733 * t4_ring_rx - Process responses from an SGE response queue.
734 *
735 * This function processes responses from an SGE response queue up to the supplied budget.
736 * Responses include received packets as well as control messages from FW
737 * or HW.
738 * It returns a chain of mblks containing the received data, to be
739 * passed up to mac_ring_rx().
740 */
741 mblk_t *
t4_ring_rx(struct sge_rxq * rxq,int budget)742 t4_ring_rx(struct sge_rxq *rxq, int budget)
743 {
744 struct sge_iq *iq = &rxq->iq;
745 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
746 struct adapter *sc = iq->adapter;
747 struct rsp_ctrl *ctrl;
748 const struct rss_header *rss;
749 int ndescs = 0, fl_bufs_used = 0;
750 int rsp_type;
751 uint32_t lq;
752 mblk_t *mblk_head = NULL, **mblk_tail, *m;
753 struct cpl_rx_pkt *cpl;
754 uint32_t received_bytes = 0, pkt_len = 0;
755 bool csum_ok;
756 uint16_t err_vec;
757
758 mblk_tail = &mblk_head;
759
760 while (is_new_response(iq, &ctrl)) {
761
762 membar_consumer();
763
764 m = NULL;
765 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
766 lq = be32_to_cpu(ctrl->pldbuflen_qid);
767 rss = (const void *)iq->cdesc;
768
769 switch (rsp_type) {
770 case X_RSPD_TYPE_FLBUF:
771
772 ASSERT(iq->flags & IQ_HAS_FL);
773
774 if (CPL_RX_PKT == rss->opcode) {
775 cpl = (void *)(rss + 1);
776 pkt_len = be16_to_cpu(cpl->len);
777
778 if (iq->polling && ((received_bytes + pkt_len) > budget))
779 goto done;
780
781 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
782 if (m == NULL)
783 goto done;
784
785 iq->intr_next = iq->intr_params;
786 m->b_rptr += sc->sge.pktshift;
787 if (sc->params.tp.rx_pkt_encap)
788 /* It is enabled only in T6 config file */
789 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec));
790 else
791 err_vec = ntohs(cpl->err_vec);
792
793 csum_ok = cpl->csum_calc && !err_vec;
794
795 /* TODO: what about cpl->ip_frag? */
796 if (csum_ok && !cpl->ip_frag) {
797 mac_hcksum_set(m, 0, 0, 0, 0xffff,
798 HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
799 HCK_IPV4_HDRCKSUM_OK);
800 rxq->rxcsum++;
801 }
802 rxq->rxpkts++;
803 rxq->rxbytes += pkt_len;
804 received_bytes += pkt_len;
805
806 *mblk_tail = m;
807 mblk_tail = &m->b_next;
808
809 break;
810 }
811
812 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
813 if (m == NULL)
814 goto done;
815 /* FALLTHROUGH */
816
817 case X_RSPD_TYPE_CPL:
818 ASSERT(rss->opcode < NUM_CPL_CMDS);
819 sc->cpl_handler[rss->opcode](iq, rss, m);
820 break;
821
822 default:
823 break;
824 }
825 iq_next(iq);
826 ++ndescs;
827 if (!iq->polling && (ndescs == budget))
828 break;
829 }
830
831 done:
832
833 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
834 V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) |
835 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
836
837 if ((fl_bufs_used > 0) || (iq->flags & IQ_HAS_FL)) {
838 int starved;
839 FL_LOCK(fl);
840 fl->needed += fl_bufs_used;
841 starved = refill_fl(sc, fl, fl->cap / 8);
842 FL_UNLOCK(fl);
843 if (starved)
844 add_fl_to_sfl(sc, fl);
845 }
846 return (mblk_head);
847 }
848
849 /*
850 * Deals with anything and everything on the given ingress queue.
851 */
852 static int
service_iq(struct sge_iq * iq,int budget)853 service_iq(struct sge_iq *iq, int budget)
854 {
855 struct sge_iq *q;
856 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
857 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
858 struct adapter *sc = iq->adapter;
859 struct rsp_ctrl *ctrl;
860 const struct rss_header *rss;
861 int ndescs = 0, limit, fl_bufs_used = 0;
862 int rsp_type;
863 uint32_t lq;
864 int starved;
865 mblk_t *m;
866 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
867
868 limit = budget ? budget : iq->qsize / 8;
869
870 /*
871 * We always come back and check the descriptor ring for new indirect
872 * interrupts and other responses after running a single handler.
873 */
874 for (;;) {
875 while (is_new_response(iq, &ctrl)) {
876
877 membar_consumer();
878
879 m = NULL;
880 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
881 lq = be32_to_cpu(ctrl->pldbuflen_qid);
882 rss = (const void *)iq->cdesc;
883
884 switch (rsp_type) {
885 case X_RSPD_TYPE_FLBUF:
886
887 ASSERT(iq->flags & IQ_HAS_FL);
888
889 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
890 if (m == NULL) {
891 /*
892 * Rearm the iq with a
893 * longer-than-default timer
894 */
895 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
896 V_INGRESSQID((u32)iq->cntxt_id) |
897 V_SEINTARM(V_QINTR_TIMER_IDX(SGE_NTIMERS-1)));
898 if (fl_bufs_used > 0) {
899 ASSERT(iq->flags & IQ_HAS_FL);
900 FL_LOCK(fl);
901 fl->needed += fl_bufs_used;
902 starved = refill_fl(sc, fl, fl->cap / 8);
903 FL_UNLOCK(fl);
904 if (starved)
905 add_fl_to_sfl(sc, fl);
906 }
907 return (0);
908 }
909
910 /* FALLTHRU */
911 case X_RSPD_TYPE_CPL:
912
913 ASSERT(rss->opcode < NUM_CPL_CMDS);
914 sc->cpl_handler[rss->opcode](iq, rss, m);
915 break;
916
917 case X_RSPD_TYPE_INTR:
918
919 /*
920 * Interrupts should be forwarded only to queues
921 * that are not forwarding their interrupts.
922 * This means service_iq can recurse but only 1
923 * level deep.
924 */
925 ASSERT(budget == 0);
926
927 q = sc->sge.iqmap[lq - sc->sge.iq_start];
928 if (atomic_cas_uint(&q->state, IQS_IDLE,
929 IQS_BUSY) == IQS_IDLE) {
930 if (service_iq(q, q->qsize / 8) == 0) {
931 (void) atomic_cas_uint(
932 &q->state, IQS_BUSY,
933 IQS_IDLE);
934 } else {
935 STAILQ_INSERT_TAIL(&iql, q,
936 link);
937 }
938 }
939 break;
940
941 default:
942 break;
943 }
944
945 iq_next(iq);
946 if (++ndescs == limit) {
947 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
948 V_CIDXINC(ndescs) |
949 V_INGRESSQID(iq->cntxt_id) |
950 V_SEINTARM(V_QINTR_TIMER_IDX(
951 X_TIMERREG_UPDATE_CIDX)));
952 ndescs = 0;
953
954 if (fl_bufs_used > 0) {
955 ASSERT(iq->flags & IQ_HAS_FL);
956 FL_LOCK(fl);
957 fl->needed += fl_bufs_used;
958 (void) refill_fl(sc, fl, fl->cap / 8);
959 FL_UNLOCK(fl);
960 fl_bufs_used = 0;
961 }
962
963 if (budget != 0)
964 return (EINPROGRESS);
965 }
966 }
967
968 if (STAILQ_EMPTY(&iql) != 0)
969 break;
970
971 /*
972 * Process the head only, and send it to the back of the list if
973 * it's still not done.
974 */
975 q = STAILQ_FIRST(&iql);
976 STAILQ_REMOVE_HEAD(&iql, link);
977 if (service_iq(q, q->qsize / 8) == 0)
978 (void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE);
979 else
980 STAILQ_INSERT_TAIL(&iql, q, link);
981 }
982
983 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
984 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
985
986 if (iq->flags & IQ_HAS_FL) {
987
988 FL_LOCK(fl);
989 fl->needed += fl_bufs_used;
990 starved = refill_fl(sc, fl, fl->cap / 4);
991 FL_UNLOCK(fl);
992 if (starved != 0)
993 add_fl_to_sfl(sc, fl);
994 }
995
996 return (0);
997 }
998
999 #ifdef TCP_OFFLOAD_ENABLE
1000 int
t4_mgmt_tx(struct adapter * sc,mblk_t * m)1001 t4_mgmt_tx(struct adapter *sc, mblk_t *m)
1002 {
1003 return (t4_wrq_tx(sc, &sc->sge.mgmtq, m));
1004 }
1005
1006 /*
1007 * Doesn't fail. Holds on to work requests it can't send right away.
1008 */
1009 int
t4_wrq_tx_locked(struct adapter * sc,struct sge_wrq * wrq,mblk_t * m0)1010 t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, mblk_t *m0)
1011 {
1012 struct sge_eq *eq = &wrq->eq;
1013 struct mblk_pair *wr_list = &wrq->wr_list;
1014 int can_reclaim;
1015 caddr_t dst;
1016 mblk_t *wr, *next;
1017
1018 TXQ_LOCK_ASSERT_OWNED(wrq);
1019 #ifdef TCP_OFFLOAD_ENABLE
1020 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_OFLD ||
1021 (eq->flags & EQ_TYPEMASK) == EQ_CTRL);
1022 #else
1023 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_CTRL);
1024 #endif
1025
1026 if (m0 != NULL) {
1027 if (wr_list->head != NULL)
1028 wr_list->tail->b_next = m0;
1029 else
1030 wr_list->head = m0;
1031 while (m0->b_next)
1032 m0 = m0->b_next;
1033 wr_list->tail = m0;
1034 }
1035
1036 can_reclaim = reclaimable(eq);
1037 eq->cidx += can_reclaim;
1038 eq->avail += can_reclaim;
1039 if (eq->cidx >= eq->cap)
1040 eq->cidx -= eq->cap;
1041
1042 for (wr = wr_list->head; wr; wr = next) {
1043 int ndesc, len = 0;
1044 mblk_t *m;
1045
1046 next = wr->b_next;
1047 wr->b_next = NULL;
1048
1049 for (m = wr; m; m = m->b_cont)
1050 len += MBLKL(m);
1051
1052 ASSERT(len > 0 && (len & 0x7) == 0);
1053 ASSERT(len <= SGE_MAX_WR_LEN);
1054
1055 ndesc = howmany(len, EQ_ESIZE);
1056 if (eq->avail < ndesc) {
1057 wr->b_next = next;
1058 wrq->no_desc++;
1059 break;
1060 }
1061
1062 dst = (void *)&eq->desc[eq->pidx];
1063 for (m = wr; m; m = m->b_cont)
1064 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m));
1065
1066 eq->pidx += ndesc;
1067 eq->avail -= ndesc;
1068 if (eq->pidx >= eq->cap)
1069 eq->pidx -= eq->cap;
1070
1071 eq->pending += ndesc;
1072 if (eq->pending > 16)
1073 ring_tx_db(sc, eq);
1074
1075 wrq->tx_wrs++;
1076 freemsg(wr);
1077
1078 if (eq->avail < 8) {
1079 can_reclaim = reclaimable(eq);
1080 eq->cidx += can_reclaim;
1081 eq->avail += can_reclaim;
1082 if (eq->cidx >= eq->cap)
1083 eq->cidx -= eq->cap;
1084 }
1085 }
1086
1087 if (eq->pending != 0)
1088 ring_tx_db(sc, eq);
1089
1090 if (wr == NULL)
1091 wr_list->head = wr_list->tail = NULL;
1092 else {
1093 wr_list->head = wr;
1094
1095 ASSERT(wr_list->tail->b_next == NULL);
1096 }
1097
1098 return (0);
1099 }
1100 #endif
1101
1102 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */
1103 #define TXPKTS_PKT_HDR ((\
1104 sizeof (struct ulp_txpkt) + \
1105 sizeof (struct ulptx_idata) + \
1106 sizeof (struct cpl_tx_pkt_core)) / 8)
1107
1108 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */
1109 #define TXPKTS_WR_HDR (\
1110 sizeof (struct fw_eth_tx_pkts_wr) / 8 + \
1111 TXPKTS_PKT_HDR)
1112
1113 /* Header of a tx WR, before SGL of first packet (in flits) */
1114 #define TXPKT_WR_HDR ((\
1115 sizeof (struct fw_eth_tx_pkt_wr) + \
1116 sizeof (struct cpl_tx_pkt_core)) / 8)
1117
1118 /* Header of a tx LSO WR, before SGL of first packet (in flits) */
1119 #define TXPKT_LSO_WR_HDR ((\
1120 sizeof (struct fw_eth_tx_pkt_wr) + \
1121 sizeof(struct cpl_tx_pkt_lso_core) + \
1122 sizeof (struct cpl_tx_pkt_core)) / 8)
1123
1124 mblk_t *
t4_eth_tx(void * arg,mblk_t * frame)1125 t4_eth_tx(void *arg, mblk_t *frame)
1126 {
1127 struct sge_txq *txq = (struct sge_txq *) arg;
1128 struct port_info *pi = txq->port;
1129 struct adapter *sc = pi->adapter;
1130 struct sge_eq *eq = &txq->eq;
1131 mblk_t *next_frame;
1132 int rc, coalescing;
1133 struct txpkts txpkts;
1134 struct txinfo txinfo;
1135
1136 txpkts.npkt = 0; /* indicates there's nothing in txpkts */
1137 coalescing = 0;
1138
1139 TXQ_LOCK(txq);
1140 if (eq->avail < 8)
1141 (void) reclaim_tx_descs(txq, 8);
1142 for (; frame; frame = next_frame) {
1143
1144 if (eq->avail < 8)
1145 break;
1146
1147 next_frame = frame->b_next;
1148 frame->b_next = NULL;
1149
1150 if (next_frame != NULL)
1151 coalescing = 1;
1152
1153 rc = get_frame_txinfo(txq, &frame, &txinfo, coalescing);
1154 if (rc != 0) {
1155 if (rc == ENOMEM) {
1156
1157 /* Short of resources, suspend tx */
1158
1159 frame->b_next = next_frame;
1160 break;
1161 }
1162
1163 /*
1164 * Unrecoverable error for this frame, throw it
1165 * away and move on to the next.
1166 */
1167
1168 freemsg(frame);
1169 continue;
1170 }
1171
1172 if (coalescing != 0 &&
1173 add_to_txpkts(txq, &txpkts, frame, &txinfo) == 0) {
1174
1175 /* Successfully absorbed into txpkts */
1176
1177 write_ulp_cpl_sgl(pi, txq, &txpkts, &txinfo);
1178 goto doorbell;
1179 }
1180
1181 /*
1182 * We weren't coalescing to begin with, or current frame could
1183 * not be coalesced (add_to_txpkts flushes txpkts if a frame
1184 * given to it can't be coalesced). Either way there should be
1185 * nothing in txpkts.
1186 */
1187 ASSERT(txpkts.npkt == 0);
1188
1189 /* We're sending out individual frames now */
1190 coalescing = 0;
1191
1192 if (eq->avail < 8)
1193 (void) reclaim_tx_descs(txq, 8);
1194 rc = write_txpkt_wr(pi, txq, frame, &txinfo);
1195 if (rc != 0) {
1196
1197 /* Short of hardware descriptors, suspend tx */
1198
1199 /*
1200 * This is an unlikely but expensive failure. We've
1201 * done all the hard work (DMA bindings etc.) and now we
1202 * can't send out the frame. What's worse, we have to
1203 * spend even more time freeing up everything in txinfo.
1204 */
1205 txq->qfull++;
1206 free_txinfo_resources(txq, &txinfo);
1207
1208 frame->b_next = next_frame;
1209 break;
1210 }
1211
1212 doorbell:
1213 /* Fewer and fewer doorbells as the queue fills up */
1214 if (eq->pending >= (1 << (fls(eq->qsize - eq->avail) / 2))) {
1215 txq->txbytes += txinfo.len;
1216 txq->txpkts++;
1217 ring_tx_db(sc, eq);
1218 }
1219 (void) reclaim_tx_descs(txq, 32);
1220 }
1221
1222 if (txpkts.npkt > 0)
1223 write_txpkts_wr(txq, &txpkts);
1224
1225 /*
1226 * frame not NULL means there was an error but we haven't thrown it
1227 * away. This can happen when we're short of tx descriptors (qfull) or
1228 * maybe even DMA handles (dma_hdl_failed). Either way, a credit flush
1229 * and reclaim will get things going again.
1230 *
1231 * If eq->avail is already 0 we know a credit flush was requested in the
1232 * WR that reduced it to 0 so we don't need another flush (we don't have
1233 * any descriptor for a flush WR anyway, duh).
1234 */
1235 if (frame && eq->avail > 0)
1236 write_txqflush_wr(txq);
1237
1238 if (eq->pending != 0)
1239 ring_tx_db(sc, eq);
1240
1241 (void) reclaim_tx_descs(txq, eq->qsize);
1242 TXQ_UNLOCK(txq);
1243
1244 return (frame);
1245 }
1246
1247 static inline void
init_iq(struct sge_iq * iq,struct adapter * sc,int tmr_idx,int8_t pktc_idx,int qsize,uint8_t esize)1248 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int8_t pktc_idx,
1249 int qsize, uint8_t esize)
1250 {
1251 ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS);
1252 ASSERT(pktc_idx < SGE_NCOUNTERS); /* -ve is ok, means don't use */
1253
1254 iq->flags = 0;
1255 iq->adapter = sc;
1256 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
1257 iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
1258 if (pktc_idx >= 0) {
1259 iq->intr_params |= F_QINTR_CNT_EN;
1260 iq->intr_pktc_idx = pktc_idx;
1261 }
1262 iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */
1263 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */
1264 }
1265
1266 static inline void
init_fl(struct sge_fl * fl,uint16_t qsize)1267 init_fl(struct sge_fl *fl, uint16_t qsize)
1268 {
1269
1270 fl->qsize = qsize;
1271 fl->allocb_fail = 0;
1272 }
1273
1274 static inline void
init_eq(struct adapter * sc,struct sge_eq * eq,uint16_t eqtype,uint16_t qsize,uint8_t tx_chan,uint16_t iqid)1275 init_eq(struct adapter *sc, struct sge_eq *eq, uint16_t eqtype, uint16_t qsize,
1276 uint8_t tx_chan, uint16_t iqid)
1277 {
1278 struct sge *s = &sc->sge;
1279 uint32_t r;
1280
1281 ASSERT(tx_chan < NCHAN);
1282 ASSERT(eqtype <= EQ_TYPEMASK);
1283
1284 if (is_t5(sc->params.chip)) {
1285 r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF);
1286 r >>= S_QUEUESPERPAGEPF0 +
1287 (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf;
1288 s->s_qpp = r & M_QUEUESPERPAGEPF0;
1289 }
1290
1291 eq->flags = eqtype & EQ_TYPEMASK;
1292 eq->tx_chan = tx_chan;
1293 eq->iqid = iqid;
1294 eq->qsize = qsize;
1295 }
1296
1297 /*
1298 * Allocates the ring for an ingress queue and an optional freelist. If the
1299 * freelist is specified it will be allocated and then associated with the
1300 * ingress queue.
1301 *
1302 * Returns errno on failure. Resources allocated up to that point may still be
1303 * allocated. Caller is responsible for cleanup in case this function fails.
1304 *
1305 * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then
1306 * the intr_idx specifies the vector, starting from 0. Otherwise it specifies
1307 * the index of the queue to which its interrupts will be forwarded.
1308 */
1309 static int
alloc_iq_fl(struct port_info * pi,struct sge_iq * iq,struct sge_fl * fl,int intr_idx,int cong)1310 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
1311 int intr_idx, int cong)
1312 {
1313 int rc, i, cntxt_id;
1314 size_t len;
1315 struct fw_iq_cmd c;
1316 struct adapter *sc = iq->adapter;
1317 uint32_t v = 0;
1318
1319 len = iq->qsize * iq->esize;
1320 rc = alloc_desc_ring(sc, len, DDI_DMA_READ, &iq->dhdl, &iq->ahdl,
1321 &iq->ba, (caddr_t *)&iq->desc);
1322 if (rc != 0)
1323 return (rc);
1324
1325 bzero(&c, sizeof (c));
1326 c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
1327 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
1328 V_FW_IQ_CMD_VFN(0));
1329
1330 c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
1331 FW_LEN16(c));
1332
1333 /* Special handling for firmware event queue */
1334 if (iq == &sc->sge.fwq)
1335 v |= F_FW_IQ_CMD_IQASYNCH;
1336
1337 if (iq->flags & IQ_INTR)
1338 ASSERT(intr_idx < sc->intr_count);
1339 else
1340 v |= F_FW_IQ_CMD_IQANDST;
1341 v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx);
1342
1343 c.type_to_iqandstindex = cpu_to_be32(v |
1344 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
1345 V_FW_IQ_CMD_VIID(pi->viid) |
1346 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
1347 c.iqdroprss_to_iqesize = cpu_to_be16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
1348 F_FW_IQ_CMD_IQGTSMODE |
1349 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) |
1350 V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4));
1351 c.iqsize = cpu_to_be16(iq->qsize);
1352 c.iqaddr = cpu_to_be64(iq->ba);
1353 if (cong >= 0)
1354 c.iqns_to_fl0congen = BE_32(F_FW_IQ_CMD_IQFLINTCONGEN |
1355 V_FW_IQ_CMD_IQTYPE(cong ?
1356 FW_IQ_IQTYPE_NIC : FW_IQ_IQTYPE_OFLD));
1357
1358 if (fl != NULL) {
1359 unsigned int chip_ver = CHELSIO_CHIP_VERSION(sc->params.chip);
1360
1361 mutex_init(&fl->lock, NULL, MUTEX_DRIVER,
1362 DDI_INTR_PRI(sc->intr_pri));
1363 fl->flags |= FL_MTX;
1364
1365 len = fl->qsize * RX_FL_ESIZE;
1366 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &fl->dhdl,
1367 &fl->ahdl, &fl->ba, (caddr_t *)&fl->desc);
1368 if (rc != 0)
1369 return (rc);
1370
1371 /* Allocate space for one software descriptor per buffer. */
1372 fl->cap = (fl->qsize - sc->sge.stat_len / RX_FL_ESIZE) * 8;
1373 fl->sdesc = kmem_zalloc(sizeof (struct fl_sdesc) * fl->cap,
1374 KM_SLEEP);
1375 fl->needed = fl->cap;
1376 fl->lowat = roundup(sc->sge.fl_starve_threshold, 8);
1377
1378 c.iqns_to_fl0congen |=
1379 cpu_to_be32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
1380 F_FW_IQ_CMD_FL0PACKEN | F_FW_IQ_CMD_FL0PADEN);
1381 if (cong >= 0) {
1382 c.iqns_to_fl0congen |=
1383 BE_32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) |
1384 F_FW_IQ_CMD_FL0CONGCIF |
1385 F_FW_IQ_CMD_FL0CONGEN);
1386 }
1387
1388 /* In T6, for egress queue type FL there is internal overhead
1389 * of 16B for header going into FLM module. Hence the maximum
1390 * allowed burst size is 448 bytes. For T4/T5, the hardware
1391 * doesn't coalesce fetch requests if more than 64 bytes of
1392 * Free List pointers are provided, so we use a 128-byte Fetch
1393 * Burst Minimum there (T6 implements coalescing so we can use
1394 * the smaller 64-byte value there).
1395 */
1396
1397 c.fl0dcaen_to_fl0cidxfthresh =
1398 cpu_to_be16(V_FW_IQ_CMD_FL0FBMIN(chip_ver <= CHELSIO_T5
1399 ? X_FETCHBURSTMIN_128B
1400 : X_FETCHBURSTMIN_64B) |
1401 V_FW_IQ_CMD_FL0FBMAX(chip_ver <= CHELSIO_T5
1402 ? X_FETCHBURSTMAX_512B
1403 : X_FETCHBURSTMAX_256B));
1404 c.fl0size = cpu_to_be16(fl->qsize);
1405 c.fl0addr = cpu_to_be64(fl->ba);
1406 }
1407
1408 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1409 if (rc != 0) {
1410 cxgb_printf(sc->dip, CE_WARN,
1411 "failed to create ingress queue: %d", rc);
1412 return (rc);
1413 }
1414
1415 iq->cdesc = iq->desc;
1416 iq->cidx = 0;
1417 iq->gen = 1;
1418 iq->intr_next = iq->intr_params;
1419 iq->adapter = sc;
1420 iq->cntxt_id = be16_to_cpu(c.iqid);
1421 iq->abs_id = be16_to_cpu(c.physiqid);
1422 iq->flags |= IQ_ALLOCATED;
1423 mutex_init(&iq->lock, NULL,
1424 MUTEX_DRIVER, DDI_INTR_PRI(DDI_INTR_PRI(sc->intr_pri)));
1425 iq->polling = 0;
1426
1427 cntxt_id = iq->cntxt_id - sc->sge.iq_start;
1428 if (cntxt_id >= sc->sge.iqmap_sz) {
1429 panic("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
1430 cntxt_id, sc->sge.iqmap_sz - 1);
1431 }
1432 sc->sge.iqmap[cntxt_id] = iq;
1433
1434 if (fl != NULL) {
1435 fl->cntxt_id = be16_to_cpu(c.fl0id);
1436 fl->pidx = fl->cidx = 0;
1437 fl->copy_threshold = rx_copy_threshold;
1438
1439 cntxt_id = fl->cntxt_id - sc->sge.eq_start;
1440 if (cntxt_id >= sc->sge.eqmap_sz) {
1441 panic("%s: fl->cntxt_id (%d) more than the max (%d)",
1442 __func__, cntxt_id, sc->sge.eqmap_sz - 1);
1443 }
1444 sc->sge.eqmap[cntxt_id] = (void *)fl;
1445
1446 FL_LOCK(fl);
1447 (void) refill_fl(sc, fl, fl->lowat);
1448 FL_UNLOCK(fl);
1449
1450 iq->flags |= IQ_HAS_FL;
1451 }
1452
1453 if (is_t5(sc->params.chip) && cong >= 0) {
1454 uint32_t param, val;
1455
1456 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1457 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
1458 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id);
1459 if (cong == 0)
1460 val = 1 << 19;
1461 else {
1462 val = 2 << 19;
1463 for (i = 0; i < 4; i++) {
1464 if (cong & (1 << i))
1465 val |= 1 << (i << 2);
1466 }
1467 }
1468
1469 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
1470 if (rc != 0) {
1471 /* report error but carry on */
1472 cxgb_printf(sc->dip, CE_WARN,
1473 "failed to set congestion manager context for "
1474 "ingress queue %d: %d", iq->cntxt_id, rc);
1475 }
1476 }
1477
1478 /* Enable IQ interrupts */
1479 iq->state = IQS_IDLE;
1480 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) |
1481 V_INGRESSQID(iq->cntxt_id));
1482
1483 return (0);
1484 }
1485
1486 static int
free_iq_fl(struct port_info * pi,struct sge_iq * iq,struct sge_fl * fl)1487 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl)
1488 {
1489 int rc;
1490
1491 if (iq != NULL) {
1492 struct adapter *sc = iq->adapter;
1493 dev_info_t *dip;
1494
1495 dip = pi ? pi->dip : sc->dip;
1496 if (iq->flags & IQ_ALLOCATED) {
1497 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0,
1498 FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id,
1499 fl ? fl->cntxt_id : 0xffff, 0xffff);
1500 if (rc != 0) {
1501 cxgb_printf(dip, CE_WARN,
1502 "failed to free queue %p: %d", iq, rc);
1503 return (rc);
1504 }
1505 mutex_destroy(&iq->lock);
1506 iq->flags &= ~IQ_ALLOCATED;
1507 }
1508
1509 if (iq->desc != NULL) {
1510 (void) free_desc_ring(&iq->dhdl, &iq->ahdl);
1511 iq->desc = NULL;
1512 }
1513
1514 bzero(iq, sizeof (*iq));
1515 }
1516
1517 if (fl != NULL) {
1518 if (fl->sdesc != NULL) {
1519 FL_LOCK(fl);
1520 free_fl_bufs(fl);
1521 FL_UNLOCK(fl);
1522
1523 kmem_free(fl->sdesc, sizeof (struct fl_sdesc) *
1524 fl->cap);
1525 fl->sdesc = NULL;
1526 }
1527
1528 if (fl->desc != NULL) {
1529 (void) free_desc_ring(&fl->dhdl, &fl->ahdl);
1530 fl->desc = NULL;
1531 }
1532
1533 if (fl->flags & FL_MTX) {
1534 mutex_destroy(&fl->lock);
1535 fl->flags &= ~FL_MTX;
1536 }
1537
1538 bzero(fl, sizeof (struct sge_fl));
1539 }
1540
1541 return (0);
1542 }
1543
1544 static int
alloc_fwq(struct adapter * sc)1545 alloc_fwq(struct adapter *sc)
1546 {
1547 int rc, intr_idx;
1548 struct sge_iq *fwq = &sc->sge.fwq;
1549
1550 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE);
1551 fwq->flags |= IQ_INTR; /* always */
1552 intr_idx = sc->intr_count > 1 ? 1 : 0;
1553 rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1);
1554 if (rc != 0) {
1555 cxgb_printf(sc->dip, CE_WARN,
1556 "failed to create firmware event queue: %d.", rc);
1557 return (rc);
1558 }
1559
1560 return (0);
1561 }
1562
1563 static int
free_fwq(struct adapter * sc)1564 free_fwq(struct adapter *sc)
1565 {
1566
1567 return (free_iq_fl(NULL, &sc->sge.fwq, NULL));
1568 }
1569
1570 #ifdef TCP_OFFLOAD_ENABLE
1571 static int
alloc_mgmtq(struct adapter * sc)1572 alloc_mgmtq(struct adapter *sc)
1573 {
1574 int rc;
1575 struct sge_wrq *mgmtq = &sc->sge.mgmtq;
1576
1577 init_eq(sc, &mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan,
1578 sc->sge.fwq.cntxt_id);
1579 rc = alloc_wrq(sc, NULL, mgmtq, 0);
1580 if (rc != 0) {
1581 cxgb_printf(sc->dip, CE_WARN,
1582 "failed to create management queue: %d\n", rc);
1583 return (rc);
1584 }
1585
1586 return (0);
1587 }
1588 #endif
1589
1590 static int
alloc_rxq(struct port_info * pi,struct sge_rxq * rxq,int intr_idx,int i)1591 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int i)
1592 {
1593 int rc;
1594
1595 rxq->port = pi;
1596 rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx,
1597 t4_get_tp_ch_map(pi->adapter, pi->tx_chan));
1598 if (rc != 0)
1599 return (rc);
1600
1601 rxq->ksp = setup_rxq_kstats(pi, rxq, i);
1602
1603 return (rc);
1604 }
1605
1606 static int
free_rxq(struct port_info * pi,struct sge_rxq * rxq)1607 free_rxq(struct port_info *pi, struct sge_rxq *rxq)
1608 {
1609 int rc;
1610
1611 if (rxq->ksp != NULL) {
1612 kstat_delete(rxq->ksp);
1613 rxq->ksp = NULL;
1614 }
1615
1616 rc = free_iq_fl(pi, &rxq->iq, &rxq->fl);
1617 if (rc == 0)
1618 bzero(&rxq->fl, sizeof (*rxq) - offsetof(struct sge_rxq, fl));
1619
1620 return (rc);
1621 }
1622
1623 #ifdef TCP_OFFLOAD_ENABLE
1624 static int
alloc_ofld_rxq(struct port_info * pi,struct sge_ofld_rxq * ofld_rxq,int intr_idx)1625 alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq,
1626 int intr_idx)
1627 {
1628 int rc;
1629
1630 rc = alloc_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx,
1631 t4_get_tp_ch_map(pi->adapter, pi->tx_chan));
1632 if (rc != 0)
1633 return (rc);
1634
1635 return (rc);
1636 }
1637
1638 static int
free_ofld_rxq(struct port_info * pi,struct sge_ofld_rxq * ofld_rxq)1639 free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq)
1640 {
1641 int rc;
1642
1643 rc = free_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl);
1644 if (rc == 0)
1645 bzero(&ofld_rxq->fl, sizeof (*ofld_rxq) -
1646 offsetof(struct sge_ofld_rxq, fl));
1647
1648 return (rc);
1649 }
1650 #endif
1651
1652 static int
ctrl_eq_alloc(struct adapter * sc,struct sge_eq * eq)1653 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
1654 {
1655 int rc, cntxt_id;
1656 struct fw_eq_ctrl_cmd c;
1657
1658 bzero(&c, sizeof (c));
1659
1660 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
1661 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) |
1662 V_FW_EQ_CTRL_CMD_VFN(0));
1663 c.alloc_to_len16 = BE_32(F_FW_EQ_CTRL_CMD_ALLOC |
1664 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c));
1665 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* TODO */
1666 c.physeqid_pkd = BE_32(0);
1667 c.fetchszm_to_iqid =
1668 BE_32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1669 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) |
1670 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
1671 c.dcaen_to_eqsize =
1672 BE_32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1673 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1674 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1675 V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize));
1676 c.eqaddr = BE_64(eq->ba);
1677
1678 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1679 if (rc != 0) {
1680 cxgb_printf(sc->dip, CE_WARN,
1681 "failed to create control queue %d: %d", eq->tx_chan, rc);
1682 return (rc);
1683 }
1684 eq->flags |= EQ_ALLOCATED;
1685
1686 eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(BE_32(c.cmpliqid_eqid));
1687 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1688 if (cntxt_id >= sc->sge.eqmap_sz)
1689 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1690 cntxt_id, sc->sge.eqmap_sz - 1);
1691 sc->sge.eqmap[cntxt_id] = eq;
1692
1693 return (rc);
1694 }
1695
1696 static int
eth_eq_alloc(struct adapter * sc,struct port_info * pi,struct sge_eq * eq)1697 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1698 {
1699 int rc, cntxt_id;
1700 struct fw_eq_eth_cmd c;
1701
1702 bzero(&c, sizeof (c));
1703
1704 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
1705 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
1706 V_FW_EQ_ETH_CMD_VFN(0));
1707 c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC |
1708 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
1709 c.autoequiqe_to_viid = BE_32(F_FW_EQ_ETH_CMD_AUTOEQUIQE |
1710 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(pi->viid));
1711 c.fetchszm_to_iqid =
1712 BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1713 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
1714 V_FW_EQ_ETH_CMD_IQID(eq->iqid));
1715 c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1716 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1717 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1718 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize));
1719 c.eqaddr = BE_64(eq->ba);
1720
1721 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1722 if (rc != 0) {
1723 cxgb_printf(pi->dip, CE_WARN,
1724 "failed to create Ethernet egress queue: %d", rc);
1725 return (rc);
1726 }
1727 eq->flags |= EQ_ALLOCATED;
1728
1729 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd));
1730 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1731 if (cntxt_id >= sc->sge.eqmap_sz)
1732 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1733 cntxt_id, sc->sge.eqmap_sz - 1);
1734 sc->sge.eqmap[cntxt_id] = eq;
1735
1736 return (rc);
1737 }
1738
1739 #ifdef TCP_OFFLOAD_ENABLE
1740 static int
ofld_eq_alloc(struct adapter * sc,struct port_info * pi,struct sge_eq * eq)1741 ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1742 {
1743 int rc, cntxt_id;
1744 struct fw_eq_ofld_cmd c;
1745
1746 bzero(&c, sizeof (c));
1747
1748 c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST |
1749 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) |
1750 V_FW_EQ_OFLD_CMD_VFN(0));
1751 c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC |
1752 F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c));
1753 c.fetchszm_to_iqid =
1754 htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1755 V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) |
1756 F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid));
1757 c.dcaen_to_eqsize =
1758 BE_32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1759 V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1760 V_FW_EQ_OFLD_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1761 V_FW_EQ_OFLD_CMD_EQSIZE(eq->qsize));
1762 c.eqaddr = BE_64(eq->ba);
1763
1764 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1765 if (rc != 0) {
1766 cxgb_printf(pi->dip, CE_WARN,
1767 "failed to create egress queue for TCP offload: %d", rc);
1768 return (rc);
1769 }
1770 eq->flags |= EQ_ALLOCATED;
1771
1772 eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(BE_32(c.eqid_pkd));
1773 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1774 if (cntxt_id >= sc->sge.eqmap_sz)
1775 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1776 cntxt_id, sc->sge.eqmap_sz - 1);
1777 sc->sge.eqmap[cntxt_id] = eq;
1778
1779 return (rc);
1780 }
1781 #endif
1782
1783 static int
alloc_eq(struct adapter * sc,struct port_info * pi,struct sge_eq * eq)1784 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1785 {
1786 int rc;
1787 size_t len;
1788
1789 mutex_init(&eq->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(sc->intr_pri));
1790 eq->flags |= EQ_MTX;
1791
1792 len = eq->qsize * EQ_ESIZE;
1793 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &eq->desc_dhdl,
1794 &eq->desc_ahdl, &eq->ba, (caddr_t *)&eq->desc);
1795 if (rc != 0)
1796 return (rc);
1797
1798 eq->cap = eq->qsize - sc->sge.stat_len / EQ_ESIZE;
1799 eq->spg = (void *)&eq->desc[eq->cap];
1800 eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */
1801 eq->pidx = eq->cidx = 0;
1802 eq->doorbells = sc->doorbells;
1803
1804 switch (eq->flags & EQ_TYPEMASK) {
1805 case EQ_CTRL:
1806 rc = ctrl_eq_alloc(sc, eq);
1807 break;
1808
1809 case EQ_ETH:
1810 rc = eth_eq_alloc(sc, pi, eq);
1811 break;
1812
1813 #ifdef TCP_OFFLOAD_ENABLE
1814 case EQ_OFLD:
1815 rc = ofld_eq_alloc(sc, pi, eq);
1816 break;
1817 #endif
1818
1819 default:
1820 panic("%s: invalid eq type %d.", __func__,
1821 eq->flags & EQ_TYPEMASK);
1822 }
1823
1824 if (eq->doorbells &
1825 (DOORBELL_UDB | DOORBELL_UDBWC | DOORBELL_WCWR)) {
1826 uint32_t s_qpp = sc->sge.s_qpp;
1827 uint32_t mask = (1 << s_qpp) - 1;
1828 volatile uint8_t *udb;
1829
1830 udb = (volatile uint8_t *)sc->reg1p + UDBS_DB_OFFSET;
1831 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */
1832 eq->udb_qid = eq->cntxt_id & mask; /* id in page */
1833 if (eq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE)
1834 eq->doorbells &= ~DOORBELL_WCWR;
1835 else {
1836 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */
1837 eq->udb_qid = 0;
1838 }
1839 eq->udb = (volatile void *)udb;
1840 }
1841
1842 if (rc != 0) {
1843 cxgb_printf(sc->dip, CE_WARN,
1844 "failed to allocate egress queue(%d): %d",
1845 eq->flags & EQ_TYPEMASK, rc);
1846 }
1847
1848 return (rc);
1849 }
1850
1851 static int
free_eq(struct adapter * sc,struct sge_eq * eq)1852 free_eq(struct adapter *sc, struct sge_eq *eq)
1853 {
1854 int rc;
1855
1856 if (eq->flags & EQ_ALLOCATED) {
1857 switch (eq->flags & EQ_TYPEMASK) {
1858 case EQ_CTRL:
1859 rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0,
1860 eq->cntxt_id);
1861 break;
1862
1863 case EQ_ETH:
1864 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0,
1865 eq->cntxt_id);
1866 break;
1867
1868 #ifdef TCP_OFFLOAD_ENABLE
1869 case EQ_OFLD:
1870 rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0,
1871 eq->cntxt_id);
1872 break;
1873 #endif
1874
1875 default:
1876 panic("%s: invalid eq type %d.", __func__,
1877 eq->flags & EQ_TYPEMASK);
1878 }
1879 if (rc != 0) {
1880 cxgb_printf(sc->dip, CE_WARN,
1881 "failed to free egress queue (%d): %d",
1882 eq->flags & EQ_TYPEMASK, rc);
1883 return (rc);
1884 }
1885 eq->flags &= ~EQ_ALLOCATED;
1886 }
1887
1888 if (eq->desc != NULL) {
1889 (void) free_desc_ring(&eq->desc_dhdl, &eq->desc_ahdl);
1890 eq->desc = NULL;
1891 }
1892
1893 if (eq->flags & EQ_MTX)
1894 mutex_destroy(&eq->lock);
1895
1896 bzero(eq, sizeof (*eq));
1897 return (0);
1898 }
1899
1900 #ifdef TCP_OFFLOAD_ENABLE
1901 /* ARGSUSED */
1902 static int
alloc_wrq(struct adapter * sc,struct port_info * pi,struct sge_wrq * wrq,int idx)1903 alloc_wrq(struct adapter *sc, struct port_info *pi, struct sge_wrq *wrq,
1904 int idx)
1905 {
1906 int rc;
1907
1908 rc = alloc_eq(sc, pi, &wrq->eq);
1909 if (rc != 0)
1910 return (rc);
1911
1912 wrq->adapter = sc;
1913 wrq->wr_list.head = NULL;
1914 wrq->wr_list.tail = NULL;
1915
1916 /*
1917 * TODO: use idx to figure out what kind of wrq this is and install
1918 * useful kstats for it.
1919 */
1920
1921 return (rc);
1922 }
1923
1924 static int
free_wrq(struct adapter * sc,struct sge_wrq * wrq)1925 free_wrq(struct adapter *sc, struct sge_wrq *wrq)
1926 {
1927 int rc;
1928
1929 rc = free_eq(sc, &wrq->eq);
1930 if (rc != 0)
1931 return (rc);
1932
1933 bzero(wrq, sizeof (*wrq));
1934 return (0);
1935 }
1936 #endif
1937
1938 static int
alloc_txq(struct port_info * pi,struct sge_txq * txq,int idx)1939 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx)
1940 {
1941 int rc, i;
1942 struct adapter *sc = pi->adapter;
1943 struct sge_eq *eq = &txq->eq;
1944
1945 rc = alloc_eq(sc, pi, eq);
1946 if (rc != 0)
1947 return (rc);
1948
1949 txq->port = pi;
1950 txq->sdesc = kmem_zalloc(sizeof (struct tx_sdesc) * eq->cap, KM_SLEEP);
1951 txq->txb_size = eq->qsize * tx_copy_threshold;
1952 rc = alloc_tx_copybuffer(sc, txq->txb_size, &txq->txb_dhdl,
1953 &txq->txb_ahdl, &txq->txb_ba, &txq->txb_va);
1954 if (rc == 0)
1955 txq->txb_avail = txq->txb_size;
1956 else
1957 txq->txb_avail = txq->txb_size = 0;
1958
1959 /*
1960 * TODO: is this too low? Worst case would need around 4 times qsize
1961 * (all tx descriptors filled to the brim with SGLs, with each entry in
1962 * the SGL coming from a distinct DMA handle). Increase tx_dhdl_total
1963 * if you see too many dma_hdl_failed.
1964 */
1965 txq->tx_dhdl_total = eq->qsize * 2;
1966 txq->tx_dhdl = kmem_zalloc(sizeof (ddi_dma_handle_t) *
1967 txq->tx_dhdl_total, KM_SLEEP);
1968 for (i = 0; i < txq->tx_dhdl_total; i++) {
1969 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx,
1970 DDI_DMA_SLEEP, 0, &txq->tx_dhdl[i]);
1971 if (rc != DDI_SUCCESS) {
1972 cxgb_printf(sc->dip, CE_WARN,
1973 "%s: failed to allocate DMA handle (%d)",
1974 __func__, rc);
1975 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
1976 }
1977 txq->tx_dhdl_avail++;
1978 }
1979
1980 txq->ksp = setup_txq_kstats(pi, txq, idx);
1981
1982 return (rc);
1983 }
1984
1985 static int
free_txq(struct port_info * pi,struct sge_txq * txq)1986 free_txq(struct port_info *pi, struct sge_txq *txq)
1987 {
1988 int i;
1989 struct adapter *sc = pi->adapter;
1990 struct sge_eq *eq = &txq->eq;
1991
1992 if (txq->ksp != NULL) {
1993 kstat_delete(txq->ksp);
1994 txq->ksp = NULL;
1995 }
1996
1997 if (txq->txb_va != NULL) {
1998 (void) free_desc_ring(&txq->txb_dhdl, &txq->txb_ahdl);
1999 txq->txb_va = NULL;
2000 }
2001
2002 if (txq->sdesc != NULL) {
2003 struct tx_sdesc *sd;
2004 ddi_dma_handle_t hdl;
2005
2006 TXQ_LOCK(txq);
2007 while (eq->cidx != eq->pidx) {
2008 sd = &txq->sdesc[eq->cidx];
2009
2010 for (i = sd->hdls_used; i; i--) {
2011 hdl = txq->tx_dhdl[txq->tx_dhdl_cidx];
2012 (void) ddi_dma_unbind_handle(hdl);
2013 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total)
2014 txq->tx_dhdl_cidx = 0;
2015 }
2016
2017 ASSERT(sd->m);
2018 freemsgchain(sd->m);
2019
2020 eq->cidx += sd->desc_used;
2021 if (eq->cidx >= eq->cap)
2022 eq->cidx -= eq->cap;
2023
2024 txq->txb_avail += txq->txb_used;
2025 }
2026 ASSERT(txq->tx_dhdl_cidx == txq->tx_dhdl_pidx);
2027 ASSERT(txq->txb_avail == txq->txb_size);
2028 TXQ_UNLOCK(txq);
2029
2030 kmem_free(txq->sdesc, sizeof (struct tx_sdesc) * eq->cap);
2031 txq->sdesc = NULL;
2032 }
2033
2034 if (txq->tx_dhdl != NULL) {
2035 for (i = 0; i < txq->tx_dhdl_total; i++) {
2036 if (txq->tx_dhdl[i] != NULL)
2037 ddi_dma_free_handle(&txq->tx_dhdl[i]);
2038 }
2039 }
2040
2041 (void) free_eq(sc, &txq->eq);
2042
2043 bzero(txq, sizeof (*txq));
2044 return (0);
2045 }
2046
2047 /*
2048 * Allocates a block of contiguous memory for DMA. Can be used to allocate
2049 * memory for descriptor rings or for tx/rx copy buffers.
2050 *
2051 * Caller does not have to clean up anything if this function fails, it cleans
2052 * up after itself.
2053 *
2054 * Caller provides the following:
2055 * len length of the block of memory to allocate.
2056 * flags DDI_DMA_* flags to use (CONSISTENT/STREAMING, READ/WRITE/RDWR)
2057 * acc_attr device access attributes for the allocation.
2058 * dma_attr DMA attributes for the allocation
2059 *
2060 * If the function is successful it fills up this information:
2061 * dma_hdl DMA handle for the allocated memory
2062 * acc_hdl access handle for the allocated memory
2063 * ba bus address of the allocated memory
2064 * va KVA of the allocated memory.
2065 */
2066 static int
alloc_dma_memory(struct adapter * sc,size_t len,int flags,ddi_device_acc_attr_t * acc_attr,ddi_dma_attr_t * dma_attr,ddi_dma_handle_t * dma_hdl,ddi_acc_handle_t * acc_hdl,uint64_t * pba,caddr_t * pva)2067 alloc_dma_memory(struct adapter *sc, size_t len, int flags,
2068 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr,
2069 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
2070 uint64_t *pba, caddr_t *pva)
2071 {
2072 int rc;
2073 ddi_dma_handle_t dhdl;
2074 ddi_acc_handle_t ahdl;
2075 ddi_dma_cookie_t cookie;
2076 uint_t ccount;
2077 caddr_t va;
2078 size_t real_len;
2079
2080 *pva = NULL;
2081
2082 /*
2083 * DMA handle.
2084 */
2085 rc = ddi_dma_alloc_handle(sc->dip, dma_attr, DDI_DMA_SLEEP, 0, &dhdl);
2086 if (rc != DDI_SUCCESS) {
2087 cxgb_printf(sc->dip, CE_WARN,
2088 "failed to allocate DMA handle: %d", rc);
2089
2090 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
2091 }
2092
2093 /*
2094 * Memory suitable for DMA.
2095 */
2096 rc = ddi_dma_mem_alloc(dhdl, len, acc_attr,
2097 flags & DDI_DMA_CONSISTENT ? DDI_DMA_CONSISTENT : DDI_DMA_STREAMING,
2098 DDI_DMA_SLEEP, 0, &va, &real_len, &ahdl);
2099 if (rc != DDI_SUCCESS) {
2100 cxgb_printf(sc->dip, CE_WARN,
2101 "failed to allocate DMA memory: %d", rc);
2102
2103 ddi_dma_free_handle(&dhdl);
2104 return (ENOMEM);
2105 }
2106
2107 if (len != real_len) {
2108 cxgb_printf(sc->dip, CE_WARN,
2109 "%s: len (%u) != real_len (%u)\n", len, real_len);
2110 }
2111
2112 /*
2113 * DMA bindings.
2114 */
2115 rc = ddi_dma_addr_bind_handle(dhdl, NULL, va, real_len, flags, NULL,
2116 NULL, &cookie, &ccount);
2117 if (rc != DDI_DMA_MAPPED) {
2118 cxgb_printf(sc->dip, CE_WARN,
2119 "failed to map DMA memory: %d", rc);
2120
2121 ddi_dma_mem_free(&ahdl);
2122 ddi_dma_free_handle(&dhdl);
2123 return (ENOMEM);
2124 }
2125 if (ccount != 1) {
2126 cxgb_printf(sc->dip, CE_WARN,
2127 "unusable DMA mapping (%d segments)", ccount);
2128 (void) free_desc_ring(&dhdl, &ahdl);
2129 }
2130
2131 bzero(va, real_len);
2132 *dma_hdl = dhdl;
2133 *acc_hdl = ahdl;
2134 *pba = cookie.dmac_laddress;
2135 *pva = va;
2136
2137 return (0);
2138 }
2139
2140 static int
free_dma_memory(ddi_dma_handle_t * dhdl,ddi_acc_handle_t * ahdl)2141 free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl)
2142 {
2143 (void) ddi_dma_unbind_handle(*dhdl);
2144 ddi_dma_mem_free(ahdl);
2145 ddi_dma_free_handle(dhdl);
2146
2147 return (0);
2148 }
2149
2150 static int
alloc_desc_ring(struct adapter * sc,size_t len,int rw,ddi_dma_handle_t * dma_hdl,ddi_acc_handle_t * acc_hdl,uint64_t * pba,caddr_t * pva)2151 alloc_desc_ring(struct adapter *sc, size_t len, int rw,
2152 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
2153 uint64_t *pba, caddr_t *pva)
2154 {
2155 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_desc;
2156 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc;
2157
2158 return (alloc_dma_memory(sc, len, DDI_DMA_CONSISTENT | rw, acc_attr,
2159 dma_attr, dma_hdl, acc_hdl, pba, pva));
2160 }
2161
2162 static int
free_desc_ring(ddi_dma_handle_t * dhdl,ddi_acc_handle_t * ahdl)2163 free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl)
2164 {
2165 return (free_dma_memory(dhdl, ahdl));
2166 }
2167
2168 static int
alloc_tx_copybuffer(struct adapter * sc,size_t len,ddi_dma_handle_t * dma_hdl,ddi_acc_handle_t * acc_hdl,uint64_t * pba,caddr_t * pva)2169 alloc_tx_copybuffer(struct adapter *sc, size_t len,
2170 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
2171 uint64_t *pba, caddr_t *pva)
2172 {
2173 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_tx;
2174 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; /* NOT dma_attr_tx */
2175
2176 return (alloc_dma_memory(sc, len, DDI_DMA_STREAMING | DDI_DMA_WRITE,
2177 acc_attr, dma_attr, dma_hdl, acc_hdl, pba, pva));
2178 }
2179
2180 static inline bool
is_new_response(const struct sge_iq * iq,struct rsp_ctrl ** ctrl)2181 is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl)
2182 {
2183 (void) ddi_dma_sync(iq->dhdl, (uintptr_t)iq->cdesc -
2184 (uintptr_t)iq->desc, iq->esize, DDI_DMA_SYNC_FORKERNEL);
2185
2186 *ctrl = (void *)((uintptr_t)iq->cdesc +
2187 (iq->esize - sizeof (struct rsp_ctrl)));
2188
2189 return ((((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen));
2190 }
2191
2192 static inline void
iq_next(struct sge_iq * iq)2193 iq_next(struct sge_iq *iq)
2194 {
2195 iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize);
2196 if (++iq->cidx == iq->qsize - 1) {
2197 iq->cidx = 0;
2198 iq->gen ^= 1;
2199 iq->cdesc = iq->desc;
2200 }
2201 }
2202
2203 /*
2204 * Fill up the freelist by upto nbufs and maybe ring its doorbell.
2205 *
2206 * Returns non-zero to indicate that it should be added to the list of starving
2207 * freelists.
2208 */
2209 static int
refill_fl(struct adapter * sc,struct sge_fl * fl,int nbufs)2210 refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
2211 {
2212 uint64_t *d = &fl->desc[fl->pidx];
2213 struct fl_sdesc *sd = &fl->sdesc[fl->pidx];
2214
2215 FL_LOCK_ASSERT_OWNED(fl);
2216 ASSERT(nbufs >= 0);
2217
2218 if (nbufs > fl->needed)
2219 nbufs = fl->needed;
2220
2221 while (nbufs--) {
2222 if (sd->rxb != NULL) {
2223 if (sd->rxb->ref_cnt == 1) {
2224 /*
2225 * Buffer is available for recycling. Two ways
2226 * this can happen:
2227 *
2228 * a) All the packets DMA'd into it last time
2229 * around were within the rx_copy_threshold
2230 * and no part of the buffer was ever passed
2231 * up (ref_cnt never went over 1).
2232 *
2233 * b) Packets DMA'd into the buffer were passed
2234 * up but have all been freed by the upper
2235 * layers by now (ref_cnt went over 1 but is
2236 * now back to 1).
2237 *
2238 * Either way the bus address in the descriptor
2239 * ring is already valid.
2240 */
2241 ASSERT(*d == cpu_to_be64(sd->rxb->ba));
2242 d++;
2243 goto recycled;
2244 } else {
2245 /*
2246 * Buffer still in use and we need a
2247 * replacement. But first release our reference
2248 * on the existing buffer.
2249 */
2250 rxbuf_free(sd->rxb);
2251 }
2252 }
2253
2254 sd->rxb = rxbuf_alloc(sc->sge.rxbuf_cache, KM_NOSLEEP, 1);
2255 if (sd->rxb == NULL)
2256 break;
2257 *d++ = cpu_to_be64(sd->rxb->ba);
2258
2259 recycled: fl->pending++;
2260 sd++;
2261 fl->needed--;
2262 if (++fl->pidx == fl->cap) {
2263 fl->pidx = 0;
2264 sd = fl->sdesc;
2265 d = fl->desc;
2266 }
2267 }
2268
2269 if (fl->pending >= 8)
2270 ring_fl_db(sc, fl);
2271
2272 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
2273 }
2274
2275 #ifndef TAILQ_FOREACH_SAFE
2276 #define TAILQ_FOREACH_SAFE(var, head, field, tvar) \
2277 for ((var) = TAILQ_FIRST((head)); \
2278 (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \
2279 (var) = (tvar))
2280 #endif
2281
2282 /*
2283 * Attempt to refill all starving freelists.
2284 */
2285 static void
refill_sfl(void * arg)2286 refill_sfl(void *arg)
2287 {
2288 struct adapter *sc = arg;
2289 struct sge_fl *fl, *fl_temp;
2290
2291 mutex_enter(&sc->sfl_lock);
2292 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) {
2293 FL_LOCK(fl);
2294 (void) refill_fl(sc, fl, 64);
2295 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) {
2296 TAILQ_REMOVE(&sc->sfl, fl, link);
2297 fl->flags &= ~FL_STARVING;
2298 }
2299 FL_UNLOCK(fl);
2300 }
2301
2302 if (!TAILQ_EMPTY(&sc->sfl) != 0)
2303 sc->sfl_timer = timeout(refill_sfl, sc, drv_usectohz(100000));
2304 mutex_exit(&sc->sfl_lock);
2305 }
2306
2307 static void
add_fl_to_sfl(struct adapter * sc,struct sge_fl * fl)2308 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl)
2309 {
2310 mutex_enter(&sc->sfl_lock);
2311 FL_LOCK(fl);
2312 if ((fl->flags & FL_DOOMED) == 0) {
2313 if (TAILQ_EMPTY(&sc->sfl) != 0) {
2314 sc->sfl_timer = timeout(refill_sfl, sc,
2315 drv_usectohz(100000));
2316 }
2317 fl->flags |= FL_STARVING;
2318 TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
2319 }
2320 FL_UNLOCK(fl);
2321 mutex_exit(&sc->sfl_lock);
2322 }
2323
2324 static void
free_fl_bufs(struct sge_fl * fl)2325 free_fl_bufs(struct sge_fl *fl)
2326 {
2327 struct fl_sdesc *sd;
2328 unsigned int i;
2329
2330 FL_LOCK_ASSERT_OWNED(fl);
2331
2332 for (i = 0; i < fl->cap; i++) {
2333 sd = &fl->sdesc[i];
2334
2335 if (sd->rxb != NULL) {
2336 rxbuf_free(sd->rxb);
2337 sd->rxb = NULL;
2338 }
2339 }
2340 }
2341
2342 /*
2343 * Note that fl->cidx and fl->offset are left unchanged in case of failure.
2344 */
2345 static mblk_t *
get_fl_payload(struct adapter * sc,struct sge_fl * fl,uint32_t len_newbuf,int * fl_bufs_used)2346 get_fl_payload(struct adapter *sc, struct sge_fl *fl,
2347 uint32_t len_newbuf, int *fl_bufs_used)
2348 {
2349 struct mblk_pair frame = {0};
2350 struct rxbuf *rxb;
2351 mblk_t *m = NULL;
2352 uint_t nbuf = 0, len, copy, n;
2353 uint32_t cidx, offset, rcidx, roffset;
2354
2355 /*
2356 * The SGE won't pack a new frame into the current buffer if the entire
2357 * payload doesn't fit in the remaining space. Move on to the next buf
2358 * in that case.
2359 */
2360 rcidx = fl->cidx;
2361 roffset = fl->offset;
2362 if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) {
2363 fl->offset = 0;
2364 if (++fl->cidx == fl->cap)
2365 fl->cidx = 0;
2366 nbuf++;
2367 }
2368 cidx = fl->cidx;
2369 offset = fl->offset;
2370
2371 len = G_RSPD_LEN(len_newbuf); /* pktshift + payload length */
2372 copy = (len <= fl->copy_threshold);
2373 if (copy != 0) {
2374 frame.head = m = allocb(len, BPRI_HI);
2375 if (m == NULL) {
2376 fl->allocb_fail++;
2377 cmn_err(CE_WARN,"%s: mbuf allocation failure "
2378 "count = %llu", __func__,
2379 (unsigned long long)fl->allocb_fail);
2380 fl->cidx = rcidx;
2381 fl->offset = roffset;
2382 return (NULL);
2383 }
2384 }
2385
2386 while (len) {
2387 rxb = fl->sdesc[cidx].rxb;
2388 n = min(len, rxb->buf_size - offset);
2389
2390 (void) ddi_dma_sync(rxb->dhdl, offset, n,
2391 DDI_DMA_SYNC_FORKERNEL);
2392
2393 if (copy != 0)
2394 bcopy(rxb->va + offset, m->b_wptr, n);
2395 else {
2396 m = desballoc((unsigned char *)rxb->va + offset, n,
2397 BPRI_HI, &rxb->freefunc);
2398 if (m == NULL) {
2399 fl->allocb_fail++;
2400 cmn_err(CE_WARN,
2401 "%s: mbuf allocation failure "
2402 "count = %llu", __func__,
2403 (unsigned long long)fl->allocb_fail);
2404 if (frame.head)
2405 freemsgchain(frame.head);
2406 fl->cidx = rcidx;
2407 fl->offset = roffset;
2408 return (NULL);
2409 }
2410 atomic_inc_uint(&rxb->ref_cnt);
2411 if (frame.head != NULL)
2412 frame.tail->b_cont = m;
2413 else
2414 frame.head = m;
2415 frame.tail = m;
2416 }
2417 m->b_wptr += n;
2418 len -= n;
2419 offset += roundup(n, sc->sge.fl_align);
2420 ASSERT(offset <= rxb->buf_size);
2421 if (offset == rxb->buf_size) {
2422 offset = 0;
2423 if (++cidx == fl->cap)
2424 cidx = 0;
2425 nbuf++;
2426 }
2427 }
2428
2429 fl->cidx = cidx;
2430 fl->offset = offset;
2431 (*fl_bufs_used) += nbuf;
2432
2433 ASSERT(frame.head != NULL);
2434 return (frame.head);
2435 }
2436
2437 /*
2438 * We'll do immediate data tx for non-LSO, but only when not coalescing. We're
2439 * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes
2440 * of immediate data.
2441 */
2442 #define IMM_LEN ( \
2443 2 * EQ_ESIZE \
2444 - sizeof (struct fw_eth_tx_pkt_wr) \
2445 - sizeof (struct cpl_tx_pkt_core))
2446
2447 /*
2448 * Returns non-zero on failure, no need to cleanup anything in that case.
2449 *
2450 * Note 1: We always try to pull up the mblk if required and return E2BIG only
2451 * if this fails.
2452 *
2453 * Note 2: We'll also pullup incoming mblk if HW_LSO is set and the first mblk
2454 * does not have the TCP header in it.
2455 */
2456 static int
get_frame_txinfo(struct sge_txq * txq,mblk_t ** fp,struct txinfo * txinfo,int sgl_only)2457 get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, struct txinfo *txinfo,
2458 int sgl_only)
2459 {
2460 uint32_t flags = 0, len, n;
2461 mblk_t *m = *fp;
2462 int rc;
2463
2464 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb and dma_hdls */
2465
2466 mac_hcksum_get(m, NULL, NULL, NULL, NULL, &flags);
2467 txinfo->flags = flags;
2468
2469 mac_lso_get(m, &txinfo->mss, &flags);
2470 txinfo->flags |= flags;
2471
2472 if (flags & HW_LSO)
2473 sgl_only = 1; /* Do not allow immediate data with LSO */
2474
2475 start: txinfo->nsegs = 0;
2476 txinfo->hdls_used = 0;
2477 txinfo->txb_used = 0;
2478 txinfo->len = 0;
2479
2480 /* total length and a rough estimate of # of segments */
2481 n = 0;
2482 for (; m; m = m->b_cont) {
2483 len = MBLKL(m);
2484 n += (len / PAGE_SIZE) + 1;
2485 txinfo->len += len;
2486 }
2487 m = *fp;
2488
2489 if (n >= TX_SGL_SEGS || (flags & HW_LSO && MBLKL(m) < 50)) {
2490 txq->pullup_early++;
2491 m = msgpullup(*fp, -1);
2492 if (m == NULL) {
2493 txq->pullup_failed++;
2494 return (E2BIG); /* (*fp) left as it was */
2495 }
2496 freemsg(*fp);
2497 *fp = m;
2498 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags);
2499 }
2500
2501 if (txinfo->len <= IMM_LEN && !sgl_only)
2502 return (0); /* nsegs = 0 tells caller to use imm. tx */
2503
2504 if (txinfo->len <= txq->copy_threshold &&
2505 copy_into_txb(txq, m, txinfo->len, txinfo) == 0)
2506 goto done;
2507
2508 for (; m; m = m->b_cont) {
2509
2510 len = MBLKL(m);
2511
2512 /* Use tx copy buffer if this mblk is small enough */
2513 if (len <= txq->copy_threshold &&
2514 copy_into_txb(txq, m, len, txinfo) == 0)
2515 continue;
2516
2517 /* Add DMA bindings for this mblk to the SGL */
2518 rc = add_mblk(txq, txinfo, m, len);
2519
2520 if (rc == E2BIG ||
2521 (txinfo->nsegs == TX_SGL_SEGS && m->b_cont)) {
2522
2523 txq->pullup_late++;
2524 m = msgpullup(*fp, -1);
2525 if (m != NULL) {
2526 free_txinfo_resources(txq, txinfo);
2527 freemsg(*fp);
2528 *fp = m;
2529 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags);
2530 goto start;
2531 }
2532
2533 txq->pullup_failed++;
2534 rc = E2BIG;
2535 }
2536
2537 if (rc != 0) {
2538 free_txinfo_resources(txq, txinfo);
2539 return (rc);
2540 }
2541 }
2542
2543 ASSERT(txinfo->nsegs > 0 && txinfo->nsegs <= TX_SGL_SEGS);
2544
2545 done:
2546
2547 /*
2548 * Store the # of flits required to hold this frame's SGL in nflits. An
2549 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by
2550 * multiple (len0 + len1, addr0, addr1) tuples. If addr1 is not used
2551 * then len1 must be set to 0.
2552 */
2553 n = txinfo->nsegs - 1;
2554 txinfo->nflits = (3 * n) / 2 + (n & 1) + 2;
2555 if (n & 1)
2556 txinfo->sgl.sge[n / 2].len[1] = cpu_to_be32(0);
2557
2558 txinfo->sgl.cmd_nsge = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_DSGL) |
2559 V_ULPTX_NSGE(txinfo->nsegs));
2560
2561 return (0);
2562 }
2563
2564 static inline int
fits_in_txb(struct sge_txq * txq,int len,int * waste)2565 fits_in_txb(struct sge_txq *txq, int len, int *waste)
2566 {
2567 if (txq->txb_avail < len)
2568 return (0);
2569
2570 if (txq->txb_next + len <= txq->txb_size) {
2571 *waste = 0;
2572 return (1);
2573 }
2574
2575 *waste = txq->txb_size - txq->txb_next;
2576
2577 return (txq->txb_avail - *waste < len ? 0 : 1);
2578 }
2579
2580 #define TXB_CHUNK 64
2581
2582 /*
2583 * Copies the specified # of bytes into txq's tx copy buffer and updates txinfo
2584 * and txq to indicate resources used. Caller has to make sure that those many
2585 * bytes are available in the mblk chain (b_cont linked).
2586 */
2587 static inline int
copy_into_txb(struct sge_txq * txq,mblk_t * m,int len,struct txinfo * txinfo)2588 copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, struct txinfo *txinfo)
2589 {
2590 int waste, n;
2591
2592 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb */
2593
2594 if (!fits_in_txb(txq, len, &waste)) {
2595 txq->txb_full++;
2596 return (ENOMEM);
2597 }
2598
2599 if (waste != 0) {
2600 ASSERT((waste & (TXB_CHUNK - 1)) == 0);
2601 txinfo->txb_used += waste;
2602 txq->txb_avail -= waste;
2603 txq->txb_next = 0;
2604 }
2605
2606 for (n = 0; n < len; m = m->b_cont) {
2607 bcopy(m->b_rptr, txq->txb_va + txq->txb_next + n, MBLKL(m));
2608 n += MBLKL(m);
2609 }
2610
2611 add_seg(txinfo, txq->txb_ba + txq->txb_next, len);
2612
2613 n = roundup(len, TXB_CHUNK);
2614 txinfo->txb_used += n;
2615 txq->txb_avail -= n;
2616 txq->txb_next += n;
2617 ASSERT(txq->txb_next <= txq->txb_size);
2618 if (txq->txb_next == txq->txb_size)
2619 txq->txb_next = 0;
2620
2621 return (0);
2622 }
2623
2624 static inline void
add_seg(struct txinfo * txinfo,uint64_t ba,uint32_t len)2625 add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len)
2626 {
2627 ASSERT(txinfo->nsegs < TX_SGL_SEGS); /* must have room */
2628
2629 if (txinfo->nsegs != 0) {
2630 int idx = txinfo->nsegs - 1;
2631 txinfo->sgl.sge[idx / 2].len[idx & 1] = cpu_to_be32(len);
2632 txinfo->sgl.sge[idx / 2].addr[idx & 1] = cpu_to_be64(ba);
2633 } else {
2634 txinfo->sgl.len0 = cpu_to_be32(len);
2635 txinfo->sgl.addr0 = cpu_to_be64(ba);
2636 }
2637 txinfo->nsegs++;
2638 }
2639
2640 /*
2641 * This function cleans up any partially allocated resources when it fails so
2642 * there's nothing for the caller to clean up in that case.
2643 *
2644 * EIO indicates permanent failure. Caller should drop the frame containing
2645 * this mblk and continue.
2646 *
2647 * E2BIG indicates that the SGL length for this mblk exceeds the hardware
2648 * limit. Caller should pull up the frame before trying to send it out.
2649 * (This error means our pullup_early heuristic did not work for this frame)
2650 *
2651 * ENOMEM indicates a temporary shortage of resources (DMA handles, other DMA
2652 * resources, etc.). Caller should suspend the tx queue and wait for reclaim to
2653 * free up resources.
2654 */
2655 static inline int
add_mblk(struct sge_txq * txq,struct txinfo * txinfo,mblk_t * m,int len)2656 add_mblk(struct sge_txq *txq, struct txinfo *txinfo, mblk_t *m, int len)
2657 {
2658 ddi_dma_handle_t dhdl;
2659 ddi_dma_cookie_t cookie;
2660 uint_t ccount = 0;
2661 int rc;
2662
2663 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate dhdls */
2664
2665 if (txq->tx_dhdl_avail == 0) {
2666 txq->dma_hdl_failed++;
2667 return (ENOMEM);
2668 }
2669
2670 dhdl = txq->tx_dhdl[txq->tx_dhdl_pidx];
2671 rc = ddi_dma_addr_bind_handle(dhdl, NULL, (caddr_t)m->b_rptr, len,
2672 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie,
2673 &ccount);
2674 if (rc != DDI_DMA_MAPPED) {
2675 txq->dma_map_failed++;
2676
2677 ASSERT(rc != DDI_DMA_INUSE && rc != DDI_DMA_PARTIAL_MAP);
2678
2679 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EIO);
2680 }
2681
2682 if (ccount + txinfo->nsegs > TX_SGL_SEGS) {
2683 (void) ddi_dma_unbind_handle(dhdl);
2684 return (E2BIG);
2685 }
2686
2687 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size);
2688 while (--ccount) {
2689 ddi_dma_nextcookie(dhdl, &cookie);
2690 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size);
2691 }
2692
2693 if (++txq->tx_dhdl_pidx == txq->tx_dhdl_total)
2694 txq->tx_dhdl_pidx = 0;
2695 txq->tx_dhdl_avail--;
2696 txinfo->hdls_used++;
2697
2698 return (0);
2699 }
2700
2701 /*
2702 * Releases all the txq resources used up in the specified txinfo.
2703 */
2704 static void
free_txinfo_resources(struct sge_txq * txq,struct txinfo * txinfo)2705 free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo)
2706 {
2707 int n;
2708
2709 TXQ_LOCK_ASSERT_OWNED(txq); /* dhdls, txb */
2710
2711 n = txinfo->txb_used;
2712 if (n > 0) {
2713 txq->txb_avail += n;
2714 if (n <= txq->txb_next)
2715 txq->txb_next -= n;
2716 else {
2717 n -= txq->txb_next;
2718 txq->txb_next = txq->txb_size - n;
2719 }
2720 }
2721
2722 for (n = txinfo->hdls_used; n > 0; n--) {
2723 if (txq->tx_dhdl_pidx > 0)
2724 txq->tx_dhdl_pidx--;
2725 else
2726 txq->tx_dhdl_pidx = txq->tx_dhdl_total - 1;
2727 txq->tx_dhdl_avail++;
2728 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_pidx]);
2729 }
2730 }
2731
2732 /*
2733 * Returns 0 to indicate that m has been accepted into a coalesced tx work
2734 * request. It has either been folded into txpkts or txpkts was flushed and m
2735 * has started a new coalesced work request (as the first frame in a fresh
2736 * txpkts).
2737 *
2738 * Returns non-zero to indicate a failure - caller is responsible for
2739 * transmitting m, if there was anything in txpkts it has been flushed.
2740 */
2741 static int
add_to_txpkts(struct sge_txq * txq,struct txpkts * txpkts,mblk_t * m,struct txinfo * txinfo)2742 add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m,
2743 struct txinfo *txinfo)
2744 {
2745 struct sge_eq *eq = &txq->eq;
2746 int can_coalesce;
2747 struct tx_sdesc *txsd;
2748 uint8_t flits;
2749
2750 TXQ_LOCK_ASSERT_OWNED(txq);
2751
2752 if (txpkts->npkt > 0) {
2753 flits = TXPKTS_PKT_HDR + txinfo->nflits;
2754 can_coalesce = (txinfo->flags & HW_LSO) == 0 &&
2755 txpkts->nflits + flits <= TX_WR_FLITS &&
2756 txpkts->nflits + flits <= eq->avail * 8 &&
2757 txpkts->plen + txinfo->len < 65536;
2758
2759 if (can_coalesce != 0) {
2760 txpkts->tail->b_next = m;
2761 txpkts->tail = m;
2762 txpkts->npkt++;
2763 txpkts->nflits += flits;
2764 txpkts->plen += txinfo->len;
2765
2766 txsd = &txq->sdesc[eq->pidx];
2767 txsd->txb_used += txinfo->txb_used;
2768 txsd->hdls_used += txinfo->hdls_used;
2769
2770 return (0);
2771 }
2772
2773 /*
2774 * Couldn't coalesce m into txpkts. The first order of business
2775 * is to send txpkts on its way. Then we'll revisit m.
2776 */
2777 write_txpkts_wr(txq, txpkts);
2778 }
2779
2780 /*
2781 * Check if we can start a new coalesced tx work request with m as
2782 * the first packet in it.
2783 */
2784
2785 ASSERT(txpkts->npkt == 0);
2786 ASSERT(txinfo->len < 65536);
2787
2788 flits = TXPKTS_WR_HDR + txinfo->nflits;
2789 can_coalesce = (txinfo->flags & HW_LSO) == 0 &&
2790 flits <= eq->avail * 8 && flits <= TX_WR_FLITS;
2791
2792 if (can_coalesce == 0)
2793 return (EINVAL);
2794
2795 /*
2796 * Start a fresh coalesced tx WR with m as the first frame in it.
2797 */
2798 txpkts->tail = m;
2799 txpkts->npkt = 1;
2800 txpkts->nflits = flits;
2801 txpkts->flitp = &eq->desc[eq->pidx].flit[2];
2802 txpkts->plen = txinfo->len;
2803
2804 txsd = &txq->sdesc[eq->pidx];
2805 txsd->m = m;
2806 txsd->txb_used = txinfo->txb_used;
2807 txsd->hdls_used = txinfo->hdls_used;
2808
2809 return (0);
2810 }
2811
2812 /*
2813 * Note that write_txpkts_wr can never run out of hardware descriptors (but
2814 * write_txpkt_wr can). add_to_txpkts ensures that a frame is accepted for
2815 * coalescing only if sufficient hardware descriptors are available.
2816 */
2817 static void
write_txpkts_wr(struct sge_txq * txq,struct txpkts * txpkts)2818 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts)
2819 {
2820 struct sge_eq *eq = &txq->eq;
2821 struct fw_eth_tx_pkts_wr *wr;
2822 struct tx_sdesc *txsd;
2823 uint32_t ctrl;
2824 uint16_t ndesc;
2825
2826 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */
2827
2828 ndesc = howmany(txpkts->nflits, 8);
2829
2830 wr = (void *)&eq->desc[eq->pidx];
2831 wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR) |
2832 V_FW_WR_IMMDLEN(0)); /* immdlen does not matter in this WR */
2833 ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2));
2834 if (eq->avail == ndesc)
2835 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
2836 wr->equiq_to_len16 = cpu_to_be32(ctrl);
2837 wr->plen = cpu_to_be16(txpkts->plen);
2838 wr->npkt = txpkts->npkt;
2839 wr->r3 = wr->type = 0;
2840
2841 /* Everything else already written */
2842
2843 txsd = &txq->sdesc[eq->pidx];
2844 txsd->desc_used = ndesc;
2845
2846 txq->txb_used += txsd->txb_used / TXB_CHUNK;
2847 txq->hdl_used += txsd->hdls_used;
2848
2849 ASSERT(eq->avail >= ndesc);
2850
2851 eq->pending += ndesc;
2852 eq->avail -= ndesc;
2853 eq->pidx += ndesc;
2854 if (eq->pidx >= eq->cap)
2855 eq->pidx -= eq->cap;
2856
2857 txq->txpkts_pkts += txpkts->npkt;
2858 txq->txpkts_wrs++;
2859 txpkts->npkt = 0; /* emptied */
2860 }
2861
2862 static int
write_txpkt_wr(struct port_info * pi,struct sge_txq * txq,mblk_t * m,struct txinfo * txinfo)2863 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m,
2864 struct txinfo *txinfo)
2865 {
2866 struct sge_eq *eq = &txq->eq;
2867 struct fw_eth_tx_pkt_wr *wr;
2868 struct cpl_tx_pkt_core *cpl;
2869 uint32_t ctrl; /* used in many unrelated places */
2870 uint64_t ctrl1;
2871 int nflits, ndesc;
2872 struct tx_sdesc *txsd;
2873 caddr_t dst;
2874
2875 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */
2876
2877 /*
2878 * Do we have enough flits to send this frame out?
2879 */
2880 ctrl = sizeof (struct cpl_tx_pkt_core);
2881 if (txinfo->flags & HW_LSO) {
2882 nflits = TXPKT_LSO_WR_HDR;
2883 ctrl += sizeof(struct cpl_tx_pkt_lso_core);
2884 } else
2885 nflits = TXPKT_WR_HDR;
2886 if (txinfo->nsegs > 0)
2887 nflits += txinfo->nflits;
2888 else {
2889 nflits += howmany(txinfo->len, 8);
2890 ctrl += txinfo->len;
2891 }
2892 ndesc = howmany(nflits, 8);
2893 if (ndesc > eq->avail)
2894 return (ENOMEM);
2895
2896 /* Firmware work request header */
2897 wr = (void *)&eq->desc[eq->pidx];
2898 wr->op_immdlen = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
2899 V_FW_WR_IMMDLEN(ctrl));
2900 ctrl = V_FW_WR_LEN16(howmany(nflits, 2));
2901 if (eq->avail == ndesc)
2902 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
2903 wr->equiq_to_len16 = cpu_to_be32(ctrl);
2904 wr->r3 = 0;
2905
2906 if (txinfo->flags & HW_LSO) {
2907 uint16_t etype;
2908 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
2909 char *p = (void *)m->b_rptr;
2910 ctrl = V_LSO_OPCODE((u32)CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE |
2911 F_LSO_LAST_SLICE;
2912
2913 etype = ntohs(((struct ether_header *)p)->ether_type);
2914 if (etype == ETHERTYPE_VLAN) {
2915 ctrl |= V_LSO_ETHHDR_LEN(1);
2916 etype = ntohs(((struct ether_vlan_header *)p)->ether_type);
2917 p += sizeof (struct ether_vlan_header);
2918 } else {
2919 p += sizeof (struct ether_header);
2920 }
2921
2922 switch (etype) {
2923 case ETHERTYPE_IP:
2924 ctrl |= V_LSO_IPHDR_LEN(IPH_HDR_LENGTH(p) / 4);
2925 p += IPH_HDR_LENGTH(p);
2926 break;
2927 case ETHERTYPE_IPV6:
2928 ctrl |= F_LSO_IPV6;
2929 ctrl |= V_LSO_IPHDR_LEN(sizeof (ip6_t) / 4);
2930 p += sizeof (ip6_t);
2931 default:
2932 break;
2933 }
2934
2935 ctrl |= V_LSO_TCPHDR_LEN(TCP_HDR_LENGTH((tcph_t *)p) / 4);
2936
2937 lso->lso_ctrl = cpu_to_be32(ctrl);
2938 lso->ipid_ofst = cpu_to_be16(0);
2939 lso->mss = cpu_to_be16(txinfo->mss);
2940 lso->seqno_offset = cpu_to_be32(0);
2941 if (is_t4(pi->adapter->params.chip))
2942 lso->len = cpu_to_be32(txinfo->len);
2943 else
2944 lso->len = cpu_to_be32(V_LSO_T5_XFER_SIZE(txinfo->len));
2945
2946 cpl = (void *)(lso + 1);
2947
2948 txq->tso_wrs++;
2949 } else
2950 cpl = (void *)(wr + 1);
2951
2952 /* Checksum offload */
2953 ctrl1 = 0;
2954 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM))
2955 ctrl1 |= F_TXPKT_IPCSUM_DIS;
2956 if (!(txinfo->flags & HCK_FULLCKSUM))
2957 ctrl1 |= F_TXPKT_L4CSUM_DIS;
2958 if (ctrl1 == 0)
2959 txq->txcsum++; /* some hardware assistance provided */
2960
2961 /* CPL header */
2962 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) |
2963 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
2964 cpl->pack = 0;
2965 cpl->len = cpu_to_be16(txinfo->len);
2966 cpl->ctrl1 = cpu_to_be64(ctrl1);
2967
2968 /* Software descriptor */
2969 txsd = &txq->sdesc[eq->pidx];
2970 txsd->m = m;
2971 txsd->txb_used = txinfo->txb_used;
2972 txsd->hdls_used = txinfo->hdls_used;
2973 /* LINTED: E_ASSIGN_NARROW_CONV */
2974 txsd->desc_used = ndesc;
2975
2976 txq->txb_used += txinfo->txb_used / TXB_CHUNK;
2977 txq->hdl_used += txinfo->hdls_used;
2978
2979 eq->pending += ndesc;
2980 eq->avail -= ndesc;
2981 eq->pidx += ndesc;
2982 if (eq->pidx >= eq->cap)
2983 eq->pidx -= eq->cap;
2984
2985 /* SGL */
2986 dst = (void *)(cpl + 1);
2987 if (txinfo->nsegs > 0) {
2988 txq->sgl_wrs++;
2989 copy_to_txd(eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8);
2990
2991 /* Need to zero-pad to a 16 byte boundary if not on one */
2992 if ((uintptr_t)dst & 0xf)
2993 /* LINTED: E_BAD_PTR_CAST_ALIGN */
2994 *(uint64_t *)dst = 0;
2995
2996 } else {
2997 txq->imm_wrs++;
2998 #ifdef DEBUG
2999 ctrl = txinfo->len;
3000 #endif
3001 for (; m; m = m->b_cont) {
3002 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m));
3003 #ifdef DEBUG
3004 ctrl -= MBLKL(m);
3005 #endif
3006 }
3007 ASSERT(ctrl == 0);
3008 }
3009
3010 txq->txpkt_wrs++;
3011 return (0);
3012 }
3013
3014 static inline void
write_ulp_cpl_sgl(struct port_info * pi,struct sge_txq * txq,struct txpkts * txpkts,struct txinfo * txinfo)3015 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
3016 struct txpkts *txpkts, struct txinfo *txinfo)
3017 {
3018 struct ulp_txpkt *ulpmc;
3019 struct ulptx_idata *ulpsc;
3020 struct cpl_tx_pkt_core *cpl;
3021 uintptr_t flitp, start, end;
3022 uint64_t ctrl;
3023 caddr_t dst;
3024
3025 ASSERT(txpkts->npkt > 0);
3026
3027 start = (uintptr_t)txq->eq.desc;
3028 end = (uintptr_t)txq->eq.spg;
3029
3030 /* Checksum offload */
3031 ctrl = 0;
3032 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM))
3033 ctrl |= F_TXPKT_IPCSUM_DIS;
3034 if (!(txinfo->flags & HCK_FULLCKSUM))
3035 ctrl |= F_TXPKT_L4CSUM_DIS;
3036 if (ctrl == 0)
3037 txq->txcsum++; /* some hardware assistance provided */
3038
3039 /*
3040 * The previous packet's SGL must have ended at a 16 byte boundary (this
3041 * is required by the firmware/hardware). It follows that flitp cannot
3042 * wrap around between the ULPTX master command and ULPTX subcommand (8
3043 * bytes each), and that it can not wrap around in the middle of the
3044 * cpl_tx_pkt_core either.
3045 */
3046 flitp = (uintptr_t)txpkts->flitp;
3047 ASSERT((flitp & 0xf) == 0);
3048
3049 /* ULP master command */
3050 ulpmc = (void *)flitp;
3051 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
3052 ulpmc->len = htonl(howmany(sizeof (*ulpmc) + sizeof (*ulpsc) +
3053 sizeof (*cpl) + 8 * txinfo->nflits, 16));
3054
3055 /* ULP subcommand */
3056 ulpsc = (void *)(ulpmc + 1);
3057 ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) |
3058 F_ULP_TX_SC_MORE);
3059 ulpsc->len = cpu_to_be32(sizeof (struct cpl_tx_pkt_core));
3060
3061 flitp += sizeof (*ulpmc) + sizeof (*ulpsc);
3062 if (flitp == end)
3063 flitp = start;
3064
3065 /* CPL_TX_PKT */
3066 cpl = (void *)flitp;
3067 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) |
3068 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
3069 cpl->pack = 0;
3070 cpl->len = cpu_to_be16(txinfo->len);
3071 cpl->ctrl1 = cpu_to_be64(ctrl);
3072
3073 flitp += sizeof (*cpl);
3074 if (flitp == end)
3075 flitp = start;
3076
3077 /* SGL for this frame */
3078 dst = (caddr_t)flitp;
3079 copy_to_txd(&txq->eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8);
3080 flitp = (uintptr_t)dst;
3081
3082 /* Zero pad and advance to a 16 byte boundary if not already at one. */
3083 if (flitp & 0xf) {
3084
3085 /* no matter what, flitp should be on an 8 byte boundary */
3086 ASSERT((flitp & 0x7) == 0);
3087
3088 *(uint64_t *)flitp = 0;
3089 flitp += sizeof (uint64_t);
3090 txpkts->nflits++;
3091 }
3092
3093 if (flitp == end)
3094 flitp = start;
3095
3096 txpkts->flitp = (void *)flitp;
3097 }
3098
3099 static inline void
copy_to_txd(struct sge_eq * eq,caddr_t from,caddr_t * to,int len)3100 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
3101 {
3102 if ((uintptr_t)(*to) + len <= (uintptr_t)eq->spg) {
3103 bcopy(from, *to, len);
3104 (*to) += len;
3105 } else {
3106 int portion = (uintptr_t)eq->spg - (uintptr_t)(*to);
3107
3108 bcopy(from, *to, portion);
3109 from += portion;
3110 portion = len - portion; /* remaining */
3111 bcopy(from, (void *)eq->desc, portion);
3112 (*to) = (caddr_t)eq->desc + portion;
3113 }
3114 }
3115
3116 static inline void
ring_tx_db(struct adapter * sc,struct sge_eq * eq)3117 ring_tx_db(struct adapter *sc, struct sge_eq *eq)
3118 {
3119 int val, db_mode;
3120 u_int db = eq->doorbells;
3121
3122 if (eq->pending > 1)
3123 db &= ~DOORBELL_WCWR;
3124
3125 if (eq->pending > eq->pidx) {
3126 int offset = eq->cap - (eq->pending - eq->pidx);
3127
3128 /* pidx has wrapped around since last doorbell */
3129
3130 (void) ddi_dma_sync(eq->desc_dhdl,
3131 offset * sizeof (struct tx_desc), 0,
3132 DDI_DMA_SYNC_FORDEV);
3133 (void) ddi_dma_sync(eq->desc_dhdl,
3134 0, eq->pidx * sizeof (struct tx_desc),
3135 DDI_DMA_SYNC_FORDEV);
3136 } else if (eq->pending > 0) {
3137 (void) ddi_dma_sync(eq->desc_dhdl,
3138 (eq->pidx - eq->pending) * sizeof (struct tx_desc),
3139 eq->pending * sizeof (struct tx_desc),
3140 DDI_DMA_SYNC_FORDEV);
3141 }
3142
3143 membar_producer();
3144
3145 if (is_t4(sc->params.chip))
3146 val = V_PIDX(eq->pending);
3147 else
3148 val = V_PIDX_T5(eq->pending);
3149
3150 db_mode = (1 << (ffs(db) - 1));
3151 switch (db_mode) {
3152 case DOORBELL_UDB:
3153 *eq->udb = LE_32(V_QID(eq->udb_qid) | val);
3154 break;
3155
3156 case DOORBELL_WCWR:
3157 {
3158 volatile uint64_t *dst, *src;
3159 int i;
3160 /*
3161 * Queues whose 128B doorbell segment fits in
3162 * the page do not use relative qid
3163 * (udb_qid is always 0). Only queues with
3164 * doorbell segments can do WCWR.
3165 */
3166 ASSERT(eq->udb_qid == 0 && eq->pending == 1);
3167
3168 dst = (volatile void *)((uintptr_t)eq->udb +
3169 UDBS_WR_OFFSET - UDBS_DB_OFFSET);
3170 i = eq->pidx ? eq->pidx - 1 : eq->cap - 1;
3171 src = (void *)&eq->desc[i];
3172 while (src != (void *)&eq->desc[i + 1])
3173 *dst++ = *src++;
3174 membar_producer();
3175 break;
3176 }
3177
3178 case DOORBELL_UDBWC:
3179 *eq->udb = LE_32(V_QID(eq->udb_qid) | val);
3180 membar_producer();
3181 break;
3182
3183 case DOORBELL_KDB:
3184 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
3185 V_QID(eq->cntxt_id) | val);
3186 break;
3187 }
3188
3189 eq->pending = 0;
3190 }
3191
3192 static int
reclaim_tx_descs(struct sge_txq * txq,int howmany)3193 reclaim_tx_descs(struct sge_txq *txq, int howmany)
3194 {
3195 struct tx_sdesc *txsd;
3196 uint_t cidx, can_reclaim, reclaimed, txb_freed, hdls_freed;
3197 struct sge_eq *eq = &txq->eq;
3198
3199 EQ_LOCK_ASSERT_OWNED(eq);
3200
3201 cidx = eq->spg->cidx; /* stable snapshot */
3202 cidx = be16_to_cpu(cidx);
3203
3204 if (cidx >= eq->cidx)
3205 can_reclaim = cidx - eq->cidx;
3206 else
3207 can_reclaim = cidx + eq->cap - eq->cidx;
3208
3209 if (can_reclaim == 0)
3210 return (0);
3211
3212 txb_freed = hdls_freed = reclaimed = 0;
3213 do {
3214 int ndesc;
3215
3216 txsd = &txq->sdesc[eq->cidx];
3217 ndesc = txsd->desc_used;
3218
3219 /* Firmware doesn't return "partial" credits. */
3220 ASSERT(can_reclaim >= ndesc);
3221
3222 /*
3223 * We always keep mblk around, even for immediate data. If mblk
3224 * is NULL, this has to be the software descriptor for a credit
3225 * flush work request.
3226 */
3227 if (txsd->m != NULL)
3228 freemsgchain(txsd->m);
3229 #ifdef DEBUG
3230 else {
3231 ASSERT(txsd->txb_used == 0);
3232 ASSERT(txsd->hdls_used == 0);
3233 ASSERT(ndesc == 1);
3234 }
3235 #endif
3236
3237 txb_freed += txsd->txb_used;
3238 hdls_freed += txsd->hdls_used;
3239 reclaimed += ndesc;
3240
3241 eq->cidx += ndesc;
3242 if (eq->cidx >= eq->cap)
3243 eq->cidx -= eq->cap;
3244
3245 can_reclaim -= ndesc;
3246
3247 } while (can_reclaim && reclaimed < howmany);
3248
3249 eq->avail += reclaimed;
3250 ASSERT(eq->avail < eq->cap); /* avail tops out at (cap - 1) */
3251
3252 txq->txb_avail += txb_freed;
3253
3254 txq->tx_dhdl_avail += hdls_freed;
3255 ASSERT(txq->tx_dhdl_avail <= txq->tx_dhdl_total);
3256 for (; hdls_freed; hdls_freed--) {
3257 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_cidx]);
3258 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total)
3259 txq->tx_dhdl_cidx = 0;
3260 }
3261
3262 return (reclaimed);
3263 }
3264
3265 static void
write_txqflush_wr(struct sge_txq * txq)3266 write_txqflush_wr(struct sge_txq *txq)
3267 {
3268 struct sge_eq *eq = &txq->eq;
3269 struct fw_eq_flush_wr *wr;
3270 struct tx_sdesc *txsd;
3271
3272 EQ_LOCK_ASSERT_OWNED(eq);
3273 ASSERT(eq->avail > 0);
3274
3275 wr = (void *)&eq->desc[eq->pidx];
3276 bzero(wr, sizeof (*wr));
3277 wr->opcode = FW_EQ_FLUSH_WR;
3278 wr->equiq_to_len16 = cpu_to_be32(V_FW_WR_LEN16(sizeof (*wr) / 16) |
3279 F_FW_WR_EQUEQ | F_FW_WR_EQUIQ);
3280
3281 txsd = &txq->sdesc[eq->pidx];
3282 txsd->m = NULL;
3283 txsd->txb_used = 0;
3284 txsd->hdls_used = 0;
3285 txsd->desc_used = 1;
3286
3287 eq->pending++;
3288 eq->avail--;
3289 if (++eq->pidx == eq->cap)
3290 eq->pidx = 0;
3291 }
3292
3293 static int
t4_eth_rx(struct sge_iq * iq,const struct rss_header * rss,mblk_t * m)3294 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
3295 {
3296 bool csum_ok;
3297 uint16_t err_vec;
3298 struct sge_rxq *rxq = (void *)iq;
3299 struct mblk_pair chain = {0};
3300 struct adapter *sc = iq->adapter;
3301 const struct cpl_rx_pkt *cpl = (const void *)(rss + 1);
3302
3303 iq->intr_next = iq->intr_params;
3304
3305 m->b_rptr += sc->sge.pktshift;
3306
3307 /* Compressed error vector is enabled for T6 only */
3308 if (sc->params.tp.rx_pkt_encap)
3309 /* It is enabled only in T6 config file */
3310 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec));
3311 else
3312 err_vec = ntohs(cpl->err_vec);
3313
3314 csum_ok = cpl->csum_calc && !err_vec;
3315 /* TODO: what about cpl->ip_frag? */
3316 if (csum_ok && !cpl->ip_frag) {
3317 mac_hcksum_set(m, 0, 0, 0, 0xffff,
3318 HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
3319 HCK_IPV4_HDRCKSUM_OK);
3320 rxq->rxcsum++;
3321 }
3322
3323 /* Add to the chain that we'll send up */
3324 if (chain.head != NULL)
3325 chain.tail->b_next = m;
3326 else
3327 chain.head = m;
3328 chain.tail = m;
3329
3330 t4_mac_rx(rxq->port, rxq, chain.head);
3331
3332 rxq->rxpkts++;
3333 rxq->rxbytes += be16_to_cpu(cpl->len);
3334 return (0);
3335 }
3336
3337 #define FL_HW_IDX(idx) ((idx) >> 3)
3338
3339 static inline void
ring_fl_db(struct adapter * sc,struct sge_fl * fl)3340 ring_fl_db(struct adapter *sc, struct sge_fl *fl)
3341 {
3342 int desc_start, desc_last, ndesc;
3343 uint32_t v = sc->params.arch.sge_fl_db ;
3344
3345 ndesc = FL_HW_IDX(fl->pending);
3346
3347 /* Hold back one credit if pidx = cidx */
3348 if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx))
3349 ndesc--;
3350
3351 /*
3352 * There are chances of ndesc modified above (to avoid pidx = cidx).
3353 * If there is nothing to post, return.
3354 */
3355 if (ndesc <= 0)
3356 return;
3357
3358 desc_last = FL_HW_IDX(fl->pidx);
3359
3360 if (fl->pidx < fl->pending) {
3361 /* There was a wrap */
3362 desc_start = FL_HW_IDX(fl->pidx + fl->cap - fl->pending);
3363
3364 /* From desc_start to the end of list */
3365 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 0,
3366 DDI_DMA_SYNC_FORDEV);
3367
3368 /* From start of list to the desc_last */
3369 if (desc_last != 0)
3370 (void) ddi_dma_sync(fl->dhdl, 0, desc_last *
3371 RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3372 } else {
3373 /* There was no wrap, sync from start_desc to last_desc */
3374 desc_start = FL_HW_IDX(fl->pidx - fl->pending);
3375 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE,
3376 ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3377 }
3378
3379 if (is_t4(sc->params.chip))
3380 v |= V_PIDX(ndesc);
3381 else
3382 v |= V_PIDX_T5(ndesc);
3383 v |= V_QID(fl->cntxt_id) | V_PIDX(ndesc);
3384
3385 membar_producer();
3386
3387 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v);
3388
3389 /*
3390 * Update pending count:
3391 * Deduct the number of descriptors posted
3392 */
3393 fl->pending -= ndesc * 8;
3394 }
3395
3396 static void
tx_reclaim_task(void * arg)3397 tx_reclaim_task(void *arg)
3398 {
3399 struct sge_txq *txq = arg;
3400
3401 TXQ_LOCK(txq);
3402 reclaim_tx_descs(txq, txq->eq.qsize);
3403 TXQ_UNLOCK(txq);
3404 }
3405
3406 /* ARGSUSED */
3407 static int
handle_sge_egr_update(struct sge_iq * iq,const struct rss_header * rss,mblk_t * m)3408 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
3409 mblk_t *m)
3410 {
3411 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
3412 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
3413 struct adapter *sc = iq->adapter;
3414 struct sge *s = &sc->sge;
3415 struct sge_eq *eq;
3416 struct sge_txq *txq;
3417
3418 txq = (void *)s->eqmap[qid - s->eq_start];
3419 eq = &txq->eq;
3420 txq->qflush++;
3421 t4_mac_tx_update(txq->port, txq);
3422
3423 ddi_taskq_dispatch(sc->tq[eq->tx_chan], tx_reclaim_task,
3424 (void *)txq, DDI_NOSLEEP);
3425
3426 return (0);
3427 }
3428
3429 static int
handle_fw_rpl(struct sge_iq * iq,const struct rss_header * rss,mblk_t * m)3430 handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
3431 {
3432 struct adapter *sc = iq->adapter;
3433 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
3434
3435 ASSERT(m == NULL);
3436
3437 if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) {
3438 const struct rss_header *rss2;
3439
3440 rss2 = (const struct rss_header *)&cpl->data[0];
3441 return (sc->cpl_handler[rss2->opcode](iq, rss2, m));
3442 }
3443 return (sc->fw_msg_handler[cpl->type](sc, &cpl->data[0]));
3444 }
3445
3446 int
t4_alloc_tx_maps(struct adapter * sc,struct tx_maps * txmaps,int count,int flags)3447 t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count,
3448 int flags)
3449 {
3450 int i, rc;
3451
3452 txmaps->map_total = count;
3453 txmaps->map_avail = txmaps->map_cidx = txmaps->map_pidx = 0;
3454
3455 txmaps->map = kmem_zalloc(sizeof (ddi_dma_handle_t) *
3456 txmaps->map_total, flags);
3457
3458 for (i = 0; i < count; i++) {
3459 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx,
3460 DDI_DMA_SLEEP, 0, &txmaps->map[i]);
3461 if (rc != DDI_SUCCESS) {
3462 cxgb_printf(sc->dip, CE_WARN,
3463 "%s: failed to allocate DMA handle (%d)",
3464 __func__, rc);
3465 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
3466 }
3467 txmaps->map_avail++;
3468 }
3469
3470 return (0);
3471 }
3472
3473 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG)
3474 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR)
3475 #define KS_U_SET(x, y) kstatp->x.value.ul = (y)
3476 #define KS_U_FROM(x, y) kstatp->x.value.ul = (y)->x
3477 #define KS_C_SET(x, ...) \
3478 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__)
3479
3480 /*
3481 * cxgbe:X:config
3482 */
3483 struct cxgbe_port_config_kstats {
3484 kstat_named_t idx;
3485 kstat_named_t nrxq;
3486 kstat_named_t ntxq;
3487 kstat_named_t first_rxq;
3488 kstat_named_t first_txq;
3489 kstat_named_t controller;
3490 kstat_named_t factory_mac_address;
3491 };
3492
3493 /*
3494 * cxgbe:X:info
3495 */
3496 struct cxgbe_port_info_kstats {
3497 kstat_named_t transceiver;
3498 kstat_named_t rx_ovflow0;
3499 kstat_named_t rx_ovflow1;
3500 kstat_named_t rx_ovflow2;
3501 kstat_named_t rx_ovflow3;
3502 kstat_named_t rx_trunc0;
3503 kstat_named_t rx_trunc1;
3504 kstat_named_t rx_trunc2;
3505 kstat_named_t rx_trunc3;
3506 kstat_named_t tx_pause;
3507 kstat_named_t rx_pause;
3508 };
3509
3510 static kstat_t *
setup_port_config_kstats(struct port_info * pi)3511 setup_port_config_kstats(struct port_info *pi)
3512 {
3513 kstat_t *ksp;
3514 struct cxgbe_port_config_kstats *kstatp;
3515 int ndata;
3516 dev_info_t *pdip = ddi_get_parent(pi->dip);
3517 uint8_t *ma = &pi->hw_addr[0];
3518
3519 ndata = sizeof (struct cxgbe_port_config_kstats) /
3520 sizeof (kstat_named_t);
3521
3522 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "config",
3523 "net", KSTAT_TYPE_NAMED, ndata, 0);
3524 if (ksp == NULL) {
3525 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats.");
3526 return (NULL);
3527 }
3528
3529 kstatp = (struct cxgbe_port_config_kstats *)ksp->ks_data;
3530
3531 KS_UINIT(idx);
3532 KS_UINIT(nrxq);
3533 KS_UINIT(ntxq);
3534 KS_UINIT(first_rxq);
3535 KS_UINIT(first_txq);
3536 KS_CINIT(controller);
3537 KS_CINIT(factory_mac_address);
3538
3539 KS_U_SET(idx, pi->port_id);
3540 KS_U_SET(nrxq, pi->nrxq);
3541 KS_U_SET(ntxq, pi->ntxq);
3542 KS_U_SET(first_rxq, pi->first_rxq);
3543 KS_U_SET(first_txq, pi->first_txq);
3544 KS_C_SET(controller, "%s%d", ddi_driver_name(pdip),
3545 ddi_get_instance(pdip));
3546 KS_C_SET(factory_mac_address, "%02X%02X%02X%02X%02X%02X",
3547 ma[0], ma[1], ma[2], ma[3], ma[4], ma[5]);
3548
3549 /* Do NOT set ksp->ks_update. These kstats do not change. */
3550
3551 /* Install the kstat */
3552 ksp->ks_private = (void *)pi;
3553 kstat_install(ksp);
3554
3555 return (ksp);
3556 }
3557
3558 static kstat_t *
setup_port_info_kstats(struct port_info * pi)3559 setup_port_info_kstats(struct port_info *pi)
3560 {
3561 kstat_t *ksp;
3562 struct cxgbe_port_info_kstats *kstatp;
3563 int ndata;
3564
3565 ndata = sizeof (struct cxgbe_port_info_kstats) / sizeof (kstat_named_t);
3566
3567 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "info",
3568 "net", KSTAT_TYPE_NAMED, ndata, 0);
3569 if (ksp == NULL) {
3570 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats.");
3571 return (NULL);
3572 }
3573
3574 kstatp = (struct cxgbe_port_info_kstats *)ksp->ks_data;
3575
3576 KS_CINIT(transceiver);
3577 KS_UINIT(rx_ovflow0);
3578 KS_UINIT(rx_ovflow1);
3579 KS_UINIT(rx_ovflow2);
3580 KS_UINIT(rx_ovflow3);
3581 KS_UINIT(rx_trunc0);
3582 KS_UINIT(rx_trunc1);
3583 KS_UINIT(rx_trunc2);
3584 KS_UINIT(rx_trunc3);
3585 KS_UINIT(tx_pause);
3586 KS_UINIT(rx_pause);
3587
3588 /* Install the kstat */
3589 ksp->ks_update = update_port_info_kstats;
3590 ksp->ks_private = (void *)pi;
3591 kstat_install(ksp);
3592
3593 return (ksp);
3594 }
3595
3596 static int
update_port_info_kstats(kstat_t * ksp,int rw)3597 update_port_info_kstats(kstat_t *ksp, int rw)
3598 {
3599 struct cxgbe_port_info_kstats *kstatp =
3600 (struct cxgbe_port_info_kstats *)ksp->ks_data;
3601 struct port_info *pi = ksp->ks_private;
3602 static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX",
3603 "active TWINAX", "LRM" };
3604 uint32_t bgmap;
3605
3606 if (rw == KSTAT_WRITE)
3607 return (0);
3608
3609 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
3610 KS_C_SET(transceiver, "unplugged");
3611 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
3612 KS_C_SET(transceiver, "unknown");
3613 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
3614 KS_C_SET(transceiver, "unsupported");
3615 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str))
3616 KS_C_SET(transceiver, "%s", mod_str[pi->mod_type]);
3617 else
3618 KS_C_SET(transceiver, "type %d", pi->mod_type);
3619
3620 #define GET_STAT(name) t4_read_reg64(pi->adapter, \
3621 PORT_REG(pi->port_id, A_MPS_PORT_STAT_##name##_L))
3622 #define GET_STAT_COM(name) t4_read_reg64(pi->adapter, \
3623 A_MPS_STAT_##name##_L)
3624
3625 bgmap = G_NUMPORTS(t4_read_reg(pi->adapter, A_MPS_CMN_CTL));
3626 if (bgmap == 0)
3627 bgmap = (pi->port_id == 0) ? 0xf : 0;
3628 else if (bgmap == 1)
3629 bgmap = (pi->port_id < 2) ? (3 << (2 * pi->port_id)) : 0;
3630 else
3631 bgmap = 1;
3632
3633 KS_U_SET(rx_ovflow0, (bgmap & 1) ?
3634 GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0);
3635 KS_U_SET(rx_ovflow1, (bgmap & 2) ?
3636 GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0);
3637 KS_U_SET(rx_ovflow2, (bgmap & 4) ?
3638 GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0);
3639 KS_U_SET(rx_ovflow3, (bgmap & 8) ?
3640 GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0);
3641 KS_U_SET(rx_trunc0, (bgmap & 1) ?
3642 GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0);
3643 KS_U_SET(rx_trunc1, (bgmap & 2) ?
3644 GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0);
3645 KS_U_SET(rx_trunc2, (bgmap & 4) ?
3646 GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0);
3647 KS_U_SET(rx_trunc3, (bgmap & 8) ?
3648 GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0);
3649
3650 KS_U_SET(tx_pause, GET_STAT(TX_PORT_PAUSE));
3651 KS_U_SET(rx_pause, GET_STAT(RX_PORT_PAUSE));
3652
3653 return (0);
3654
3655 }
3656
3657 /*
3658 * cxgbe:X:rxqY
3659 */
3660 struct rxq_kstats {
3661 kstat_named_t rxcsum;
3662 kstat_named_t rxpkts;
3663 kstat_named_t rxbytes;
3664 kstat_named_t nomem;
3665 };
3666
3667 static kstat_t *
setup_rxq_kstats(struct port_info * pi,struct sge_rxq * rxq,int idx)3668 setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, int idx)
3669 {
3670 struct kstat *ksp;
3671 struct rxq_kstats *kstatp;
3672 int ndata;
3673 char str[16];
3674
3675 ndata = sizeof (struct rxq_kstats) / sizeof (kstat_named_t);
3676 (void) snprintf(str, sizeof (str), "rxq%u", idx);
3677
3678 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "rxq",
3679 KSTAT_TYPE_NAMED, ndata, 0);
3680 if (ksp == NULL) {
3681 cxgb_printf(pi->dip, CE_WARN,
3682 "%s: failed to initialize rxq kstats for queue %d.",
3683 __func__, idx);
3684 return (NULL);
3685 }
3686
3687 kstatp = (struct rxq_kstats *)ksp->ks_data;
3688
3689 KS_UINIT(rxcsum);
3690 KS_UINIT(rxpkts);
3691 KS_UINIT(rxbytes);
3692 KS_UINIT(nomem);
3693
3694 ksp->ks_update = update_rxq_kstats;
3695 ksp->ks_private = (void *)rxq;
3696 kstat_install(ksp);
3697
3698 return (ksp);
3699 }
3700
3701 static int
update_rxq_kstats(kstat_t * ksp,int rw)3702 update_rxq_kstats(kstat_t *ksp, int rw)
3703 {
3704 struct rxq_kstats *kstatp = (struct rxq_kstats *)ksp->ks_data;
3705 struct sge_rxq *rxq = ksp->ks_private;
3706
3707 if (rw == KSTAT_WRITE)
3708 return (0);
3709
3710 KS_U_FROM(rxcsum, rxq);
3711 KS_U_FROM(rxpkts, rxq);
3712 KS_U_FROM(rxbytes, rxq);
3713 KS_U_FROM(nomem, rxq);
3714
3715 return (0);
3716 }
3717
3718 /*
3719 * cxgbe:X:txqY
3720 */
3721 struct txq_kstats {
3722 kstat_named_t txcsum;
3723 kstat_named_t tso_wrs;
3724 kstat_named_t imm_wrs;
3725 kstat_named_t sgl_wrs;
3726 kstat_named_t txpkt_wrs;
3727 kstat_named_t txpkts_wrs;
3728 kstat_named_t txpkts_pkts;
3729 kstat_named_t txb_used;
3730 kstat_named_t hdl_used;
3731 kstat_named_t txb_full;
3732 kstat_named_t dma_hdl_failed;
3733 kstat_named_t dma_map_failed;
3734 kstat_named_t qfull;
3735 kstat_named_t qflush;
3736 kstat_named_t pullup_early;
3737 kstat_named_t pullup_late;
3738 kstat_named_t pullup_failed;
3739 };
3740
3741 static kstat_t *
setup_txq_kstats(struct port_info * pi,struct sge_txq * txq,int idx)3742 setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, int idx)
3743 {
3744 struct kstat *ksp;
3745 struct txq_kstats *kstatp;
3746 int ndata;
3747 char str[16];
3748
3749 ndata = sizeof (struct txq_kstats) / sizeof (kstat_named_t);
3750 (void) snprintf(str, sizeof (str), "txq%u", idx);
3751
3752 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "txq",
3753 KSTAT_TYPE_NAMED, ndata, 0);
3754 if (ksp == NULL) {
3755 cxgb_printf(pi->dip, CE_WARN,
3756 "%s: failed to initialize txq kstats for queue %d.",
3757 __func__, idx);
3758 return (NULL);
3759 }
3760
3761 kstatp = (struct txq_kstats *)ksp->ks_data;
3762
3763 KS_UINIT(txcsum);
3764 KS_UINIT(tso_wrs);
3765 KS_UINIT(imm_wrs);
3766 KS_UINIT(sgl_wrs);
3767 KS_UINIT(txpkt_wrs);
3768 KS_UINIT(txpkts_wrs);
3769 KS_UINIT(txpkts_pkts);
3770 KS_UINIT(txb_used);
3771 KS_UINIT(hdl_used);
3772 KS_UINIT(txb_full);
3773 KS_UINIT(dma_hdl_failed);
3774 KS_UINIT(dma_map_failed);
3775 KS_UINIT(qfull);
3776 KS_UINIT(qflush);
3777 KS_UINIT(pullup_early);
3778 KS_UINIT(pullup_late);
3779 KS_UINIT(pullup_failed);
3780
3781 ksp->ks_update = update_txq_kstats;
3782 ksp->ks_private = (void *)txq;
3783 kstat_install(ksp);
3784
3785 return (ksp);
3786 }
3787
3788 static int
update_txq_kstats(kstat_t * ksp,int rw)3789 update_txq_kstats(kstat_t *ksp, int rw)
3790 {
3791 struct txq_kstats *kstatp = (struct txq_kstats *)ksp->ks_data;
3792 struct sge_txq *txq = ksp->ks_private;
3793
3794 if (rw == KSTAT_WRITE)
3795 return (0);
3796
3797 KS_U_FROM(txcsum, txq);
3798 KS_U_FROM(tso_wrs, txq);
3799 KS_U_FROM(imm_wrs, txq);
3800 KS_U_FROM(sgl_wrs, txq);
3801 KS_U_FROM(txpkt_wrs, txq);
3802 KS_U_FROM(txpkts_wrs, txq);
3803 KS_U_FROM(txpkts_pkts, txq);
3804 KS_U_FROM(txb_used, txq);
3805 KS_U_FROM(hdl_used, txq);
3806 KS_U_FROM(txb_full, txq);
3807 KS_U_FROM(dma_hdl_failed, txq);
3808 KS_U_FROM(dma_map_failed, txq);
3809 KS_U_FROM(qfull, txq);
3810 KS_U_FROM(qflush, txq);
3811 KS_U_FROM(pullup_early, txq);
3812 KS_U_FROM(pullup_late, txq);
3813 KS_U_FROM(pullup_failed, txq);
3814
3815 return (0);
3816 }
3817