1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * This file is part of the Chelsio T4 support code.
14 *
15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved.
16 *
17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
20 * release for licensing terms and conditions.
21 */
22
23 #include <sys/ddi.h>
24 #include <sys/sunddi.h>
25 #include <sys/sunndi.h>
26 #include <sys/atomic.h>
27 #include <sys/dlpi.h>
28 #include <sys/pattr.h>
29 #include <sys/strsubr.h>
30 #include <sys/stream.h>
31 #include <sys/strsun.h>
32 #include <inet/ip.h>
33 #include <inet/tcp.h>
34
35 #include "version.h"
36 #include "common/common.h"
37 #include "common/t4_msg.h"
38 #include "common/t4_regs.h"
39 #include "common/t4_regs_values.h"
40
41 /* TODO: Tune. */
42 int rx_buf_size = 8192;
43 int tx_copy_threshold = 256;
44 uint16_t rx_copy_threshold = 256;
45
46 enum {
47 SPG_SIZE = 64, /* Size of status page */
48 FL_ALIGN = CACHE_LINE, /* packet buffer alignment in FL buffers */
49 FL_PKTSHIFT = 2 /* payload is at this offset in packet buffer */
50 };
51
52 /* Used to track coalesced tx work request */
53 struct txpkts {
54 mblk_t *tail; /* head is in the software descriptor */
55 uint64_t *flitp; /* ptr to flit where next pkt should start */
56 uint8_t npkt; /* # of packets in this work request */
57 uint8_t nflits; /* # of flits used by this work request */
58 uint16_t plen; /* total payload (sum of all packets) */
59 };
60
61 /* All information needed to tx a frame */
62 struct txinfo {
63 uint32_t len; /* Total length of frame */
64 uint32_t flags; /* Checksum and LSO flags */
65 uint32_t mss; /* MSS for LSO */
66 uint8_t nsegs; /* # of segments in the SGL, 0 means imm. tx */
67 uint8_t nflits; /* # of flits needed for the SGL */
68 uint8_t hdls_used; /* # of DMA handles used */
69 uint32_t txb_used; /* txb_space used */
70 struct ulptx_sgl sgl __attribute__((aligned(8)));
71 struct ulptx_sge_pair reserved[TX_SGL_SEGS / 2];
72 };
73
74 static int service_iq(struct sge_iq *iq, int budget);
75 static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx,
76 int8_t pktc_idx, int qsize, uint8_t esize);
77 static inline void init_fl(struct sge_fl *fl, uint16_t qsize);
78 static inline void init_eq(struct sge_eq *eq, uint16_t eqtype, uint16_t qsize,
79 uint8_t tx_chan, uint16_t iqid);
80 static int alloc_iq_fl(struct port_info *pi, struct sge_iq *iq,
81 struct sge_fl *fl, int intr_idx, int cong);
82 static int free_iq_fl(struct port_info *pi, struct sge_iq *iq,
83 struct sge_fl *fl);
84 static int alloc_fwq(struct adapter *sc);
85 static int free_fwq(struct adapter *sc);
86 static int alloc_mgmtq(struct adapter *sc);
87 static int alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx,
88 int i);
89 static int free_rxq(struct port_info *pi, struct sge_rxq *rxq);
90 #ifndef TCP_OFFLOAD_DISABLE
91 static int alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq,
92 int intr_idx);
93 static int free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq);
94 #endif
95 static int ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq);
96 static int eth_eq_alloc(struct adapter *sc, struct port_info *pi,
97 struct sge_eq *eq);
98 #ifndef TCP_OFFLOAD_DISABLE
99 static int ofld_eq_alloc(struct adapter *sc, struct port_info *pi,
100 struct sge_eq *eq);
101 #endif
102 static int alloc_eq(struct adapter *sc, struct port_info *pi,
103 struct sge_eq *eq);
104 static int free_eq(struct adapter *sc, struct sge_eq *eq);
105 static int alloc_wrq(struct adapter *sc, struct port_info *pi,
106 struct sge_wrq *wrq, int idx);
107 static int free_wrq(struct adapter *sc, struct sge_wrq *wrq);
108 static int alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx);
109 static int free_txq(struct port_info *pi, struct sge_txq *txq);
110 static int alloc_dma_memory(struct adapter *sc, size_t len, int flags,
111 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr,
112 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
113 caddr_t *pva);
114 static int free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl);
115 static int alloc_desc_ring(struct adapter *sc, size_t len, int rw,
116 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
117 caddr_t *pva);
118 static int free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl);
119 static int alloc_tx_copybuffer(struct adapter *sc, size_t len,
120 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
121 caddr_t *pva);
122 static inline bool is_new_response(const struct sge_iq *iq,
123 struct rsp_ctrl **ctrl);
124 static inline void iq_next(struct sge_iq *iq);
125 static int refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs);
126 static void refill_sfl(void *arg);
127 static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl);
128 static void free_fl_bufs(struct sge_fl *fl);
129 static mblk_t *get_fl_payload(struct sge_fl *fl, uint32_t len_newbuf,
130 int *fl_bufs_used);
131 static int get_frame_txinfo(struct sge_txq *txq, mblk_t **fp,
132 struct txinfo *txinfo, int sgl_only);
133 static inline int fits_in_txb(struct sge_txq *txq, int len, int *waste);
134 static inline int copy_into_txb(struct sge_txq *txq, mblk_t *m, int len,
135 struct txinfo *txinfo);
136 static inline void add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len);
137 static inline int add_mblk(struct sge_txq *txq, struct txinfo *txinfo,
138 mblk_t *m, int len);
139 static void free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo);
140 static int add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m,
141 struct txinfo *txinfo);
142 static void write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts);
143 static int write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m,
144 struct txinfo *txinfo);
145 static inline void write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
146 struct txpkts *txpkts, struct txinfo *txinfo);
147 static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to,
148 int len);
149 static inline void ring_tx_db(struct adapter *sc, struct sge_eq *eq);
150 static int reclaim_tx_descs(struct sge_txq *txq, int howmany);
151 static void write_txqflush_wr(struct sge_txq *txq);
152 static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss,
153 mblk_t *m);
154 static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl);
155 static kstat_t *setup_port_config_kstats(struct port_info *pi);
156 static kstat_t *setup_port_info_kstats(struct port_info *pi);
157 static kstat_t *setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq,
158 int idx);
159 static int update_rxq_kstats(kstat_t *ksp, int rw);
160 static int update_port_info_kstats(kstat_t *ksp, int rw);
161 static kstat_t *setup_txq_kstats(struct port_info *pi, struct sge_txq *txq,
162 int idx);
163 static int update_txq_kstats(kstat_t *ksp, int rw);
164 static int handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss,
165 mblk_t *m);
166
167 static inline int
reclaimable(struct sge_eq * eq)168 reclaimable(struct sge_eq *eq)
169 {
170 unsigned int cidx;
171
172 cidx = eq->spg->cidx; /* stable snapshot */
173 cidx = be16_to_cpu(cidx);
174
175 if (cidx >= eq->cidx)
176 return (cidx - eq->cidx);
177 else
178 return (cidx + eq->cap - eq->cidx);
179 }
180
181 void
t4_sge_init(struct adapter * sc)182 t4_sge_init(struct adapter *sc)
183 {
184 struct driver_properties *p = &sc->props;
185 ddi_dma_attr_t *dma_attr;
186 ddi_device_acc_attr_t *acc_attr;
187 uint32_t v;
188
189 /*
190 * Device access and DMA attributes for descriptor rings
191 */
192 acc_attr = &sc->sge.acc_attr_desc;
193 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
194 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
195 acc_attr->devacc_attr_dataorder = DDI_STRICTORDER_ACC;
196
197 dma_attr = &sc->sge.dma_attr_desc;
198 dma_attr->dma_attr_version = DMA_ATTR_V0;
199 dma_attr->dma_attr_addr_lo = 0;
200 dma_attr->dma_attr_addr_hi = UINT64_MAX;
201 dma_attr->dma_attr_count_max = UINT64_MAX;
202 dma_attr->dma_attr_align = 512;
203 dma_attr->dma_attr_burstsizes = 0xfff;
204 dma_attr->dma_attr_minxfer = 1;
205 dma_attr->dma_attr_maxxfer = UINT64_MAX;
206 dma_attr->dma_attr_seg = UINT64_MAX;
207 dma_attr->dma_attr_sgllen = 1;
208 dma_attr->dma_attr_granular = 1;
209 dma_attr->dma_attr_flags = 0;
210
211 /*
212 * Device access and DMA attributes for tx buffers
213 */
214 acc_attr = &sc->sge.acc_attr_tx;
215 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
216 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
217
218 dma_attr = &sc->sge.dma_attr_tx;
219 dma_attr->dma_attr_version = DMA_ATTR_V0;
220 dma_attr->dma_attr_addr_lo = 0;
221 dma_attr->dma_attr_addr_hi = UINT64_MAX;
222 dma_attr->dma_attr_count_max = UINT64_MAX;
223 dma_attr->dma_attr_align = 1;
224 dma_attr->dma_attr_burstsizes = 0xfff;
225 dma_attr->dma_attr_minxfer = 1;
226 dma_attr->dma_attr_maxxfer = UINT64_MAX;
227 dma_attr->dma_attr_seg = UINT64_MAX;
228 dma_attr->dma_attr_sgllen = TX_SGL_SEGS;
229 dma_attr->dma_attr_granular = 1;
230 dma_attr->dma_attr_flags = 0;
231
232 /*
233 * Device access and DMA attributes for rx buffers
234 */
235 sc->sge.rxb_params.dip = sc->dip;
236 sc->sge.rxb_params.buf_size = rx_buf_size;
237
238 acc_attr = &sc->sge.rxb_params.acc_attr_rx;
239 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
240 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
241
242 dma_attr = &sc->sge.rxb_params.dma_attr_rx;
243 dma_attr->dma_attr_version = DMA_ATTR_V0;
244 dma_attr->dma_attr_addr_lo = 0;
245 dma_attr->dma_attr_addr_hi = UINT64_MAX;
246 dma_attr->dma_attr_count_max = UINT64_MAX;
247 /*
248 * Low 4 bits of an rx buffer address have a special meaning to the SGE
249 * and an rx buf cannot have an address with any of these bits set.
250 * FL_ALIGN is >= 32 so we're sure things are ok.
251 */
252 dma_attr->dma_attr_align = FL_ALIGN;
253 dma_attr->dma_attr_burstsizes = 0xfff;
254 dma_attr->dma_attr_minxfer = 1;
255 dma_attr->dma_attr_maxxfer = UINT64_MAX;
256 dma_attr->dma_attr_seg = UINT64_MAX;
257 dma_attr->dma_attr_sgllen = 1;
258 dma_attr->dma_attr_granular = 1;
259 dma_attr->dma_attr_flags = 0;
260
261 sc->sge.rxbuf_cache = rxbuf_cache_create(&sc->sge.rxb_params);
262
263 v = t4_read_reg(sc, A_SGE_CONM_CTRL);
264 sc->sge.fl_starve_threshold = G_EGRTHRESHOLD(v) * 2 + 1;
265
266 t4_set_reg_field(sc, A_SGE_CONTROL, V_PKTSHIFT(M_PKTSHIFT) |
267 V_INGPADBOUNDARY(M_INGPADBOUNDARY) |
268 F_EGRSTATUSPAGESIZE, V_INGPADBOUNDARY(ilog2(FL_ALIGN) - 5) |
269 V_PKTSHIFT(FL_PKTSHIFT) |
270 F_RXPKTCPLMODE |
271 V_EGRSTATUSPAGESIZE(SPG_SIZE == 128));
272
273 t4_set_reg_field(sc, A_SGE_HOST_PAGE_SIZE,
274 V_HOSTPAGESIZEPF0(M_HOSTPAGESIZEPF0),
275 V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10));
276
277 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, rx_buf_size);
278
279 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD,
280 V_THRESHOLD_0(p->counter_val[0]) |
281 V_THRESHOLD_1(p->counter_val[1]) |
282 V_THRESHOLD_2(p->counter_val[2]) |
283 V_THRESHOLD_3(p->counter_val[3]));
284
285 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1,
286 V_TIMERVALUE0(us_to_core_ticks(sc, p->timer_val[0])) |
287 V_TIMERVALUE1(us_to_core_ticks(sc, p->timer_val[1])));
288 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3,
289 V_TIMERVALUE2(us_to_core_ticks(sc, p->timer_val[2])) |
290 V_TIMERVALUE3(us_to_core_ticks(sc, p->timer_val[3])));
291 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5,
292 V_TIMERVALUE4(us_to_core_ticks(sc, p->timer_val[4])) |
293 V_TIMERVALUE5(us_to_core_ticks(sc, p->timer_val[5])));
294
295 (void) t4_register_cpl_handler(sc, CPL_FW4_MSG, handle_fw_rpl);
296 (void) t4_register_cpl_handler(sc, CPL_FW6_MSG, handle_fw_rpl);
297 (void) t4_register_cpl_handler(sc, CPL_RX_PKT, t4_eth_rx);
298 }
299
300 /*
301 * Allocate and initialize the firmware event queue and the forwarded interrupt
302 * queues, if any. The adapter owns all these queues as they are not associated
303 * with any particular port.
304 *
305 * Returns errno on failure. Resources allocated up to that point may still be
306 * allocated. Caller is responsible for cleanup in case this function fails.
307 */
308 int
t4_setup_adapter_queues(struct adapter * sc)309 t4_setup_adapter_queues(struct adapter *sc)
310 {
311 int rc;
312
313 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
314
315 /*
316 * Firmware event queue
317 */
318 rc = alloc_fwq(sc);
319 if (rc != 0)
320 return (rc);
321
322 /*
323 * Management queue. This is just a control queue that uses the fwq as
324 * its associated iq.
325 */
326 rc = alloc_mgmtq(sc);
327
328 return (rc);
329 }
330
331 /*
332 * Idempotent
333 */
334 int
t4_teardown_adapter_queues(struct adapter * sc)335 t4_teardown_adapter_queues(struct adapter *sc)
336 {
337
338 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
339
340 (void) free_fwq(sc);
341
342 return (0);
343 }
344
345 static inline int
first_vector(struct port_info * pi)346 first_vector(struct port_info *pi)
347 {
348 struct adapter *sc = pi->adapter;
349 int rc = T4_EXTRA_INTR, i;
350
351 if (sc->intr_count == 1)
352 return (0);
353
354 for_each_port(sc, i) {
355 struct port_info *p = sc->port[i];
356
357 if (i == pi->port_id)
358 break;
359
360 #ifndef TCP_OFFLOAD_DISABLE
361 if (!(sc->flags & INTR_FWD))
362 rc += p->nrxq + p->nofldrxq;
363 else
364 rc += max(p->nrxq, p->nofldrxq);
365 #else
366 /*
367 * Not compiled with offload support and intr_count > 1. Only
368 * NIC queues exist and they'd better be taking direct
369 * interrupts.
370 */
371 KASSERT(!(sc->flags & INTR_FWD));
372 rc += p->nrxq;
373 #endif
374 }
375 return (rc);
376 }
377
378 /*
379 * Given an arbitrary "index," come up with an iq that can be used by other
380 * queues (of this port) for interrupt forwarding, SGE egress updates, etc.
381 * The iq returned is guaranteed to be something that takes direct interrupts.
382 */
383 static struct sge_iq *
port_intr_iq(struct port_info * pi,int idx)384 port_intr_iq(struct port_info *pi, int idx)
385 {
386 struct adapter *sc = pi->adapter;
387 struct sge *s = &sc->sge;
388 struct sge_iq *iq = NULL;
389
390 if (sc->intr_count == 1)
391 return (&sc->sge.fwq);
392
393 #ifndef TCP_OFFLOAD_DISABLE
394 if (!(sc->flags & INTR_FWD)) {
395 idx %= pi->nrxq + pi->nofldrxq;
396
397 if (idx >= pi->nrxq) {
398 idx -= pi->nrxq;
399 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
400 } else
401 iq = &s->rxq[pi->first_rxq + idx].iq;
402
403 } else {
404 idx %= max(pi->nrxq, pi->nofldrxq);
405
406 if (pi->nrxq >= pi->nofldrxq)
407 iq = &s->rxq[pi->first_rxq + idx].iq;
408 else
409 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
410 }
411 #else
412 /*
413 * Not compiled with offload support and intr_count > 1. Only NIC
414 * queues exist and they'd better be taking direct interrupts.
415 */
416 ASSERT(!(sc->flags & INTR_FWD));
417
418 idx %= pi->nrxq;
419 iq = &s->rxq[pi->first_rxq + idx].iq;
420 #endif
421
422 return (iq);
423 }
424
425 int
t4_setup_port_queues(struct port_info * pi)426 t4_setup_port_queues(struct port_info *pi)
427 {
428 int rc = 0, i, intr_idx, j, iqid;
429 struct sge_rxq *rxq;
430 struct sge_txq *txq;
431 #ifndef TCP_OFFLOAD_DISABLE
432 struct sge_wrq *ctrlq;
433 struct sge_ofld_rxq *ofld_rxq;
434 struct sge_wrq *ofld_txq;
435 #endif
436 struct adapter *sc = pi->adapter;
437 struct driver_properties *p = &sc->props;
438
439 pi->ksp_config = setup_port_config_kstats(pi);
440 pi->ksp_info = setup_port_info_kstats(pi);
441
442 /* Interrupt vector to start from (when using multiple vectors) */
443 intr_idx = first_vector(pi);
444
445 /*
446 * First pass over all rx queues (NIC and TOE):
447 * a) initialize iq and fl
448 * b) allocate queue iff it will take direct interrupts.
449 */
450
451 for_each_rxq(pi, i, rxq) {
452
453 init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, p->qsize_rxq,
454 RX_IQ_ESIZE);
455
456 init_fl(&rxq->fl, p->qsize_rxq / 8); /* 8 bufs in each entry */
457
458 #ifndef TCP_OFFLOAD_DISABLE
459 if ((!(sc->flags & INTR_FWD)) ||
460 (sc->intr_count > 1 && pi->nrxq >= pi->nofldrxq))
461 #else
462 if (!(sc->flags & INTR_FWD))
463 #endif
464 {
465 rxq->iq.flags |= IQ_INTR;
466 rc = alloc_rxq(pi, rxq, intr_idx, i);
467 if (rc != 0)
468 goto done;
469 intr_idx++;
470 }
471
472 }
473
474 #ifndef TCP_OFFLOAD_DISABLE
475 for_each_ofld_rxq(pi, i, ofld_rxq) {
476
477 init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
478 p->qsize_rxq, RX_IQ_ESIZE);
479
480 init_fl(&ofld_rxq->fl, p->qsize_rxq / 8);
481
482 if (!(sc->flags & INTR_FWD) ||
483 (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) {
484 ofld_rxq->iq.flags = IQ_INTR;
485 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx);
486 if (rc != 0)
487 goto done;
488
489 intr_idx++;
490 }
491 }
492 #endif
493
494 /*
495 * Second pass over all rx queues (NIC and TOE). The queues forwarding
496 * their interrupts are allocated now.
497 */
498 j = 0;
499 for_each_rxq(pi, i, rxq) {
500 if (rxq->iq.flags & IQ_INTR)
501 continue;
502
503 intr_idx = port_intr_iq(pi, j)->abs_id;
504
505 rc = alloc_rxq(pi, rxq, intr_idx, i);
506 if (rc != 0)
507 goto done;
508 j++;
509 }
510
511 #ifndef TCP_OFFLOAD_DISABLE
512 for_each_ofld_rxq(pi, i, ofld_rxq) {
513 if (ofld_rxq->iq.flags & IQ_INTR)
514 continue;
515
516 intr_idx = port_intr_iq(pi, j)->abs_id;
517 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx);
518 if (rc != 0)
519 goto done;
520 j++;
521 }
522 #endif
523 /*
524 * Now the tx queues. Only one pass needed.
525 */
526 j = 0;
527 for_each_txq(pi, i, txq) {
528 uint16_t iqid;
529
530 iqid = port_intr_iq(pi, j)->cntxt_id;
531 init_eq(&txq->eq, EQ_ETH, p->qsize_txq, pi->tx_chan, iqid);
532 rc = alloc_txq(pi, txq, i);
533 if (rc != 0)
534 goto done;
535 }
536
537 #ifndef TCP_OFFLOAD_DISABLE
538 for_each_ofld_txq(pi, i, ofld_txq) {
539 uint16_t iqid;
540
541 iqid = port_intr_iq(pi, j)->cntxt_id;
542 init_eq(&ofld_txq->eq, EQ_OFLD, p->qsize_txq, pi->tx_chan,
543 iqid);
544 rc = alloc_wrq(sc, pi, ofld_txq, i);
545 if (rc != 0)
546 goto done;
547 }
548 #endif
549
550 /*
551 * Finally, the control queue.
552 */
553 ctrlq = &sc->sge.ctrlq[pi->port_id];
554 iqid = port_intr_iq(pi, 0)->cntxt_id;
555 init_eq(&ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, iqid);
556 rc = alloc_wrq(sc, pi, ctrlq, 0);
557
558 done:
559 if (rc != 0)
560 (void) t4_teardown_port_queues(pi);
561
562 return (rc);
563 }
564
565 /*
566 * Idempotent
567 */
568 int
t4_teardown_port_queues(struct port_info * pi)569 t4_teardown_port_queues(struct port_info *pi)
570 {
571 int i;
572 struct sge_rxq *rxq;
573 struct sge_txq *txq;
574 struct adapter *sc = pi->adapter;
575 #ifndef TCP_OFFLOAD_DISABLE
576 struct sge_ofld_rxq *ofld_rxq;
577 struct sge_wrq *ofld_txq;
578 #endif
579
580 if (pi->ksp_config != NULL) {
581 kstat_delete(pi->ksp_config);
582 pi->ksp_config = NULL;
583 }
584 if (pi->ksp_info != NULL) {
585 kstat_delete(pi->ksp_info);
586 pi->ksp_info = NULL;
587 }
588
589 (void) free_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
590
591 for_each_txq(pi, i, txq) {
592 (void) free_txq(pi, txq);
593 }
594
595 #ifndef TCP_OFFLOAD_DISABLE
596 for_each_ofld_txq(pi, i, ofld_txq) {
597 (void) free_wrq(sc, ofld_txq);
598 }
599
600 for_each_ofld_rxq(pi, i, ofld_rxq) {
601 if ((ofld_rxq->iq.flags & IQ_INTR) == 0)
602 (void) free_ofld_rxq(pi, ofld_rxq);
603 }
604 #endif
605
606 for_each_rxq(pi, i, rxq) {
607 if ((rxq->iq.flags & IQ_INTR) == 0)
608 (void) free_rxq(pi, rxq);
609 }
610
611 /*
612 * Then take down the rx queues that take direct interrupts.
613 */
614
615 for_each_rxq(pi, i, rxq) {
616 if (rxq->iq.flags & IQ_INTR)
617 (void) free_rxq(pi, rxq);
618 }
619
620 #ifndef TCP_OFFLOAD_DISABLE
621 for_each_ofld_rxq(pi, i, ofld_rxq) {
622 if (ofld_rxq->iq.flags & IQ_INTR)
623 (void) free_ofld_rxq(pi, ofld_rxq);
624 }
625 #endif
626
627 return (0);
628 }
629
630 /* Deals with errors and forwarded interrupts */
631 uint_t
t4_intr_all(caddr_t arg1,caddr_t arg2)632 t4_intr_all(caddr_t arg1, caddr_t arg2)
633 {
634
635 (void) t4_intr_err(arg1, arg2);
636 (void) t4_intr(arg1, arg2);
637
638 return (DDI_INTR_CLAIMED);
639 }
640
641 /* Deals with interrupts on the given ingress queue */
642 /* ARGSUSED */
643 uint_t
t4_intr(caddr_t arg1,caddr_t arg2)644 t4_intr(caddr_t arg1, caddr_t arg2)
645 {
646 /* LINTED: E_BAD_PTR_CAST_ALIGN */
647 struct sge_iq *iq = (struct sge_iq *)arg2;
648
649 if (atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY) == IQS_IDLE) {
650 (void) service_iq(iq, 0);
651 (void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE);
652 }
653 return (DDI_INTR_CLAIMED);
654 }
655
656 /* Deals with error interrupts */
657 /* ARGSUSED */
658 uint_t
t4_intr_err(caddr_t arg1,caddr_t arg2)659 t4_intr_err(caddr_t arg1, caddr_t arg2)
660 {
661 /* LINTED: E_BAD_PTR_CAST_ALIGN */
662 struct adapter *sc = (struct adapter *)arg1;
663
664 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0);
665 (void) t4_slow_intr_handler(sc);
666
667 return (DDI_INTR_CLAIMED);
668 }
669
670 /*
671 * Deals with anything and everything on the given ingress queue.
672 */
673 static int
service_iq(struct sge_iq * iq,int budget)674 service_iq(struct sge_iq *iq, int budget)
675 {
676 struct sge_iq *q;
677 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
678 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
679 struct adapter *sc = iq->adapter;
680 struct rsp_ctrl *ctrl;
681 const struct rss_header *rss;
682 int ndescs = 0, limit, fl_bufs_used = 0;
683 int rsp_type;
684 uint32_t lq;
685 mblk_t *m;
686 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
687
688 limit = budget ? budget : iq->qsize / 8;
689
690 /*
691 * We always come back and check the descriptor ring for new indirect
692 * interrupts and other responses after running a single handler.
693 */
694 for (;;) {
695 while (is_new_response(iq, &ctrl)) {
696
697 membar_consumer();
698
699 m = NULL;
700 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
701 lq = be32_to_cpu(ctrl->pldbuflen_qid);
702 rss = (const void *)iq->cdesc;
703
704 switch (rsp_type) {
705 case X_RSPD_TYPE_FLBUF:
706
707 ASSERT(iq->flags & IQ_HAS_FL);
708
709 m = get_fl_payload(fl, lq, &fl_bufs_used);
710 if (m == NULL) {
711 panic("%s: line %d.", __func__,
712 __LINE__);
713 }
714
715 /* FALLTHRU */
716 case X_RSPD_TYPE_CPL:
717
718 ASSERT(rss->opcode < NUM_CPL_CMDS);
719 sc->cpl_handler[rss->opcode](iq, rss, m);
720 break;
721
722 case X_RSPD_TYPE_INTR:
723
724 /*
725 * Interrupts should be forwarded only to queues
726 * that are not forwarding their interrupts.
727 * This means service_iq can recurse but only 1
728 * level deep.
729 */
730 ASSERT(budget == 0);
731
732 q = sc->sge.iqmap[lq - sc->sge.iq_start];
733 if (atomic_cas_uint(&q->state, IQS_IDLE,
734 IQS_BUSY) == IQS_IDLE) {
735 if (service_iq(q, q->qsize / 8) == 0) {
736 (void) atomic_cas_uint(
737 &q->state, IQS_BUSY,
738 IQS_IDLE);
739 } else {
740 STAILQ_INSERT_TAIL(&iql, q,
741 link);
742 }
743 }
744 break;
745
746 default:
747 break;
748 }
749
750 iq_next(iq);
751 if (++ndescs == limit) {
752 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
753 V_CIDXINC(ndescs) |
754 V_INGRESSQID(iq->cntxt_id) |
755 V_SEINTARM(V_QINTR_TIMER_IDX(
756 X_TIMERREG_UPDATE_CIDX)));
757 ndescs = 0;
758
759 if (fl_bufs_used > 0) {
760 ASSERT(iq->flags & IQ_HAS_FL);
761 FL_LOCK(fl);
762 fl->needed += fl_bufs_used;
763 (void) refill_fl(sc, fl, fl->cap / 8);
764 FL_UNLOCK(fl);
765 fl_bufs_used = 0;
766 }
767
768 if (budget != 0)
769 return (EINPROGRESS);
770 }
771 }
772
773 if (STAILQ_EMPTY(&iql) != 0)
774 break;
775
776 /*
777 * Process the head only, and send it to the back of the list if
778 * it's still not done.
779 */
780 q = STAILQ_FIRST(&iql);
781 STAILQ_REMOVE_HEAD(&iql, link);
782 if (service_iq(q, q->qsize / 8) == 0)
783 (void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE);
784 else
785 STAILQ_INSERT_TAIL(&iql, q, link);
786 }
787
788 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
789 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
790
791 if (iq->flags & IQ_HAS_FL) {
792 int starved;
793
794 FL_LOCK(fl);
795 fl->needed += fl_bufs_used;
796 starved = refill_fl(sc, fl, fl->cap / 4);
797 FL_UNLOCK(fl);
798 if (starved != 0)
799 add_fl_to_sfl(sc, fl);
800 }
801
802 return (0);
803 }
804
805 int
t4_mgmt_tx(struct adapter * sc,mblk_t * m)806 t4_mgmt_tx(struct adapter *sc, mblk_t *m)
807 {
808 return (t4_wrq_tx(sc, &sc->sge.mgmtq, m));
809 }
810
811 /*
812 * Doesn't fail. Holds on to work requests it can't send right away.
813 */
814 int
t4_wrq_tx_locked(struct adapter * sc,struct sge_wrq * wrq,mblk_t * m0)815 t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, mblk_t *m0)
816 {
817 struct sge_eq *eq = &wrq->eq;
818 struct mblk_pair *wr_list = &wrq->wr_list;
819 int can_reclaim;
820 caddr_t dst;
821 mblk_t *wr, *next;
822
823 TXQ_LOCK_ASSERT_OWNED(wrq);
824 #ifndef TCP_OFFLOAD_DISABLE
825 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_OFLD ||
826 (eq->flags & EQ_TYPEMASK) == EQ_CTRL);
827 #else
828 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_CTRL);
829 #endif
830
831 if (m0 != NULL) {
832 if (wr_list->head != NULL)
833 wr_list->tail->b_next = m0;
834 else
835 wr_list->head = m0;
836 while (m0->b_next)
837 m0 = m0->b_next;
838 wr_list->tail = m0;
839 }
840
841 can_reclaim = reclaimable(eq);
842 eq->cidx += can_reclaim;
843 eq->avail += can_reclaim;
844 if (eq->cidx >= eq->cap)
845 eq->cidx -= eq->cap;
846
847 for (wr = wr_list->head; wr; wr = next) {
848 int ndesc, len = 0;
849 mblk_t *m;
850
851 next = wr->b_next;
852 wr->b_next = NULL;
853
854 for (m = wr; m; m = m->b_cont)
855 len += MBLKL(m);
856
857 ASSERT(len > 0 && (len & 0x7) == 0);
858 ASSERT(len <= SGE_MAX_WR_LEN);
859
860 ndesc = howmany(len, EQ_ESIZE);
861 if (eq->avail < ndesc) {
862 wr->b_next = next;
863 wrq->no_desc++;
864 break;
865 }
866
867 dst = (void *)&eq->desc[eq->pidx];
868 for (m = wr; m; m = m->b_cont)
869 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m));
870
871 eq->pidx += ndesc;
872 eq->avail -= ndesc;
873 if (eq->pidx >= eq->cap)
874 eq->pidx -= eq->cap;
875
876 eq->pending += ndesc;
877 if (eq->pending > 16)
878 ring_tx_db(sc, eq);
879
880 wrq->tx_wrs++;
881 freemsg(wr);
882
883 if (eq->avail < 8) {
884 can_reclaim = reclaimable(eq);
885 eq->cidx += can_reclaim;
886 eq->avail += can_reclaim;
887 if (eq->cidx >= eq->cap)
888 eq->cidx -= eq->cap;
889 }
890 }
891
892 if (eq->pending != 0)
893 ring_tx_db(sc, eq);
894
895 if (wr == NULL)
896 wr_list->head = wr_list->tail = NULL;
897 else {
898 wr_list->head = wr;
899
900 ASSERT(wr_list->tail->b_next == NULL);
901 }
902
903 return (0);
904 }
905
906 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */
907 #define TXPKTS_PKT_HDR ((\
908 sizeof (struct ulp_txpkt) + \
909 sizeof (struct ulptx_idata) + \
910 sizeof (struct cpl_tx_pkt_core)) / 8)
911
912 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */
913 #define TXPKTS_WR_HDR (\
914 sizeof (struct fw_eth_tx_pkts_wr) / 8 + \
915 TXPKTS_PKT_HDR)
916
917 /* Header of a tx WR, before SGL of first packet (in flits) */
918 #define TXPKT_WR_HDR ((\
919 sizeof (struct fw_eth_tx_pkt_wr) + \
920 sizeof (struct cpl_tx_pkt_core)) / 8)
921
922 /* Header of a tx LSO WR, before SGL of first packet (in flits) */
923 #define TXPKT_LSO_WR_HDR ((\
924 sizeof (struct fw_eth_tx_pkt_wr) + \
925 sizeof (struct cpl_tx_pkt_lso) + \
926 sizeof (struct cpl_tx_pkt_core)) / 8)
927
928 mblk_t *
t4_eth_tx(struct port_info * pi,struct sge_txq * txq,mblk_t * frame)929 t4_eth_tx(struct port_info *pi, struct sge_txq *txq, mblk_t *frame)
930 {
931 struct adapter *sc = pi->adapter;
932 struct sge_eq *eq = &txq->eq;
933 mblk_t *next_frame;
934 int rc, coalescing;
935 struct txpkts txpkts;
936 struct txinfo txinfo;
937
938 txpkts.npkt = 0; /* indicates there's nothing in txpkts */
939 coalescing = 0;
940
941 TXQ_LOCK(txq);
942 if (eq->avail < 8)
943 (void) reclaim_tx_descs(txq, 8);
944 for (; frame; frame = next_frame) {
945
946 if (eq->avail < 8)
947 break;
948
949 next_frame = frame->b_next;
950 frame->b_next = NULL;
951
952 if (next_frame != NULL)
953 coalescing = 1;
954
955 rc = get_frame_txinfo(txq, &frame, &txinfo, coalescing);
956 if (rc != 0) {
957 if (rc == ENOMEM) {
958
959 /* Short of resources, suspend tx */
960
961 frame->b_next = next_frame;
962 break;
963 }
964
965 /*
966 * Unrecoverable error for this frame, throw it
967 * away and move on to the next.
968 */
969
970 freemsg(frame);
971 continue;
972 }
973
974 if (coalescing != 0 &&
975 add_to_txpkts(txq, &txpkts, frame, &txinfo) == 0) {
976
977 /* Successfully absorbed into txpkts */
978
979 write_ulp_cpl_sgl(pi, txq, &txpkts, &txinfo);
980 goto doorbell;
981 }
982
983 /*
984 * We weren't coalescing to begin with, or current frame could
985 * not be coalesced (add_to_txpkts flushes txpkts if a frame
986 * given to it can't be coalesced). Either way there should be
987 * nothing in txpkts.
988 */
989 ASSERT(txpkts.npkt == 0);
990
991 /* We're sending out individual frames now */
992 coalescing = 0;
993
994 if (eq->avail < 8)
995 (void) reclaim_tx_descs(txq, 8);
996 rc = write_txpkt_wr(pi, txq, frame, &txinfo);
997 if (rc != 0) {
998
999 /* Short of hardware descriptors, suspend tx */
1000
1001 /*
1002 * This is an unlikely but expensive failure. We've
1003 * done all the hard work (DMA bindings etc.) and now we
1004 * can't send out the frame. What's worse, we have to
1005 * spend even more time freeing up everything in txinfo.
1006 */
1007 txq->qfull++;
1008 free_txinfo_resources(txq, &txinfo);
1009
1010 frame->b_next = next_frame;
1011 break;
1012 }
1013
1014 doorbell:
1015 /* Fewer and fewer doorbells as the queue fills up */
1016 if (eq->pending >= (1 << (fls(eq->qsize - eq->avail) / 2)))
1017 ring_tx_db(sc, eq);
1018 (void) reclaim_tx_descs(txq, 32);
1019 }
1020
1021 if (txpkts.npkt > 0)
1022 write_txpkts_wr(txq, &txpkts);
1023
1024 /*
1025 * frame not NULL means there was an error but we haven't thrown it
1026 * away. This can happen when we're short of tx descriptors (qfull) or
1027 * maybe even DMA handles (dma_hdl_failed). Either way, a credit flush
1028 * and reclaim will get things going again.
1029 *
1030 * If eq->avail is already 0 we know a credit flush was requested in the
1031 * WR that reduced it to 0 so we don't need another flush (we don't have
1032 * any descriptor for a flush WR anyway, duh).
1033 */
1034 if (frame && eq->avail > 0)
1035 write_txqflush_wr(txq);
1036
1037 if (eq->pending != 0)
1038 ring_tx_db(sc, eq);
1039
1040 (void) reclaim_tx_descs(txq, eq->qsize);
1041 TXQ_UNLOCK(txq);
1042
1043 return (frame);
1044 }
1045
1046 static inline void
init_iq(struct sge_iq * iq,struct adapter * sc,int tmr_idx,int8_t pktc_idx,int qsize,uint8_t esize)1047 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int8_t pktc_idx, int
1048 qsize, uint8_t esize)
1049 {
1050 ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS);
1051 ASSERT(pktc_idx < SGE_NCOUNTERS); /* -ve is ok, means don't use */
1052
1053 iq->flags = 0;
1054 iq->adapter = sc;
1055 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
1056 iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
1057 if (pktc_idx >= 0) {
1058 iq->intr_params |= F_QINTR_CNT_EN;
1059 iq->intr_pktc_idx = pktc_idx;
1060 }
1061 iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */
1062 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */
1063 }
1064
1065 static inline void
init_fl(struct sge_fl * fl,uint16_t qsize)1066 init_fl(struct sge_fl *fl, uint16_t qsize)
1067 {
1068
1069 fl->qsize = qsize;
1070 }
1071
1072 static inline void
init_eq(struct sge_eq * eq,uint16_t eqtype,uint16_t qsize,uint8_t tx_chan,uint16_t iqid)1073 init_eq(struct sge_eq *eq, uint16_t eqtype, uint16_t qsize, uint8_t tx_chan,
1074 uint16_t iqid)
1075 {
1076 ASSERT(tx_chan < NCHAN);
1077 ASSERT(eqtype <= EQ_TYPEMASK);
1078
1079 eq->flags = eqtype & EQ_TYPEMASK;
1080 eq->tx_chan = tx_chan;
1081 eq->iqid = iqid;
1082 eq->qsize = qsize;
1083 }
1084
1085 /*
1086 * Allocates the ring for an ingress queue and an optional freelist. If the
1087 * freelist is specified it will be allocated and then associated with the
1088 * ingress queue.
1089 *
1090 * Returns errno on failure. Resources allocated up to that point may still be
1091 * allocated. Caller is responsible for cleanup in case this function fails.
1092 *
1093 * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then
1094 * the intr_idx specifies the vector, starting from 0. Otherwise it specifies
1095 * the index of the queue to which its interrupts will be forwarded.
1096 */
1097 static int
alloc_iq_fl(struct port_info * pi,struct sge_iq * iq,struct sge_fl * fl,int intr_idx,int cong)1098 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
1099 int intr_idx, int cong)
1100 {
1101 int rc, cntxt_id;
1102 size_t len;
1103 struct fw_iq_cmd c;
1104 struct adapter *sc = iq->adapter;
1105 uint32_t v = 0;
1106
1107 len = iq->qsize * iq->esize;
1108 rc = alloc_desc_ring(sc, len, DDI_DMA_READ, &iq->dhdl, &iq->ahdl,
1109 &iq->ba, (caddr_t *)&iq->desc);
1110 if (rc != 0)
1111 return (rc);
1112
1113 bzero(&c, sizeof (c));
1114 c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
1115 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
1116 V_FW_IQ_CMD_VFN(0));
1117
1118 c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
1119 FW_LEN16(c));
1120
1121 /* Special handling for firmware event queue */
1122 if (iq == &sc->sge.fwq)
1123 v |= F_FW_IQ_CMD_IQASYNCH;
1124
1125 if (iq->flags & IQ_INTR)
1126 ASSERT(intr_idx < sc->intr_count);
1127 else
1128 v |= F_FW_IQ_CMD_IQANDST;
1129 v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx);
1130
1131 c.type_to_iqandstindex = cpu_to_be32(v |
1132 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
1133 V_FW_IQ_CMD_VIID(pi->viid) |
1134 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
1135 c.iqdroprss_to_iqesize = cpu_to_be16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
1136 F_FW_IQ_CMD_IQGTSMODE |
1137 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) |
1138 V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4));
1139 c.iqsize = cpu_to_be16(iq->qsize);
1140 c.iqaddr = cpu_to_be64(iq->ba);
1141 if (cong >= 0)
1142 c.iqns_to_fl0congen = BE_32(F_FW_IQ_CMD_IQFLINTCONGEN);
1143
1144 if (fl != NULL) {
1145 mutex_init(&fl->lock, NULL, MUTEX_DRIVER,
1146 DDI_INTR_PRI(sc->intr_pri));
1147 fl->flags |= FL_MTX;
1148
1149 len = fl->qsize * RX_FL_ESIZE;
1150 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &fl->dhdl,
1151 &fl->ahdl, &fl->ba, (caddr_t *)&fl->desc);
1152 if (rc != 0)
1153 return (rc);
1154
1155 /* Allocate space for one software descriptor per buffer. */
1156 fl->cap = (fl->qsize - SPG_SIZE / RX_FL_ESIZE) * 8;
1157 fl->sdesc = kmem_zalloc(sizeof (struct fl_sdesc) * fl->cap,
1158 KM_SLEEP);
1159 fl->needed = fl->cap;
1160 fl->lowat = roundup(sc->sge.fl_starve_threshold, 8);
1161
1162 c.iqns_to_fl0congen |=
1163 cpu_to_be32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
1164 F_FW_IQ_CMD_FL0PACKEN | F_FW_IQ_CMD_FL0PADEN);
1165 if (cong >= 0) {
1166 c.iqns_to_fl0congen |=
1167 BE_32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) |
1168 F_FW_IQ_CMD_FL0CONGCIF |
1169 F_FW_IQ_CMD_FL0CONGEN);
1170 }
1171 c.fl0dcaen_to_fl0cidxfthresh =
1172 cpu_to_be16(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_64B) |
1173 V_FW_IQ_CMD_FL0FBMAX(X_FETCHBURSTMAX_512B));
1174 c.fl0size = cpu_to_be16(fl->qsize);
1175 c.fl0addr = cpu_to_be64(fl->ba);
1176 }
1177
1178 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1179 if (rc != 0) {
1180 cxgb_printf(sc->dip, CE_WARN,
1181 "failed to create ingress queue: %d", rc);
1182 return (rc);
1183 }
1184
1185 iq->cdesc = iq->desc;
1186 iq->cidx = 0;
1187 iq->gen = 1;
1188 iq->intr_next = iq->intr_params;
1189 iq->adapter = sc;
1190 iq->cntxt_id = be16_to_cpu(c.iqid);
1191 iq->abs_id = be16_to_cpu(c.physiqid);
1192 iq->flags |= IQ_ALLOCATED;
1193
1194 cntxt_id = iq->cntxt_id - sc->sge.iq_start;
1195 if (cntxt_id >= sc->sge.niq) {
1196 panic("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
1197 cntxt_id, sc->sge.niq - 1);
1198 }
1199 sc->sge.iqmap[cntxt_id] = iq;
1200
1201 if (fl != NULL) {
1202 fl->cntxt_id = be16_to_cpu(c.fl0id);
1203 fl->pidx = fl->cidx = 0;
1204 fl->copy_threshold = rx_copy_threshold;
1205
1206 cntxt_id = fl->cntxt_id - sc->sge.eq_start;
1207 if (cntxt_id >= sc->sge.neq) {
1208 panic("%s: fl->cntxt_id (%d) more than the max (%d)",
1209 __func__, cntxt_id, sc->sge.neq - 1);
1210 }
1211 sc->sge.eqmap[cntxt_id] = (void *)fl;
1212
1213 FL_LOCK(fl);
1214 (void) refill_fl(sc, fl, fl->lowat);
1215 FL_UNLOCK(fl);
1216
1217 iq->flags |= IQ_HAS_FL;
1218 }
1219
1220 /* Enable IQ interrupts */
1221 iq->state = IQS_IDLE;
1222 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) |
1223 V_INGRESSQID(iq->cntxt_id));
1224
1225 return (0);
1226 }
1227
1228 static int
free_iq_fl(struct port_info * pi,struct sge_iq * iq,struct sge_fl * fl)1229 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl)
1230 {
1231 int rc;
1232 struct adapter *sc = iq->adapter;
1233 dev_info_t *dip;
1234
1235 dip = pi ? pi->dip : sc->dip;
1236
1237 if (iq != NULL) {
1238 if (iq->flags & IQ_ALLOCATED) {
1239 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0,
1240 FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id,
1241 fl ? fl->cntxt_id : 0xffff, 0xffff);
1242 if (rc != 0) {
1243 cxgb_printf(dip, CE_WARN,
1244 "failed to free queue %p: %d", iq, rc);
1245 return (rc);
1246 }
1247 iq->flags &= ~IQ_ALLOCATED;
1248 }
1249
1250 if (iq->desc != NULL) {
1251 (void) free_desc_ring(&iq->dhdl, &iq->ahdl);
1252 iq->desc = NULL;
1253 }
1254
1255 bzero(iq, sizeof (*iq));
1256 }
1257
1258 if (fl != NULL) {
1259 if (fl->sdesc != NULL) {
1260 FL_LOCK(fl);
1261 free_fl_bufs(fl);
1262 FL_UNLOCK(fl);
1263
1264 kmem_free(fl->sdesc, sizeof (struct fl_sdesc) *
1265 fl->cap);
1266 fl->sdesc = NULL;
1267 }
1268
1269 if (fl->desc != NULL) {
1270 (void) free_desc_ring(&fl->dhdl, &fl->ahdl);
1271 fl->desc = NULL;
1272 }
1273
1274 if (fl->flags & FL_MTX) {
1275 mutex_destroy(&fl->lock);
1276 fl->flags &= ~FL_MTX;
1277 }
1278
1279 bzero(fl, sizeof (struct sge_fl));
1280 }
1281
1282 return (0);
1283 }
1284
1285 static int
alloc_fwq(struct adapter * sc)1286 alloc_fwq(struct adapter *sc)
1287 {
1288 int rc, intr_idx;
1289 struct sge_iq *fwq = &sc->sge.fwq;
1290
1291 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE);
1292 fwq->flags |= IQ_INTR; /* always */
1293 intr_idx = sc->intr_count > 1 ? 1 : 0;
1294 rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1);
1295 if (rc != 0) {
1296 cxgb_printf(sc->dip, CE_WARN,
1297 "failed to create firmware event queue: %d.", rc);
1298 return (rc);
1299 }
1300
1301 return (0);
1302 }
1303
1304 static int
free_fwq(struct adapter * sc)1305 free_fwq(struct adapter *sc)
1306 {
1307
1308 return (free_iq_fl(NULL, &sc->sge.fwq, NULL));
1309 }
1310
1311 static int
alloc_mgmtq(struct adapter * sc)1312 alloc_mgmtq(struct adapter *sc)
1313 {
1314 int rc;
1315 struct sge_wrq *mgmtq = &sc->sge.mgmtq;
1316
1317 init_eq(&mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan,
1318 sc->sge.fwq.cntxt_id);
1319 rc = alloc_wrq(sc, NULL, mgmtq, 0);
1320 if (rc != 0) {
1321 cxgb_printf(sc->dip, CE_WARN,
1322 "failed to create management queue: %d\n", rc);
1323 return (rc);
1324 }
1325
1326 return (0);
1327 }
1328
1329 static int
alloc_rxq(struct port_info * pi,struct sge_rxq * rxq,int intr_idx,int i)1330 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int i)
1331 {
1332 int rc;
1333
1334 rxq->port = pi;
1335 rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx, 1 << pi->tx_chan);
1336 if (rc != 0)
1337 return (rc);
1338
1339 rxq->ksp = setup_rxq_kstats(pi, rxq, i);
1340
1341 return (rc);
1342 }
1343
1344 static int
free_rxq(struct port_info * pi,struct sge_rxq * rxq)1345 free_rxq(struct port_info *pi, struct sge_rxq *rxq)
1346 {
1347 int rc;
1348
1349 if (rxq->ksp != NULL) {
1350 kstat_delete(rxq->ksp);
1351 rxq->ksp = NULL;
1352 }
1353
1354 rc = free_iq_fl(pi, &rxq->iq, &rxq->fl);
1355 if (rc == 0)
1356 bzero(&rxq->fl, sizeof (*rxq) - offsetof(struct sge_rxq, fl));
1357
1358 return (rc);
1359 }
1360
1361 #ifndef TCP_OFFLOAD_DISABLE
1362 static int
alloc_ofld_rxq(struct port_info * pi,struct sge_ofld_rxq * ofld_rxq,int intr_idx)1363 alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq,
1364 int intr_idx)
1365 {
1366 int rc;
1367
1368 rc = alloc_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx,
1369 1 << pi->tx_chan);
1370 if (rc != 0)
1371 return (rc);
1372
1373 return (rc);
1374 }
1375
1376 static int
free_ofld_rxq(struct port_info * pi,struct sge_ofld_rxq * ofld_rxq)1377 free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq)
1378 {
1379 int rc;
1380
1381 rc = free_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl);
1382 if (rc == 0)
1383 bzero(&ofld_rxq->fl, sizeof (*ofld_rxq) -
1384 offsetof(struct sge_ofld_rxq, fl));
1385
1386 return (rc);
1387 }
1388 #endif
1389
1390 static int
ctrl_eq_alloc(struct adapter * sc,struct sge_eq * eq)1391 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
1392 {
1393 int rc, cntxt_id;
1394 struct fw_eq_ctrl_cmd c;
1395
1396 bzero(&c, sizeof (c));
1397
1398 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
1399 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) |
1400 V_FW_EQ_CTRL_CMD_VFN(0));
1401 c.alloc_to_len16 = BE_32(F_FW_EQ_CTRL_CMD_ALLOC |
1402 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c));
1403 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* TODO */
1404 c.physeqid_pkd = BE_32(0);
1405 c.fetchszm_to_iqid =
1406 BE_32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1407 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) |
1408 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
1409 c.dcaen_to_eqsize =
1410 BE_32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1411 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1412 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1413 V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize));
1414 c.eqaddr = BE_64(eq->ba);
1415
1416 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1417 if (rc != 0) {
1418 cxgb_printf(sc->dip, CE_WARN,
1419 "failed to create control queue %d: %d", eq->tx_chan, rc);
1420 return (rc);
1421 }
1422 eq->flags |= EQ_ALLOCATED;
1423
1424 eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(BE_32(c.cmpliqid_eqid));
1425 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1426 if (cntxt_id >= sc->sge.neq)
1427 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1428 cntxt_id, sc->sge.neq - 1);
1429 sc->sge.eqmap[cntxt_id] = eq;
1430
1431 return (rc);
1432 }
1433
1434 static int
eth_eq_alloc(struct adapter * sc,struct port_info * pi,struct sge_eq * eq)1435 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1436 {
1437 int rc, cntxt_id;
1438 struct fw_eq_eth_cmd c;
1439
1440 bzero(&c, sizeof (c));
1441
1442 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
1443 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
1444 V_FW_EQ_ETH_CMD_VFN(0));
1445 c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC |
1446 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
1447 c.viid_pkd = BE_32(V_FW_EQ_ETH_CMD_VIID(pi->viid));
1448 c.fetchszm_to_iqid =
1449 BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1450 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
1451 V_FW_EQ_ETH_CMD_IQID(eq->iqid));
1452 c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1453 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1454 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1455 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize));
1456 c.eqaddr = BE_64(eq->ba);
1457
1458 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1459 if (rc != 0) {
1460 cxgb_printf(pi->dip, CE_WARN,
1461 "failed to create Ethernet egress queue: %d", rc);
1462 return (rc);
1463 }
1464 eq->flags |= EQ_ALLOCATED;
1465
1466 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd));
1467 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1468 if (cntxt_id >= sc->sge.neq)
1469 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1470 cntxt_id, sc->sge.neq - 1);
1471 sc->sge.eqmap[cntxt_id] = eq;
1472
1473 return (rc);
1474 }
1475
1476 #ifndef TCP_OFFLOAD_DISABLE
1477 static int
ofld_eq_alloc(struct adapter * sc,struct port_info * pi,struct sge_eq * eq)1478 ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1479 {
1480 int rc, cntxt_id;
1481 struct fw_eq_ofld_cmd c;
1482
1483 bzero(&c, sizeof (c));
1484
1485 c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST |
1486 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) |
1487 V_FW_EQ_OFLD_CMD_VFN(0));
1488 c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC |
1489 F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c));
1490 c.fetchszm_to_iqid =
1491 htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1492 V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) |
1493 F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid));
1494 c.dcaen_to_eqsize =
1495 BE_32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1496 V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1497 V_FW_EQ_OFLD_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1498 V_FW_EQ_OFLD_CMD_EQSIZE(eq->qsize));
1499 c.eqaddr = BE_64(eq->ba);
1500
1501 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1502 if (rc != 0) {
1503 cxgb_printf(pi->dip, CE_WARN,
1504 "failed to create egress queue for TCP offload: %d", rc);
1505 return (rc);
1506 }
1507 eq->flags |= EQ_ALLOCATED;
1508
1509 eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(BE_32(c.eqid_pkd));
1510 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1511 if (cntxt_id >= sc->sge.neq)
1512 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1513 cntxt_id, sc->sge.neq - 1);
1514 sc->sge.eqmap[cntxt_id] = eq;
1515
1516 return (rc);
1517 }
1518 #endif
1519
1520 static int
alloc_eq(struct adapter * sc,struct port_info * pi,struct sge_eq * eq)1521 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1522 {
1523 int rc;
1524 size_t len;
1525
1526 mutex_init(&eq->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(sc->intr_pri));
1527 eq->flags |= EQ_MTX;
1528
1529 len = eq->qsize * EQ_ESIZE;
1530 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &eq->desc_dhdl,
1531 &eq->desc_ahdl, &eq->ba, (caddr_t *)&eq->desc);
1532 if (rc != 0)
1533 return (rc);
1534
1535 eq->cap = eq->qsize - SPG_SIZE / EQ_ESIZE;
1536 eq->spg = (void *)&eq->desc[eq->cap];
1537 eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */
1538 eq->pidx = eq->cidx = 0;
1539
1540 switch (eq->flags & EQ_TYPEMASK) {
1541 case EQ_CTRL:
1542 rc = ctrl_eq_alloc(sc, eq);
1543 break;
1544
1545 case EQ_ETH:
1546 rc = eth_eq_alloc(sc, pi, eq);
1547 break;
1548
1549 #ifndef TCP_OFFLOAD_DISABLE
1550 case EQ_OFLD:
1551 rc = ofld_eq_alloc(sc, pi, eq);
1552 break;
1553 #endif
1554
1555 default:
1556 panic("%s: invalid eq type %d.", __func__,
1557 eq->flags & EQ_TYPEMASK);
1558 }
1559 if (rc != 0) {
1560 cxgb_printf(sc->dip, CE_WARN,
1561 "failed to allocate egress queue(%d): %d",
1562 eq->flags & EQ_TYPEMASK, rc);
1563 }
1564
1565 return (rc);
1566 }
1567
1568 static int
free_eq(struct adapter * sc,struct sge_eq * eq)1569 free_eq(struct adapter *sc, struct sge_eq *eq)
1570 {
1571 int rc;
1572
1573 if (eq->flags & EQ_ALLOCATED) {
1574 switch (eq->flags & EQ_TYPEMASK) {
1575 case EQ_CTRL:
1576 rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0,
1577 eq->cntxt_id);
1578 break;
1579
1580 case EQ_ETH:
1581 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0,
1582 eq->cntxt_id);
1583 break;
1584
1585 #ifndef TCP_OFFLOAD_DISABLE
1586 case EQ_OFLD:
1587 rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0,
1588 eq->cntxt_id);
1589 break;
1590 #endif
1591
1592 default:
1593 panic("%s: invalid eq type %d.", __func__,
1594 eq->flags & EQ_TYPEMASK);
1595 }
1596 if (rc != 0) {
1597 cxgb_printf(sc->dip, CE_WARN,
1598 "failed to free egress queue (%d): %d",
1599 eq->flags & EQ_TYPEMASK, rc);
1600 return (rc);
1601 }
1602 eq->flags &= ~EQ_ALLOCATED;
1603 }
1604
1605 if (eq->desc != NULL) {
1606 (void) free_desc_ring(&eq->desc_dhdl, &eq->desc_ahdl);
1607 eq->desc = NULL;
1608 }
1609
1610 if (eq->flags & EQ_MTX)
1611 mutex_destroy(&eq->lock);
1612
1613 bzero(eq, sizeof (*eq));
1614 return (0);
1615 }
1616
1617 /* ARGSUSED */
1618 static int
alloc_wrq(struct adapter * sc,struct port_info * pi,struct sge_wrq * wrq,int idx)1619 alloc_wrq(struct adapter *sc, struct port_info *pi, struct sge_wrq *wrq,
1620 int idx)
1621 {
1622 int rc;
1623
1624 rc = alloc_eq(sc, pi, &wrq->eq);
1625 if (rc != 0)
1626 return (rc);
1627
1628 wrq->adapter = sc;
1629 wrq->wr_list.head = NULL;
1630 wrq->wr_list.tail = NULL;
1631
1632 /*
1633 * TODO: use idx to figure out what kind of wrq this is and install
1634 * useful kstats for it.
1635 */
1636
1637 return (rc);
1638 }
1639
1640 static int
free_wrq(struct adapter * sc,struct sge_wrq * wrq)1641 free_wrq(struct adapter *sc, struct sge_wrq *wrq)
1642 {
1643 int rc;
1644
1645 rc = free_eq(sc, &wrq->eq);
1646 if (rc != 0)
1647 return (rc);
1648
1649 bzero(wrq, sizeof (*wrq));
1650 return (0);
1651 }
1652
1653 static int
alloc_txq(struct port_info * pi,struct sge_txq * txq,int idx)1654 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx)
1655 {
1656 int rc, i;
1657 struct adapter *sc = pi->adapter;
1658 struct sge_eq *eq = &txq->eq;
1659
1660 rc = alloc_eq(sc, pi, eq);
1661 if (rc != 0)
1662 return (rc);
1663
1664 txq->port = pi;
1665 txq->sdesc = kmem_zalloc(sizeof (struct tx_sdesc) * eq->cap, KM_SLEEP);
1666 txq->txb_size = eq->qsize * tx_copy_threshold;
1667 rc = alloc_tx_copybuffer(sc, txq->txb_size, &txq->txb_dhdl,
1668 &txq->txb_ahdl, &txq->txb_ba, &txq->txb_va);
1669 if (rc == 0)
1670 txq->txb_avail = txq->txb_size;
1671 else
1672 txq->txb_avail = txq->txb_size = 0;
1673
1674 /*
1675 * TODO: is this too low? Worst case would need around 4 times qsize
1676 * (all tx descriptors filled to the brim with SGLs, with each entry in
1677 * the SGL coming from a distinct DMA handle). Increase tx_dhdl_total
1678 * if you see too many dma_hdl_failed.
1679 */
1680 txq->tx_dhdl_total = eq->qsize * 2;
1681 txq->tx_dhdl = kmem_zalloc(sizeof (ddi_dma_handle_t) *
1682 txq->tx_dhdl_total, KM_SLEEP);
1683 for (i = 0; i < txq->tx_dhdl_total; i++) {
1684 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx,
1685 DDI_DMA_SLEEP, 0, &txq->tx_dhdl[i]);
1686 if (rc != DDI_SUCCESS) {
1687 cxgb_printf(sc->dip, CE_WARN,
1688 "%s: failed to allocate DMA handle (%d)",
1689 __func__, rc);
1690 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
1691 }
1692 txq->tx_dhdl_avail++;
1693 }
1694
1695 txq->ksp = setup_txq_kstats(pi, txq, idx);
1696
1697 return (rc);
1698 }
1699
1700 static int
free_txq(struct port_info * pi,struct sge_txq * txq)1701 free_txq(struct port_info *pi, struct sge_txq *txq)
1702 {
1703 int i;
1704 struct adapter *sc = pi->adapter;
1705 struct sge_eq *eq = &txq->eq;
1706
1707 if (txq->ksp != NULL) {
1708 kstat_delete(txq->ksp);
1709 txq->ksp = NULL;
1710 }
1711
1712 if (txq->txb_va != NULL) {
1713 (void) free_desc_ring(&txq->txb_dhdl, &txq->txb_ahdl);
1714 txq->txb_va = NULL;
1715 }
1716
1717 if (txq->sdesc != NULL) {
1718 struct tx_sdesc *sd;
1719 ddi_dma_handle_t hdl;
1720
1721 TXQ_LOCK(txq);
1722 while (eq->cidx != eq->pidx) {
1723 sd = &txq->sdesc[eq->cidx];
1724
1725 for (i = sd->hdls_used; i; i--) {
1726 hdl = txq->tx_dhdl[txq->tx_dhdl_cidx];
1727 (void) ddi_dma_unbind_handle(hdl);
1728 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total)
1729 txq->tx_dhdl_cidx = 0;
1730 }
1731
1732 ASSERT(sd->m);
1733 freemsgchain(sd->m);
1734
1735 eq->cidx += sd->desc_used;
1736 if (eq->cidx >= eq->cap)
1737 eq->cidx -= eq->cap;
1738
1739 txq->txb_avail += txq->txb_used;
1740 }
1741 ASSERT(txq->tx_dhdl_cidx == txq->tx_dhdl_pidx);
1742 ASSERT(txq->txb_avail == txq->txb_size);
1743 TXQ_UNLOCK(txq);
1744
1745 kmem_free(txq->sdesc, sizeof (struct tx_sdesc) * eq->cap);
1746 txq->sdesc = NULL;
1747 }
1748
1749 if (txq->tx_dhdl != NULL) {
1750 for (i = 0; i < txq->tx_dhdl_total; i++) {
1751 if (txq->tx_dhdl[i] != NULL)
1752 ddi_dma_free_handle(&txq->tx_dhdl[i]);
1753 }
1754 }
1755
1756 (void) free_eq(sc, &txq->eq);
1757
1758 bzero(txq, sizeof (*txq));
1759 return (0);
1760 }
1761
1762 /*
1763 * Allocates a block of contiguous memory for DMA. Can be used to allocate
1764 * memory for descriptor rings or for tx/rx copy buffers.
1765 *
1766 * Caller does not have to clean up anything if this function fails, it cleans
1767 * up after itself.
1768 *
1769 * Caller provides the following:
1770 * len length of the block of memory to allocate.
1771 * flags DDI_DMA_* flags to use (CONSISTENT/STREAMING, READ/WRITE/RDWR)
1772 * acc_attr device access attributes for the allocation.
1773 * dma_attr DMA attributes for the allocation
1774 *
1775 * If the function is successful it fills up this information:
1776 * dma_hdl DMA handle for the allocated memory
1777 * acc_hdl access handle for the allocated memory
1778 * ba bus address of the allocated memory
1779 * va KVA of the allocated memory.
1780 */
1781 static int
alloc_dma_memory(struct adapter * sc,size_t len,int flags,ddi_device_acc_attr_t * acc_attr,ddi_dma_attr_t * dma_attr,ddi_dma_handle_t * dma_hdl,ddi_acc_handle_t * acc_hdl,uint64_t * pba,caddr_t * pva)1782 alloc_dma_memory(struct adapter *sc, size_t len, int flags,
1783 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr,
1784 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
1785 uint64_t *pba, caddr_t *pva)
1786 {
1787 int rc;
1788 ddi_dma_handle_t dhdl;
1789 ddi_acc_handle_t ahdl;
1790 ddi_dma_cookie_t cookie;
1791 uint_t ccount;
1792 caddr_t va;
1793 size_t real_len;
1794
1795 *pva = NULL;
1796
1797 /*
1798 * DMA handle.
1799 */
1800 rc = ddi_dma_alloc_handle(sc->dip, dma_attr, DDI_DMA_SLEEP, 0, &dhdl);
1801 if (rc != DDI_SUCCESS) {
1802 cxgb_printf(sc->dip, CE_WARN,
1803 "failed to allocate DMA handle: %d", rc);
1804
1805 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
1806 }
1807
1808 /*
1809 * Memory suitable for DMA.
1810 */
1811 rc = ddi_dma_mem_alloc(dhdl, len, acc_attr,
1812 flags & DDI_DMA_CONSISTENT ? DDI_DMA_CONSISTENT : DDI_DMA_STREAMING,
1813 DDI_DMA_SLEEP, 0, &va, &real_len, &ahdl);
1814 if (rc != DDI_SUCCESS) {
1815 cxgb_printf(sc->dip, CE_WARN,
1816 "failed to allocate DMA memory: %d", rc);
1817
1818 ddi_dma_free_handle(&dhdl);
1819 return (ENOMEM);
1820 }
1821
1822 if (len != real_len) {
1823 cxgb_printf(sc->dip, CE_WARN,
1824 "%s: len (%u) != real_len (%u)\n", len, real_len);
1825 }
1826
1827 /*
1828 * DMA bindings.
1829 */
1830 rc = ddi_dma_addr_bind_handle(dhdl, NULL, va, real_len, flags, NULL,
1831 NULL, &cookie, &ccount);
1832 if (rc != DDI_DMA_MAPPED) {
1833 cxgb_printf(sc->dip, CE_WARN,
1834 "failed to map DMA memory: %d", rc);
1835
1836 ddi_dma_mem_free(&ahdl);
1837 ddi_dma_free_handle(&dhdl);
1838 return (ENOMEM);
1839 }
1840 if (ccount != 1) {
1841 cxgb_printf(sc->dip, CE_WARN,
1842 "unusable DMA mapping (%d segments)", ccount);
1843 (void) free_desc_ring(&dhdl, &ahdl);
1844 }
1845
1846 bzero(va, real_len);
1847 *dma_hdl = dhdl;
1848 *acc_hdl = ahdl;
1849 *pba = cookie.dmac_laddress;
1850 *pva = va;
1851
1852 return (0);
1853 }
1854
1855 static int
free_dma_memory(ddi_dma_handle_t * dhdl,ddi_acc_handle_t * ahdl)1856 free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl)
1857 {
1858 (void) ddi_dma_unbind_handle(*dhdl);
1859 ddi_dma_mem_free(ahdl);
1860 ddi_dma_free_handle(dhdl);
1861
1862 return (0);
1863 }
1864
1865 static int
alloc_desc_ring(struct adapter * sc,size_t len,int rw,ddi_dma_handle_t * dma_hdl,ddi_acc_handle_t * acc_hdl,uint64_t * pba,caddr_t * pva)1866 alloc_desc_ring(struct adapter *sc, size_t len, int rw,
1867 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
1868 uint64_t *pba, caddr_t *pva)
1869 {
1870 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_desc;
1871 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc;
1872
1873 return (alloc_dma_memory(sc, len, DDI_DMA_CONSISTENT | rw, acc_attr,
1874 dma_attr, dma_hdl, acc_hdl, pba, pva));
1875 }
1876
1877 static int
free_desc_ring(ddi_dma_handle_t * dhdl,ddi_acc_handle_t * ahdl)1878 free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl)
1879 {
1880 return (free_dma_memory(dhdl, ahdl));
1881 }
1882
1883 static int
alloc_tx_copybuffer(struct adapter * sc,size_t len,ddi_dma_handle_t * dma_hdl,ddi_acc_handle_t * acc_hdl,uint64_t * pba,caddr_t * pva)1884 alloc_tx_copybuffer(struct adapter *sc, size_t len,
1885 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
1886 uint64_t *pba, caddr_t *pva)
1887 {
1888 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_tx;
1889 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; /* NOT dma_attr_tx */
1890
1891 return (alloc_dma_memory(sc, len, DDI_DMA_STREAMING | DDI_DMA_WRITE,
1892 acc_attr, dma_attr, dma_hdl, acc_hdl, pba, pva));
1893 }
1894
1895 static inline bool
is_new_response(const struct sge_iq * iq,struct rsp_ctrl ** ctrl)1896 is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl)
1897 {
1898 (void) ddi_dma_sync(iq->dhdl, (uintptr_t)iq->cdesc -
1899 (uintptr_t)iq->desc, iq->esize, DDI_DMA_SYNC_FORKERNEL);
1900
1901 *ctrl = (void *)((uintptr_t)iq->cdesc +
1902 (iq->esize - sizeof (struct rsp_ctrl)));
1903
1904 return ((((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen));
1905 }
1906
1907 static inline void
iq_next(struct sge_iq * iq)1908 iq_next(struct sge_iq *iq)
1909 {
1910 iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize);
1911 if (++iq->cidx == iq->qsize - 1) {
1912 iq->cidx = 0;
1913 iq->gen ^= 1;
1914 iq->cdesc = iq->desc;
1915 }
1916 }
1917
1918 /*
1919 * Fill up the freelist by upto nbufs and maybe ring its doorbell.
1920 *
1921 * Returns non-zero to indicate that it should be added to the list of starving
1922 * freelists.
1923 */
1924 static int
refill_fl(struct adapter * sc,struct sge_fl * fl,int nbufs)1925 refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
1926 {
1927 uint64_t *d = &fl->desc[fl->pidx];
1928 struct fl_sdesc *sd = &fl->sdesc[fl->pidx];
1929
1930 FL_LOCK_ASSERT_OWNED(fl);
1931 ASSERT(nbufs >= 0);
1932
1933 if (nbufs > fl->needed)
1934 nbufs = fl->needed;
1935
1936 while (nbufs--) {
1937 if (sd->rxb != NULL) {
1938 if (sd->rxb->ref_cnt == 1) {
1939 /*
1940 * Buffer is available for recycling. Two ways
1941 * this can happen:
1942 *
1943 * a) All the packets DMA'd into it last time
1944 * around were within the rx_copy_threshold
1945 * and no part of the buffer was ever passed
1946 * up (ref_cnt never went over 1).
1947 *
1948 * b) Packets DMA'd into the buffer were passed
1949 * up but have all been freed by the upper
1950 * layers by now (ref_cnt went over 1 but is
1951 * now back to 1).
1952 *
1953 * Either way the bus address in the descriptor
1954 * ring is already valid.
1955 */
1956 ASSERT(*d == cpu_to_be64(sd->rxb->ba));
1957 d++;
1958 goto recycled;
1959 } else {
1960 /*
1961 * Buffer still in use and we need a
1962 * replacement. But first release our reference
1963 * on the existing buffer.
1964 */
1965 rxbuf_free(sd->rxb);
1966 }
1967 }
1968
1969 sd->rxb = rxbuf_alloc(sc->sge.rxbuf_cache, KM_NOSLEEP, 1);
1970 if (sd->rxb == NULL)
1971 break;
1972 *d++ = cpu_to_be64(sd->rxb->ba);
1973
1974 recycled: fl->pending++;
1975 sd++;
1976 fl->needed--;
1977 if (++fl->pidx == fl->cap) {
1978 fl->pidx = 0;
1979 sd = fl->sdesc;
1980 d = fl->desc;
1981 }
1982 }
1983
1984 if (fl->pending >= 8)
1985 ring_fl_db(sc, fl);
1986
1987 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
1988 }
1989
1990 #ifndef TAILQ_FOREACH_SAFE
1991 #define TAILQ_FOREACH_SAFE(var, head, field, tvar) \
1992 for ((var) = TAILQ_FIRST((head)); \
1993 (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \
1994 (var) = (tvar))
1995 #endif
1996
1997 /*
1998 * Attempt to refill all starving freelists.
1999 */
2000 static void
refill_sfl(void * arg)2001 refill_sfl(void *arg)
2002 {
2003 struct adapter *sc = arg;
2004 struct sge_fl *fl, *fl_temp;
2005
2006 mutex_enter(&sc->sfl_lock);
2007 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) {
2008 FL_LOCK(fl);
2009 (void) refill_fl(sc, fl, 64);
2010 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) {
2011 TAILQ_REMOVE(&sc->sfl, fl, link);
2012 fl->flags &= ~FL_STARVING;
2013 }
2014 FL_UNLOCK(fl);
2015 }
2016
2017 if (!TAILQ_EMPTY(&sc->sfl) != 0)
2018 sc->sfl_timer = timeout(refill_sfl, sc, drv_usectohz(100000));
2019 mutex_exit(&sc->sfl_lock);
2020 }
2021
2022 static void
add_fl_to_sfl(struct adapter * sc,struct sge_fl * fl)2023 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl)
2024 {
2025 mutex_enter(&sc->sfl_lock);
2026 FL_LOCK(fl);
2027 if ((fl->flags & FL_DOOMED) == 0) {
2028 if (TAILQ_EMPTY(&sc->sfl) != 0) {
2029 sc->sfl_timer = timeout(refill_sfl, sc,
2030 drv_usectohz(100000));
2031 }
2032 fl->flags |= FL_STARVING;
2033 TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
2034 }
2035 FL_UNLOCK(fl);
2036 mutex_exit(&sc->sfl_lock);
2037 }
2038
2039 static void
free_fl_bufs(struct sge_fl * fl)2040 free_fl_bufs(struct sge_fl *fl)
2041 {
2042 struct fl_sdesc *sd;
2043 unsigned int i;
2044
2045 FL_LOCK_ASSERT_OWNED(fl);
2046
2047 for (i = 0; i < fl->cap; i++) {
2048 sd = &fl->sdesc[i];
2049
2050 if (sd->rxb != NULL) {
2051 rxbuf_free(sd->rxb);
2052 sd->rxb = NULL;
2053 }
2054 }
2055 }
2056
2057 /*
2058 * Note that fl->cidx and fl->offset are left unchanged in case of failure.
2059 */
2060 static mblk_t *
get_fl_payload(struct sge_fl * fl,uint32_t len_newbuf,int * fl_bufs_used)2061 get_fl_payload(struct sge_fl *fl, uint32_t len_newbuf, int *fl_bufs_used)
2062 {
2063 struct mblk_pair frame = {0};
2064 struct rxbuf *rxb;
2065 mblk_t *m = NULL;
2066 uint_t nbuf = 0, len, copy, n;
2067 uint32_t cidx, offset;
2068
2069 /*
2070 * The SGE won't pack a new frame into the current buffer if the entire
2071 * payload doesn't fit in the remaining space. Move on to the next buf
2072 * in that case.
2073 */
2074 if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) {
2075 fl->offset = 0;
2076 if (++fl->cidx == fl->cap)
2077 fl->cidx = 0;
2078 nbuf++;
2079 }
2080 cidx = fl->cidx;
2081 offset = fl->offset;
2082
2083 len = G_RSPD_LEN(len_newbuf); /* pktshift + payload length */
2084 copy = (len <= fl->copy_threshold);
2085 if (copy != 0) {
2086 frame.head = m = allocb(len, BPRI_HI);
2087 if (m == NULL)
2088 return (NULL);
2089 }
2090
2091 while (len) {
2092 rxb = fl->sdesc[cidx].rxb;
2093 n = min(len, rxb->buf_size - offset);
2094
2095 (void) ddi_dma_sync(rxb->dhdl, offset, n,
2096 DDI_DMA_SYNC_FORKERNEL);
2097
2098 if (copy != 0)
2099 bcopy(rxb->va + offset, m->b_wptr, n);
2100 else {
2101 m = desballoc((unsigned char *)rxb->va + offset, n,
2102 BPRI_HI, &rxb->freefunc);
2103 if (m == NULL) {
2104 freemsg(frame.head);
2105 return (NULL);
2106 }
2107 atomic_inc_uint(&rxb->ref_cnt);
2108 if (frame.head != NULL)
2109 frame.tail->b_cont = m;
2110 else
2111 frame.head = m;
2112 frame.tail = m;
2113 }
2114 m->b_wptr += n;
2115 len -= n;
2116 offset += roundup(n, FL_ALIGN);
2117 ASSERT(offset <= rxb->buf_size);
2118 if (offset == rxb->buf_size) {
2119 offset = 0;
2120 if (++cidx == fl->cap)
2121 cidx = 0;
2122 nbuf++;
2123 }
2124 }
2125
2126 fl->cidx = cidx;
2127 fl->offset = offset;
2128 (*fl_bufs_used) += nbuf;
2129
2130 ASSERT(frame.head != NULL);
2131 return (frame.head);
2132 }
2133
2134 /*
2135 * We'll do immediate data tx for non-LSO, but only when not coalescing. We're
2136 * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes
2137 * of immediate data.
2138 */
2139 #define IMM_LEN ( \
2140 2 * EQ_ESIZE \
2141 - sizeof (struct fw_eth_tx_pkt_wr) \
2142 - sizeof (struct cpl_tx_pkt_core))
2143
2144 /*
2145 * Returns non-zero on failure, no need to cleanup anything in that case.
2146 *
2147 * Note 1: We always try to pull up the mblk if required and return E2BIG only
2148 * if this fails.
2149 *
2150 * Note 2: We'll also pullup incoming mblk if HW_LSO is set and the first mblk
2151 * does not have the TCP header in it.
2152 */
2153 static int
get_frame_txinfo(struct sge_txq * txq,mblk_t ** fp,struct txinfo * txinfo,int sgl_only)2154 get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, struct txinfo *txinfo,
2155 int sgl_only)
2156 {
2157 uint32_t flags = 0, len, n;
2158 mblk_t *m = *fp;
2159 int rc;
2160
2161 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb and dma_hdls */
2162
2163 mac_hcksum_get(m, NULL, NULL, NULL, NULL, &flags);
2164 txinfo->flags = flags;
2165
2166 mac_lso_get(m, &txinfo->mss, &flags);
2167 txinfo->flags |= flags;
2168
2169 if (flags & HW_LSO)
2170 sgl_only = 1; /* Do not allow immediate data with LSO */
2171
2172 start: txinfo->nsegs = 0;
2173 txinfo->hdls_used = 0;
2174 txinfo->txb_used = 0;
2175 txinfo->len = 0;
2176
2177 /* total length and a rough estimate of # of segments */
2178 n = 0;
2179 for (; m; m = m->b_cont) {
2180 len = MBLKL(m);
2181 n += (len / PAGE_SIZE) + 1;
2182 txinfo->len += len;
2183 }
2184 m = *fp;
2185
2186 if (n >= TX_SGL_SEGS || (flags & HW_LSO && MBLKL(m) < 50)) {
2187 txq->pullup_early++;
2188 m = msgpullup(*fp, -1);
2189 if (m == NULL) {
2190 txq->pullup_failed++;
2191 return (E2BIG); /* (*fp) left as it was */
2192 }
2193 freemsg(*fp);
2194 *fp = m;
2195 mac_hcksum_set(m, NULL, NULL, NULL, NULL, txinfo->flags);
2196 }
2197
2198 if (txinfo->len <= IMM_LEN && !sgl_only)
2199 return (0); /* nsegs = 0 tells caller to use imm. tx */
2200
2201 if (txinfo->len <= txq->copy_threshold &&
2202 copy_into_txb(txq, m, txinfo->len, txinfo) == 0)
2203 goto done;
2204
2205 for (; m; m = m->b_cont) {
2206
2207 len = MBLKL(m);
2208
2209 /* Use tx copy buffer if this mblk is small enough */
2210 if (len <= txq->copy_threshold &&
2211 copy_into_txb(txq, m, len, txinfo) == 0)
2212 continue;
2213
2214 /* Add DMA bindings for this mblk to the SGL */
2215 rc = add_mblk(txq, txinfo, m, len);
2216
2217 if (rc == E2BIG ||
2218 (txinfo->nsegs == TX_SGL_SEGS && m->b_cont)) {
2219
2220 txq->pullup_late++;
2221 m = msgpullup(*fp, -1);
2222 if (m != NULL) {
2223 free_txinfo_resources(txq, txinfo);
2224 freemsg(*fp);
2225 *fp = m;
2226 mac_hcksum_set(m, NULL, NULL, NULL, NULL,
2227 txinfo->flags);
2228 goto start;
2229 }
2230
2231 txq->pullup_failed++;
2232 rc = E2BIG;
2233 }
2234
2235 if (rc != 0) {
2236 free_txinfo_resources(txq, txinfo);
2237 return (rc);
2238 }
2239 }
2240
2241 ASSERT(txinfo->nsegs > 0 && txinfo->nsegs <= TX_SGL_SEGS);
2242
2243 done:
2244
2245 /*
2246 * Store the # of flits required to hold this frame's SGL in nflits. An
2247 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by
2248 * multiple (len0 + len1, addr0, addr1) tuples. If addr1 is not used
2249 * then len1 must be set to 0.
2250 */
2251 n = txinfo->nsegs - 1;
2252 txinfo->nflits = (3 * n) / 2 + (n & 1) + 2;
2253 if (n & 1)
2254 txinfo->sgl.sge[n / 2].len[1] = cpu_to_be32(0);
2255
2256 txinfo->sgl.cmd_nsge = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_DSGL) |
2257 V_ULPTX_NSGE(txinfo->nsegs));
2258
2259 return (0);
2260 }
2261
2262 static inline int
fits_in_txb(struct sge_txq * txq,int len,int * waste)2263 fits_in_txb(struct sge_txq *txq, int len, int *waste)
2264 {
2265 if (txq->txb_avail < len)
2266 return (0);
2267
2268 if (txq->txb_next + len <= txq->txb_size) {
2269 *waste = 0;
2270 return (1);
2271 }
2272
2273 *waste = txq->txb_size - txq->txb_next;
2274
2275 return (txq->txb_avail - *waste < len ? 0 : 1);
2276 }
2277
2278 #define TXB_CHUNK 64
2279
2280 /*
2281 * Copies the specified # of bytes into txq's tx copy buffer and updates txinfo
2282 * and txq to indicate resources used. Caller has to make sure that those many
2283 * bytes are available in the mblk chain (b_cont linked).
2284 */
2285 static inline int
copy_into_txb(struct sge_txq * txq,mblk_t * m,int len,struct txinfo * txinfo)2286 copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, struct txinfo *txinfo)
2287 {
2288 int waste, n;
2289
2290 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb */
2291
2292 if (!fits_in_txb(txq, len, &waste)) {
2293 txq->txb_full++;
2294 return (ENOMEM);
2295 }
2296
2297 if (waste != 0) {
2298 ASSERT((waste & (TXB_CHUNK - 1)) == 0);
2299 txinfo->txb_used += waste;
2300 txq->txb_avail -= waste;
2301 txq->txb_next = 0;
2302 }
2303
2304 for (n = 0; n < len; m = m->b_cont) {
2305 bcopy(m->b_rptr, txq->txb_va + txq->txb_next + n, MBLKL(m));
2306 n += MBLKL(m);
2307 }
2308
2309 add_seg(txinfo, txq->txb_ba + txq->txb_next, len);
2310
2311 n = roundup(len, TXB_CHUNK);
2312 txinfo->txb_used += n;
2313 txq->txb_avail -= n;
2314 txq->txb_next += n;
2315 ASSERT(txq->txb_next <= txq->txb_size);
2316 if (txq->txb_next == txq->txb_size)
2317 txq->txb_next = 0;
2318
2319 return (0);
2320 }
2321
2322 static inline void
add_seg(struct txinfo * txinfo,uint64_t ba,uint32_t len)2323 add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len)
2324 {
2325 ASSERT(txinfo->nsegs < TX_SGL_SEGS); /* must have room */
2326
2327 if (txinfo->nsegs != 0) {
2328 int idx = txinfo->nsegs - 1;
2329 txinfo->sgl.sge[idx / 2].len[idx & 1] = cpu_to_be32(len);
2330 txinfo->sgl.sge[idx / 2].addr[idx & 1] = cpu_to_be64(ba);
2331 } else {
2332 txinfo->sgl.len0 = cpu_to_be32(len);
2333 txinfo->sgl.addr0 = cpu_to_be64(ba);
2334 }
2335 txinfo->nsegs++;
2336 }
2337
2338 /*
2339 * This function cleans up any partially allocated resources when it fails so
2340 * there's nothing for the caller to clean up in that case.
2341 *
2342 * EIO indicates permanent failure. Caller should drop the frame containing
2343 * this mblk and continue.
2344 *
2345 * E2BIG indicates that the SGL length for this mblk exceeds the hardware
2346 * limit. Caller should pull up the frame before trying to send it out.
2347 * (This error means our pullup_early heuristic did not work for this frame)
2348 *
2349 * ENOMEM indicates a temporary shortage of resources (DMA handles, other DMA
2350 * resources, etc.). Caller should suspend the tx queue and wait for reclaim to
2351 * free up resources.
2352 */
2353 static inline int
add_mblk(struct sge_txq * txq,struct txinfo * txinfo,mblk_t * m,int len)2354 add_mblk(struct sge_txq *txq, struct txinfo *txinfo, mblk_t *m, int len)
2355 {
2356 ddi_dma_handle_t dhdl;
2357 ddi_dma_cookie_t cookie;
2358 uint_t ccount = 0;
2359 int rc;
2360
2361 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate dhdls */
2362
2363 if (txq->tx_dhdl_avail == 0) {
2364 txq->dma_hdl_failed++;
2365 return (ENOMEM);
2366 }
2367
2368 dhdl = txq->tx_dhdl[txq->tx_dhdl_pidx];
2369 rc = ddi_dma_addr_bind_handle(dhdl, NULL, (caddr_t)m->b_rptr, len,
2370 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie,
2371 &ccount);
2372 if (rc != DDI_DMA_MAPPED) {
2373 txq->dma_map_failed++;
2374
2375 ASSERT(rc != DDI_DMA_INUSE && rc != DDI_DMA_PARTIAL_MAP);
2376
2377 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EIO);
2378 }
2379
2380 if (ccount + txinfo->nsegs > TX_SGL_SEGS) {
2381 (void) ddi_dma_unbind_handle(dhdl);
2382 return (E2BIG);
2383 }
2384
2385 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size);
2386 while (--ccount) {
2387 ddi_dma_nextcookie(dhdl, &cookie);
2388 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size);
2389 }
2390
2391 if (++txq->tx_dhdl_pidx == txq->tx_dhdl_total)
2392 txq->tx_dhdl_pidx = 0;
2393 txq->tx_dhdl_avail--;
2394 txinfo->hdls_used++;
2395
2396 return (0);
2397 }
2398
2399 /*
2400 * Releases all the txq resources used up in the specified txinfo.
2401 */
2402 static void
free_txinfo_resources(struct sge_txq * txq,struct txinfo * txinfo)2403 free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo)
2404 {
2405 int n;
2406
2407 TXQ_LOCK_ASSERT_OWNED(txq); /* dhdls, txb */
2408
2409 n = txinfo->txb_used;
2410 if (n > 0) {
2411 txq->txb_avail += n;
2412 if (n <= txq->txb_next)
2413 txq->txb_next -= n;
2414 else {
2415 n -= txq->txb_next;
2416 txq->txb_next = txq->txb_size - n;
2417 }
2418 }
2419
2420 for (n = txinfo->hdls_used; n > 0; n--) {
2421 if (txq->tx_dhdl_pidx > 0)
2422 txq->tx_dhdl_pidx--;
2423 else
2424 txq->tx_dhdl_pidx = txq->tx_dhdl_total - 1;
2425 txq->tx_dhdl_avail++;
2426 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_pidx]);
2427 }
2428 }
2429
2430 /*
2431 * Returns 0 to indicate that m has been accepted into a coalesced tx work
2432 * request. It has either been folded into txpkts or txpkts was flushed and m
2433 * has started a new coalesced work request (as the first frame in a fresh
2434 * txpkts).
2435 *
2436 * Returns non-zero to indicate a failure - caller is responsible for
2437 * transmitting m, if there was anything in txpkts it has been flushed.
2438 */
2439 static int
add_to_txpkts(struct sge_txq * txq,struct txpkts * txpkts,mblk_t * m,struct txinfo * txinfo)2440 add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m,
2441 struct txinfo *txinfo)
2442 {
2443 struct sge_eq *eq = &txq->eq;
2444 int can_coalesce;
2445 struct tx_sdesc *txsd;
2446 uint8_t flits;
2447
2448 TXQ_LOCK_ASSERT_OWNED(txq);
2449
2450 if (txpkts->npkt > 0) {
2451 flits = TXPKTS_PKT_HDR + txinfo->nflits;
2452 can_coalesce = (txinfo->flags & HW_LSO) == 0 &&
2453 txpkts->nflits + flits <= TX_WR_FLITS &&
2454 txpkts->nflits + flits <= eq->avail * 8 &&
2455 txpkts->plen + txinfo->len < 65536;
2456
2457 if (can_coalesce != 0) {
2458 txpkts->tail->b_next = m;
2459 txpkts->tail = m;
2460 txpkts->npkt++;
2461 txpkts->nflits += flits;
2462 txpkts->plen += txinfo->len;
2463
2464 txsd = &txq->sdesc[eq->pidx];
2465 txsd->txb_used += txinfo->txb_used;
2466 txsd->hdls_used += txinfo->hdls_used;
2467
2468 return (0);
2469 }
2470
2471 /*
2472 * Couldn't coalesce m into txpkts. The first order of business
2473 * is to send txpkts on its way. Then we'll revisit m.
2474 */
2475 write_txpkts_wr(txq, txpkts);
2476 }
2477
2478 /*
2479 * Check if we can start a new coalesced tx work request with m as
2480 * the first packet in it.
2481 */
2482
2483 ASSERT(txpkts->npkt == 0);
2484 ASSERT(txinfo->len < 65536);
2485
2486 flits = TXPKTS_WR_HDR + txinfo->nflits;
2487 can_coalesce = (txinfo->flags & HW_LSO) == 0 &&
2488 flits <= eq->avail * 8 && flits <= TX_WR_FLITS;
2489
2490 if (can_coalesce == 0)
2491 return (EINVAL);
2492
2493 /*
2494 * Start a fresh coalesced tx WR with m as the first frame in it.
2495 */
2496 txpkts->tail = m;
2497 txpkts->npkt = 1;
2498 txpkts->nflits = flits;
2499 txpkts->flitp = &eq->desc[eq->pidx].flit[2];
2500 txpkts->plen = txinfo->len;
2501
2502 txsd = &txq->sdesc[eq->pidx];
2503 txsd->m = m;
2504 txsd->txb_used = txinfo->txb_used;
2505 txsd->hdls_used = txinfo->hdls_used;
2506
2507 return (0);
2508 }
2509
2510 /*
2511 * Note that write_txpkts_wr can never run out of hardware descriptors (but
2512 * write_txpkt_wr can). add_to_txpkts ensures that a frame is accepted for
2513 * coalescing only if sufficient hardware descriptors are available.
2514 */
2515 static void
write_txpkts_wr(struct sge_txq * txq,struct txpkts * txpkts)2516 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts)
2517 {
2518 struct sge_eq *eq = &txq->eq;
2519 struct fw_eth_tx_pkts_wr *wr;
2520 struct tx_sdesc *txsd;
2521 uint32_t ctrl;
2522 uint16_t ndesc;
2523
2524 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */
2525
2526 ndesc = howmany(txpkts->nflits, 8);
2527
2528 wr = (void *)&eq->desc[eq->pidx];
2529 wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR) |
2530 V_FW_WR_IMMDLEN(0)); /* immdlen does not matter in this WR */
2531 ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2));
2532 if (eq->avail == ndesc)
2533 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
2534 wr->equiq_to_len16 = cpu_to_be32(ctrl);
2535 wr->plen = cpu_to_be16(txpkts->plen);
2536 wr->npkt = txpkts->npkt;
2537 wr->r3 = wr->type = 0;
2538
2539 /* Everything else already written */
2540
2541 txsd = &txq->sdesc[eq->pidx];
2542 txsd->desc_used = ndesc;
2543
2544 txq->txb_used += txsd->txb_used / TXB_CHUNK;
2545 txq->hdl_used += txsd->hdls_used;
2546
2547 ASSERT(eq->avail >= ndesc);
2548
2549 eq->pending += ndesc;
2550 eq->avail -= ndesc;
2551 eq->pidx += ndesc;
2552 if (eq->pidx >= eq->cap)
2553 eq->pidx -= eq->cap;
2554
2555 txq->txpkts_pkts += txpkts->npkt;
2556 txq->txpkts_wrs++;
2557 txpkts->npkt = 0; /* emptied */
2558 }
2559
2560 static int
write_txpkt_wr(struct port_info * pi,struct sge_txq * txq,mblk_t * m,struct txinfo * txinfo)2561 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m,
2562 struct txinfo *txinfo)
2563 {
2564 struct sge_eq *eq = &txq->eq;
2565 struct fw_eth_tx_pkt_wr *wr;
2566 struct cpl_tx_pkt_core *cpl;
2567 uint32_t ctrl; /* used in many unrelated places */
2568 uint64_t ctrl1;
2569 int nflits, ndesc;
2570 struct tx_sdesc *txsd;
2571 caddr_t dst;
2572
2573 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */
2574
2575 /*
2576 * Do we have enough flits to send this frame out?
2577 */
2578 ctrl = sizeof (struct cpl_tx_pkt_core);
2579 if (txinfo->flags & HW_LSO) {
2580 nflits = TXPKT_LSO_WR_HDR;
2581 ctrl += sizeof (struct cpl_tx_pkt_lso);
2582 } else
2583 nflits = TXPKT_WR_HDR;
2584 if (txinfo->nsegs > 0)
2585 nflits += txinfo->nflits;
2586 else {
2587 nflits += howmany(txinfo->len, 8);
2588 ctrl += txinfo->len;
2589 }
2590 ndesc = howmany(nflits, 8);
2591 if (ndesc > eq->avail)
2592 return (ENOMEM);
2593
2594 /* Firmware work request header */
2595 wr = (void *)&eq->desc[eq->pidx];
2596 wr->op_immdlen = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
2597 V_FW_WR_IMMDLEN(ctrl));
2598 ctrl = V_FW_WR_LEN16(howmany(nflits, 2));
2599 if (eq->avail == ndesc)
2600 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
2601 wr->equiq_to_len16 = cpu_to_be32(ctrl);
2602 wr->r3 = 0;
2603
2604 if (txinfo->flags & HW_LSO) {
2605 struct cpl_tx_pkt_lso *lso = (void *)(wr + 1);
2606 char *p = (void *)m->b_rptr;
2607 ctrl = V_LSO_OPCODE((u32)CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE |
2608 F_LSO_LAST_SLICE;
2609
2610 /* LINTED: E_BAD_PTR_CAST_ALIGN */
2611 if (((struct ether_header *)p)->ether_type ==
2612 htons(ETHERTYPE_VLAN)) {
2613 ctrl |= V_LSO_ETHHDR_LEN(1);
2614 p += sizeof (struct ether_vlan_header);
2615 } else
2616 p += sizeof (struct ether_header);
2617
2618 /* LINTED: E_BAD_PTR_CAST_ALIGN for IPH_HDR_LENGTH() */
2619 ctrl |= V_LSO_IPHDR_LEN(IPH_HDR_LENGTH(p) / 4);
2620 /* LINTED: E_BAD_PTR_CAST_ALIGN for IPH_HDR_LENGTH() */
2621 p += IPH_HDR_LENGTH(p);
2622 ctrl |= V_LSO_TCPHDR_LEN(TCP_HDR_LENGTH((tcph_t *)p) / 4);
2623
2624 lso->lso_ctrl = cpu_to_be32(ctrl);
2625 lso->ipid_ofst = cpu_to_be16(0);
2626 lso->mss = cpu_to_be16(txinfo->mss);
2627 lso->seqno_offset = cpu_to_be32(0);
2628 lso->len = cpu_to_be32(txinfo->len);
2629
2630 cpl = (void *)(lso + 1);
2631
2632 txq->tso_wrs++;
2633 } else
2634 cpl = (void *)(wr + 1);
2635
2636 /* Checksum offload */
2637 ctrl1 = 0;
2638 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM))
2639 ctrl1 |= F_TXPKT_IPCSUM_DIS;
2640 if (!(txinfo->flags & HCK_FULLCKSUM))
2641 ctrl1 |= F_TXPKT_L4CSUM_DIS;
2642 if (ctrl1 == 0)
2643 txq->txcsum++; /* some hardware assistance provided */
2644
2645 /* CPL header */
2646 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) |
2647 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
2648 cpl->pack = 0;
2649 cpl->len = cpu_to_be16(txinfo->len);
2650 cpl->ctrl1 = cpu_to_be64(ctrl1);
2651
2652 /* Software descriptor */
2653 txsd = &txq->sdesc[eq->pidx];
2654 txsd->m = m;
2655 txsd->txb_used = txinfo->txb_used;
2656 txsd->hdls_used = txinfo->hdls_used;
2657 /* LINTED: E_ASSIGN_NARROW_CONV */
2658 txsd->desc_used = ndesc;
2659
2660 txq->txb_used += txinfo->txb_used / TXB_CHUNK;
2661 txq->hdl_used += txinfo->hdls_used;
2662
2663 eq->pending += ndesc;
2664 eq->avail -= ndesc;
2665 eq->pidx += ndesc;
2666 if (eq->pidx >= eq->cap)
2667 eq->pidx -= eq->cap;
2668
2669 /* SGL */
2670 dst = (void *)(cpl + 1);
2671 if (txinfo->nsegs > 0) {
2672 txq->sgl_wrs++;
2673 copy_to_txd(eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8);
2674
2675 /* Need to zero-pad to a 16 byte boundary if not on one */
2676 if ((uintptr_t)dst & 0xf)
2677 /* LINTED: E_BAD_PTR_CAST_ALIGN */
2678 *(uint64_t *)dst = 0;
2679
2680 } else {
2681 txq->imm_wrs++;
2682 #ifdef DEBUG
2683 ctrl = txinfo->len;
2684 #endif
2685 for (; m; m = m->b_cont) {
2686 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m));
2687 #ifdef DEBUG
2688 ctrl -= MBLKL(m);
2689 #endif
2690 }
2691 ASSERT(ctrl == 0);
2692 }
2693
2694 txq->txpkt_wrs++;
2695 return (0);
2696 }
2697
2698 static inline void
write_ulp_cpl_sgl(struct port_info * pi,struct sge_txq * txq,struct txpkts * txpkts,struct txinfo * txinfo)2699 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
2700 struct txpkts *txpkts, struct txinfo *txinfo)
2701 {
2702 struct ulp_txpkt *ulpmc;
2703 struct ulptx_idata *ulpsc;
2704 struct cpl_tx_pkt_core *cpl;
2705 uintptr_t flitp, start, end;
2706 uint64_t ctrl;
2707 caddr_t dst;
2708
2709 ASSERT(txpkts->npkt > 0);
2710
2711 start = (uintptr_t)txq->eq.desc;
2712 end = (uintptr_t)txq->eq.spg;
2713
2714 /* Checksum offload */
2715 ctrl = 0;
2716 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM))
2717 ctrl |= F_TXPKT_IPCSUM_DIS;
2718 if (!(txinfo->flags & HCK_FULLCKSUM))
2719 ctrl |= F_TXPKT_L4CSUM_DIS;
2720 if (ctrl == 0)
2721 txq->txcsum++; /* some hardware assistance provided */
2722
2723 /*
2724 * The previous packet's SGL must have ended at a 16 byte boundary (this
2725 * is required by the firmware/hardware). It follows that flitp cannot
2726 * wrap around between the ULPTX master command and ULPTX subcommand (8
2727 * bytes each), and that it can not wrap around in the middle of the
2728 * cpl_tx_pkt_core either.
2729 */
2730 flitp = (uintptr_t)txpkts->flitp;
2731 ASSERT((flitp & 0xf) == 0);
2732
2733 /* ULP master command */
2734 ulpmc = (void *)flitp;
2735 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
2736 ulpmc->len = htonl(howmany(sizeof (*ulpmc) + sizeof (*ulpsc) +
2737 sizeof (*cpl) + 8 * txinfo->nflits, 16));
2738
2739 /* ULP subcommand */
2740 ulpsc = (void *)(ulpmc + 1);
2741 ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) |
2742 F_ULP_TX_SC_MORE);
2743 ulpsc->len = cpu_to_be32(sizeof (struct cpl_tx_pkt_core));
2744
2745 flitp += sizeof (*ulpmc) + sizeof (*ulpsc);
2746 if (flitp == end)
2747 flitp = start;
2748
2749 /* CPL_TX_PKT */
2750 cpl = (void *)flitp;
2751 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) |
2752 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
2753 cpl->pack = 0;
2754 cpl->len = cpu_to_be16(txinfo->len);
2755 cpl->ctrl1 = cpu_to_be64(ctrl);
2756
2757 flitp += sizeof (*cpl);
2758 if (flitp == end)
2759 flitp = start;
2760
2761 /* SGL for this frame */
2762 dst = (caddr_t)flitp;
2763 copy_to_txd(&txq->eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8);
2764 flitp = (uintptr_t)dst;
2765
2766 /* Zero pad and advance to a 16 byte boundary if not already at one. */
2767 if (flitp & 0xf) {
2768
2769 /* no matter what, flitp should be on an 8 byte boundary */
2770 ASSERT((flitp & 0x7) == 0);
2771
2772 *(uint64_t *)flitp = 0;
2773 flitp += sizeof (uint64_t);
2774 txpkts->nflits++;
2775 }
2776
2777 if (flitp == end)
2778 flitp = start;
2779
2780 txpkts->flitp = (void *)flitp;
2781 }
2782
2783 static inline void
copy_to_txd(struct sge_eq * eq,caddr_t from,caddr_t * to,int len)2784 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
2785 {
2786 if ((uintptr_t)(*to) + len <= (uintptr_t)eq->spg) {
2787 bcopy(from, *to, len);
2788 (*to) += len;
2789 } else {
2790 int portion = (uintptr_t)eq->spg - (uintptr_t)(*to);
2791
2792 bcopy(from, *to, portion);
2793 from += portion;
2794 portion = len - portion; /* remaining */
2795 bcopy(from, (void *)eq->desc, portion);
2796 (*to) = (caddr_t)eq->desc + portion;
2797 }
2798 }
2799
2800 static inline void
ring_tx_db(struct adapter * sc,struct sge_eq * eq)2801 ring_tx_db(struct adapter *sc, struct sge_eq *eq)
2802 {
2803 if (eq->pending > eq->pidx) {
2804 int offset = eq->cap - (eq->pending - eq->pidx);
2805
2806 /* pidx has wrapped around since last doorbell */
2807
2808 (void) ddi_dma_sync(eq->desc_dhdl,
2809 offset * sizeof (struct tx_desc), 0,
2810 DDI_DMA_SYNC_FORDEV);
2811 (void) ddi_dma_sync(eq->desc_dhdl,
2812 0, eq->pidx * sizeof (struct tx_desc),
2813 DDI_DMA_SYNC_FORDEV);
2814 } else if (eq->pending > 0) {
2815 (void) ddi_dma_sync(eq->desc_dhdl,
2816 (eq->pidx - eq->pending) * sizeof (struct tx_desc),
2817 eq->pending * sizeof (struct tx_desc),
2818 DDI_DMA_SYNC_FORDEV);
2819 }
2820
2821 membar_producer();
2822
2823 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
2824 V_QID(eq->cntxt_id) | V_PIDX(eq->pending));
2825
2826 eq->pending = 0;
2827 }
2828
2829 static int
reclaim_tx_descs(struct sge_txq * txq,int howmany)2830 reclaim_tx_descs(struct sge_txq *txq, int howmany)
2831 {
2832 struct tx_sdesc *txsd;
2833 uint_t cidx, can_reclaim, reclaimed, txb_freed, hdls_freed;
2834 struct sge_eq *eq = &txq->eq;
2835
2836 EQ_LOCK_ASSERT_OWNED(eq);
2837
2838 cidx = eq->spg->cidx; /* stable snapshot */
2839 cidx = be16_to_cpu(cidx);
2840
2841 if (cidx >= eq->cidx)
2842 can_reclaim = cidx - eq->cidx;
2843 else
2844 can_reclaim = cidx + eq->cap - eq->cidx;
2845
2846 if (can_reclaim == 0)
2847 return (0);
2848
2849 txb_freed = hdls_freed = reclaimed = 0;
2850 do {
2851 int ndesc;
2852
2853 txsd = &txq->sdesc[eq->cidx];
2854 ndesc = txsd->desc_used;
2855
2856 /* Firmware doesn't return "partial" credits. */
2857 ASSERT(can_reclaim >= ndesc);
2858
2859 /*
2860 * We always keep mblk around, even for immediate data. If mblk
2861 * is NULL, this has to be the software descriptor for a credit
2862 * flush work request.
2863 */
2864 if (txsd->m != NULL)
2865 freemsgchain(txsd->m);
2866 #ifdef DEBUG
2867 else {
2868 ASSERT(txsd->txb_used == 0);
2869 ASSERT(txsd->hdls_used == 0);
2870 ASSERT(ndesc == 1);
2871 }
2872 #endif
2873
2874 txb_freed += txsd->txb_used;
2875 hdls_freed += txsd->hdls_used;
2876 reclaimed += ndesc;
2877
2878 eq->cidx += ndesc;
2879 if (eq->cidx >= eq->cap)
2880 eq->cidx -= eq->cap;
2881
2882 can_reclaim -= ndesc;
2883
2884 } while (can_reclaim && reclaimed < howmany);
2885
2886 eq->avail += reclaimed;
2887 ASSERT(eq->avail < eq->cap); /* avail tops out at (cap - 1) */
2888
2889 txq->txb_avail += txb_freed;
2890
2891 txq->tx_dhdl_avail += hdls_freed;
2892 ASSERT(txq->tx_dhdl_avail <= txq->tx_dhdl_total);
2893 for (; hdls_freed; hdls_freed--) {
2894 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_cidx]);
2895 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total)
2896 txq->tx_dhdl_cidx = 0;
2897 }
2898
2899 return (reclaimed);
2900 }
2901
2902 static void
write_txqflush_wr(struct sge_txq * txq)2903 write_txqflush_wr(struct sge_txq *txq)
2904 {
2905 struct sge_eq *eq = &txq->eq;
2906 struct fw_eq_flush_wr *wr;
2907 struct tx_sdesc *txsd;
2908
2909 EQ_LOCK_ASSERT_OWNED(eq);
2910 ASSERT(eq->avail > 0);
2911
2912 wr = (void *)&eq->desc[eq->pidx];
2913 bzero(wr, sizeof (*wr));
2914 wr->opcode = FW_EQ_FLUSH_WR;
2915 wr->equiq_to_len16 = cpu_to_be32(V_FW_WR_LEN16(sizeof (*wr) / 16) |
2916 F_FW_WR_EQUEQ | F_FW_WR_EQUIQ);
2917
2918 txsd = &txq->sdesc[eq->pidx];
2919 txsd->m = NULL;
2920 txsd->txb_used = 0;
2921 txsd->hdls_used = 0;
2922 txsd->desc_used = 1;
2923
2924 eq->pending++;
2925 eq->avail--;
2926 if (++eq->pidx == eq->cap)
2927 eq->pidx = 0;
2928 }
2929
2930 static int
t4_eth_rx(struct sge_iq * iq,const struct rss_header * rss,mblk_t * m)2931 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
2932 {
2933 struct sge_rxq *rxq = (void *)iq;
2934 struct mblk_pair chain = {0};
2935 const struct cpl_rx_pkt *cpl = (const void *)(rss + 1);
2936
2937 iq->intr_next = iq->intr_params;
2938
2939 m->b_rptr += FL_PKTSHIFT;
2940
2941 /* TODO: what about cpl->ip_frag? */
2942 if (cpl->csum_calc && !cpl->err_vec && !cpl->ip_frag) {
2943 mac_hcksum_set(m, 0, 0, 0, 0xffff,
2944 HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
2945 HCK_IPV4_HDRCKSUM_OK);
2946 rxq->rxcsum++;
2947 }
2948
2949 /* Add to the chain that we'll send up */
2950 if (chain.head != NULL)
2951 chain.tail->b_next = m;
2952 else
2953 chain.head = m;
2954 chain.tail = m;
2955
2956 t4_mac_rx(rxq->port, rxq, chain.head);
2957
2958 return (0);
2959 }
2960
2961 #define FL_HW_IDX(idx) ((idx) >> 3)
2962
2963 static inline void
ring_fl_db(struct adapter * sc,struct sge_fl * fl)2964 ring_fl_db(struct adapter *sc, struct sge_fl *fl)
2965 {
2966 int desc_start, desc_last, ndesc;
2967
2968 ndesc = FL_HW_IDX(fl->pending);
2969
2970 /* Hold back one credit if pidx = cidx */
2971 if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx))
2972 ndesc--;
2973
2974 /*
2975 * There are chances of ndesc modified above (to avoid pidx = cidx).
2976 * If there is nothing to post, return.
2977 */
2978 if (ndesc <= 0)
2979 return;
2980
2981 desc_last = FL_HW_IDX(fl->pidx);
2982
2983 if (fl->pidx < fl->pending) {
2984 /* There was a wrap */
2985 desc_start = FL_HW_IDX(fl->pidx + fl->cap - fl->pending);
2986
2987 /* From desc_start to the end of list */
2988 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 0,
2989 DDI_DMA_SYNC_FORDEV);
2990
2991 /* From start of list to the desc_last */
2992 if (desc_last != 0)
2993 (void) ddi_dma_sync(fl->dhdl, 0, desc_last *
2994 RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
2995 } else {
2996 /* There was no wrap, sync from start_desc to last_desc */
2997 desc_start = FL_HW_IDX(fl->pidx - fl->pending);
2998 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE,
2999 ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3000 }
3001
3002 membar_producer();
3003
3004 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), F_DBPRIO |
3005 V_QID(fl->cntxt_id) | V_PIDX(ndesc));
3006
3007 /*
3008 * Update pending count:
3009 * Deduct the number of descriptors posted
3010 */
3011 fl->pending -= ndesc * 8;
3012 }
3013
3014 /* ARGSUSED */
3015 static int
handle_fw_rpl(struct sge_iq * iq,const struct rss_header * rss,mblk_t * m)3016 handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
3017 {
3018 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
3019
3020 ASSERT(m == NULL);
3021
3022 if (cpl->type == FW6_TYPE_CMD_RPL)
3023 (void) t4_handle_fw_rpl(iq->adapter, cpl->data);
3024
3025 return (0);
3026 }
3027
3028 int
t4_alloc_tx_maps(struct adapter * sc,struct tx_maps * txmaps,int count,int flags)3029 t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count,
3030 int flags)
3031 {
3032 int i, rc;
3033
3034 txmaps->map_total = count;
3035 txmaps->map_avail = txmaps->map_cidx = txmaps->map_pidx = 0;
3036
3037 txmaps->map = kmem_zalloc(sizeof (ddi_dma_handle_t) *
3038 txmaps->map_total, flags);
3039
3040 for (i = 0; i < count; i++) {
3041 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx,
3042 DDI_DMA_SLEEP, 0, &txmaps->map[i]);
3043 if (rc != DDI_SUCCESS) {
3044 cxgb_printf(sc->dip, CE_WARN,
3045 "%s: failed to allocate DMA handle (%d)",
3046 __func__, rc);
3047 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
3048 }
3049 txmaps->map_avail++;
3050 }
3051
3052 return (0);
3053 }
3054
3055 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG)
3056 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR)
3057 #define KS_U_SET(x, y) kstatp->x.value.ul = (y)
3058 #define KS_U_FROM(x, y) kstatp->x.value.ul = (y)->x
3059 #define KS_C_SET(x, ...) \
3060 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__)
3061
3062 /*
3063 * cxgbe:X:config
3064 */
3065 struct cxgbe_port_config_kstats {
3066 kstat_named_t idx;
3067 kstat_named_t nrxq;
3068 kstat_named_t ntxq;
3069 kstat_named_t first_rxq;
3070 kstat_named_t first_txq;
3071 kstat_named_t controller;
3072 kstat_named_t factory_mac_address;
3073 };
3074
3075 /*
3076 * cxgbe:X:info
3077 */
3078 struct cxgbe_port_info_kstats {
3079 kstat_named_t transceiver;
3080 kstat_named_t rx_ovflow0;
3081 kstat_named_t rx_ovflow1;
3082 kstat_named_t rx_ovflow2;
3083 kstat_named_t rx_ovflow3;
3084 kstat_named_t rx_trunc0;
3085 kstat_named_t rx_trunc1;
3086 kstat_named_t rx_trunc2;
3087 kstat_named_t rx_trunc3;
3088 kstat_named_t tx_pause;
3089 kstat_named_t rx_pause;
3090 };
3091
3092 static kstat_t *
setup_port_config_kstats(struct port_info * pi)3093 setup_port_config_kstats(struct port_info *pi)
3094 {
3095 kstat_t *ksp;
3096 struct cxgbe_port_config_kstats *kstatp;
3097 int ndata;
3098 dev_info_t *pdip = ddi_get_parent(pi->dip);
3099 uint8_t *ma = &pi->hw_addr[0];
3100
3101 ndata = sizeof (struct cxgbe_port_config_kstats) /
3102 sizeof (kstat_named_t);
3103
3104 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "config",
3105 "net", KSTAT_TYPE_NAMED, ndata, 0);
3106 if (ksp == NULL) {
3107 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats.");
3108 return (NULL);
3109 }
3110
3111 kstatp = (struct cxgbe_port_config_kstats *)ksp->ks_data;
3112
3113 KS_UINIT(idx);
3114 KS_UINIT(nrxq);
3115 KS_UINIT(ntxq);
3116 KS_UINIT(first_rxq);
3117 KS_UINIT(first_txq);
3118 KS_CINIT(controller);
3119 KS_CINIT(factory_mac_address);
3120
3121 KS_U_SET(idx, pi->port_id);
3122 KS_U_SET(nrxq, pi->nrxq);
3123 KS_U_SET(ntxq, pi->ntxq);
3124 KS_U_SET(first_rxq, pi->first_rxq);
3125 KS_U_SET(first_txq, pi->first_txq);
3126 KS_C_SET(controller, "%s%d", ddi_driver_name(pdip),
3127 ddi_get_instance(pdip));
3128 KS_C_SET(factory_mac_address, "%02X%02X%02X%02X%02X%02X",
3129 ma[0], ma[1], ma[2], ma[3], ma[4], ma[5]);
3130
3131 /* Do NOT set ksp->ks_update. These kstats do not change. */
3132
3133 /* Install the kstat */
3134 ksp->ks_private = (void *)pi;
3135 kstat_install(ksp);
3136
3137 return (ksp);
3138 }
3139
3140 static kstat_t *
setup_port_info_kstats(struct port_info * pi)3141 setup_port_info_kstats(struct port_info *pi)
3142 {
3143 kstat_t *ksp;
3144 struct cxgbe_port_info_kstats *kstatp;
3145 int ndata;
3146
3147 ndata = sizeof (struct cxgbe_port_info_kstats) / sizeof (kstat_named_t);
3148
3149 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "info",
3150 "net", KSTAT_TYPE_NAMED, ndata, 0);
3151 if (ksp == NULL) {
3152 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats.");
3153 return (NULL);
3154 }
3155
3156 kstatp = (struct cxgbe_port_info_kstats *)ksp->ks_data;
3157
3158 KS_CINIT(transceiver);
3159 KS_UINIT(rx_ovflow0);
3160 KS_UINIT(rx_ovflow1);
3161 KS_UINIT(rx_ovflow2);
3162 KS_UINIT(rx_ovflow3);
3163 KS_UINIT(rx_trunc0);
3164 KS_UINIT(rx_trunc1);
3165 KS_UINIT(rx_trunc2);
3166 KS_UINIT(rx_trunc3);
3167 KS_UINIT(tx_pause);
3168 KS_UINIT(rx_pause);
3169
3170 /* Install the kstat */
3171 ksp->ks_update = update_port_info_kstats;
3172 ksp->ks_private = (void *)pi;
3173 kstat_install(ksp);
3174
3175 return (ksp);
3176 }
3177
3178 static int
update_port_info_kstats(kstat_t * ksp,int rw)3179 update_port_info_kstats(kstat_t *ksp, int rw)
3180 {
3181 struct cxgbe_port_info_kstats *kstatp =
3182 (struct cxgbe_port_info_kstats *)ksp->ks_data;
3183 struct port_info *pi = ksp->ks_private;
3184 static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX",
3185 "active TWINAX", "LRM" };
3186 uint32_t bgmap;
3187
3188 if (rw == KSTAT_WRITE)
3189 return (0);
3190
3191 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
3192 KS_C_SET(transceiver, "unplugged");
3193 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
3194 KS_C_SET(transceiver, "unknown");
3195 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
3196 KS_C_SET(transceiver, "unsupported");
3197 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str))
3198 KS_C_SET(transceiver, "%s", mod_str[pi->mod_type]);
3199 else
3200 KS_C_SET(transceiver, "type %d", pi->mod_type);
3201
3202 #define GET_STAT(name) t4_read_reg64(pi->adapter, \
3203 PORT_REG(pi->port_id, A_MPS_PORT_STAT_##name##_L))
3204 #define GET_STAT_COM(name) t4_read_reg64(pi->adapter, \
3205 A_MPS_STAT_##name##_L)
3206
3207 bgmap = G_NUMPORTS(t4_read_reg(pi->adapter, A_MPS_CMN_CTL));
3208 if (bgmap == 0)
3209 bgmap = (pi->port_id == 0) ? 0xf : 0;
3210 else if (bgmap == 1)
3211 bgmap = (pi->port_id < 2) ? (3 << (2 * pi->port_id)) : 0;
3212 else
3213 bgmap = 1;
3214
3215 KS_U_SET(rx_ovflow0, (bgmap & 1) ?
3216 GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0);
3217 KS_U_SET(rx_ovflow1, (bgmap & 2) ?
3218 GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0);
3219 KS_U_SET(rx_ovflow2, (bgmap & 4) ?
3220 GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0);
3221 KS_U_SET(rx_ovflow3, (bgmap & 8) ?
3222 GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0);
3223 KS_U_SET(rx_trunc0, (bgmap & 1) ?
3224 GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0);
3225 KS_U_SET(rx_trunc1, (bgmap & 2) ?
3226 GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0);
3227 KS_U_SET(rx_trunc2, (bgmap & 4) ?
3228 GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0);
3229 KS_U_SET(rx_trunc3, (bgmap & 8) ?
3230 GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0);
3231
3232 KS_U_SET(tx_pause, GET_STAT(TX_PORT_PAUSE));
3233 KS_U_SET(rx_pause, GET_STAT(RX_PORT_PAUSE));
3234
3235 return (0);
3236
3237 }
3238
3239 /*
3240 * cxgbe:X:rxqY
3241 */
3242 struct rxq_kstats {
3243 kstat_named_t rxcsum;
3244 kstat_named_t nomem;
3245 };
3246
3247 static kstat_t *
setup_rxq_kstats(struct port_info * pi,struct sge_rxq * rxq,int idx)3248 setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, int idx)
3249 {
3250 struct kstat *ksp;
3251 struct rxq_kstats *kstatp;
3252 int ndata;
3253 char str[16];
3254
3255 ndata = sizeof (struct rxq_kstats) / sizeof (kstat_named_t);
3256 (void) snprintf(str, sizeof (str), "rxq%u", idx);
3257
3258 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "rxq",
3259 KSTAT_TYPE_NAMED, ndata, 0);
3260 if (ksp == NULL) {
3261 cxgb_printf(pi->dip, CE_WARN,
3262 "%s: failed to initialize rxq kstats for queue %d.",
3263 __func__, idx);
3264 return (NULL);
3265 }
3266
3267 kstatp = (struct rxq_kstats *)ksp->ks_data;
3268
3269 KS_UINIT(rxcsum);
3270 KS_UINIT(nomem);
3271
3272 ksp->ks_update = update_rxq_kstats;
3273 ksp->ks_private = (void *)rxq;
3274 kstat_install(ksp);
3275
3276 return (ksp);
3277 }
3278
3279 static int
update_rxq_kstats(kstat_t * ksp,int rw)3280 update_rxq_kstats(kstat_t *ksp, int rw)
3281 {
3282 struct rxq_kstats *kstatp = (struct rxq_kstats *)ksp->ks_data;
3283 struct sge_rxq *rxq = ksp->ks_private;
3284
3285 if (rw == KSTAT_WRITE)
3286 return (0);
3287
3288 KS_U_FROM(rxcsum, rxq);
3289 KS_U_FROM(nomem, rxq);
3290
3291 return (0);
3292 }
3293
3294 /*
3295 * cxgbe:X:txqY
3296 */
3297 struct txq_kstats {
3298 kstat_named_t txcsum;
3299 kstat_named_t tso_wrs;
3300 kstat_named_t imm_wrs;
3301 kstat_named_t sgl_wrs;
3302 kstat_named_t txpkt_wrs;
3303 kstat_named_t txpkts_wrs;
3304 kstat_named_t txpkts_pkts;
3305 kstat_named_t txb_used;
3306 kstat_named_t hdl_used;
3307 kstat_named_t txb_full;
3308 kstat_named_t dma_hdl_failed;
3309 kstat_named_t dma_map_failed;
3310 kstat_named_t qfull;
3311 kstat_named_t qflush;
3312 kstat_named_t pullup_early;
3313 kstat_named_t pullup_late;
3314 kstat_named_t pullup_failed;
3315 };
3316
3317 static kstat_t *
setup_txq_kstats(struct port_info * pi,struct sge_txq * txq,int idx)3318 setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, int idx)
3319 {
3320 struct kstat *ksp;
3321 struct txq_kstats *kstatp;
3322 int ndata;
3323 char str[16];
3324
3325 ndata = sizeof (struct txq_kstats) / sizeof (kstat_named_t);
3326 (void) snprintf(str, sizeof (str), "txq%u", idx);
3327
3328 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "txq",
3329 KSTAT_TYPE_NAMED, ndata, 0);
3330 if (ksp == NULL) {
3331 cxgb_printf(pi->dip, CE_WARN,
3332 "%s: failed to initialize txq kstats for queue %d.",
3333 __func__, idx);
3334 return (NULL);
3335 }
3336
3337 kstatp = (struct txq_kstats *)ksp->ks_data;
3338
3339 KS_UINIT(txcsum);
3340 KS_UINIT(tso_wrs);
3341 KS_UINIT(imm_wrs);
3342 KS_UINIT(sgl_wrs);
3343 KS_UINIT(txpkt_wrs);
3344 KS_UINIT(txpkts_wrs);
3345 KS_UINIT(txpkts_pkts);
3346 KS_UINIT(txb_used);
3347 KS_UINIT(hdl_used);
3348 KS_UINIT(txb_full);
3349 KS_UINIT(dma_hdl_failed);
3350 KS_UINIT(dma_map_failed);
3351 KS_UINIT(qfull);
3352 KS_UINIT(qflush);
3353 KS_UINIT(pullup_early);
3354 KS_UINIT(pullup_late);
3355 KS_UINIT(pullup_failed);
3356
3357 ksp->ks_update = update_txq_kstats;
3358 ksp->ks_private = (void *)txq;
3359 kstat_install(ksp);
3360
3361 return (ksp);
3362 }
3363
3364 static int
update_txq_kstats(kstat_t * ksp,int rw)3365 update_txq_kstats(kstat_t *ksp, int rw)
3366 {
3367 struct txq_kstats *kstatp = (struct txq_kstats *)ksp->ks_data;
3368 struct sge_txq *txq = ksp->ks_private;
3369
3370 if (rw == KSTAT_WRITE)
3371 return (0);
3372
3373 KS_U_FROM(txcsum, txq);
3374 KS_U_FROM(tso_wrs, txq);
3375 KS_U_FROM(imm_wrs, txq);
3376 KS_U_FROM(sgl_wrs, txq);
3377 KS_U_FROM(txpkt_wrs, txq);
3378 KS_U_FROM(txpkts_wrs, txq);
3379 KS_U_FROM(txpkts_pkts, txq);
3380 KS_U_FROM(txb_used, txq);
3381 KS_U_FROM(hdl_used, txq);
3382 KS_U_FROM(txb_full, txq);
3383 KS_U_FROM(dma_hdl_failed, txq);
3384 KS_U_FROM(dma_map_failed, txq);
3385 KS_U_FROM(qfull, txq);
3386 KS_U_FROM(qflush, txq);
3387 KS_U_FROM(pullup_early, txq);
3388 KS_U_FROM(pullup_late, txq);
3389 KS_U_FROM(pullup_failed, txq);
3390
3391 return (0);
3392 }
3393