1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2007-2016 Solarflare Communications Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright notice,
11 * this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
26 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * The views and conclusions contained in the software and documentation are
29 * those of the authors and should not be interpreted as representing official
30 * policies, either expressed or implied, of the FreeBSD Project.
31 */
32
33 #include <sys/cdefs.h>
34 #include "efx.h"
35 #include "efx_impl.h"
36
37 #if EFSYS_OPT_QSTATS
38 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
39 do { \
40 (_etp)->et_stat[_stat]++; \
41 _NOTE(CONSTANTCONDITION) \
42 } while (B_FALSE)
43 #else
44 #define EFX_TX_QSTAT_INCR(_etp, _stat)
45 #endif
46
47 #if EFSYS_OPT_SIENA
48
49 static __checkReturn efx_rc_t
50 siena_tx_init(
51 __in efx_nic_t *enp);
52
53 static void
54 siena_tx_fini(
55 __in efx_nic_t *enp);
56
57 static __checkReturn efx_rc_t
58 siena_tx_qcreate(
59 __in efx_nic_t *enp,
60 __in unsigned int index,
61 __in unsigned int label,
62 __in efsys_mem_t *esmp,
63 __in size_t ndescs,
64 __in uint32_t id,
65 __in uint16_t flags,
66 __in efx_evq_t *eep,
67 __in efx_txq_t *etp,
68 __out unsigned int *addedp);
69
70 static void
71 siena_tx_qdestroy(
72 __in efx_txq_t *etp);
73
74 static __checkReturn efx_rc_t
75 siena_tx_qpost(
76 __in efx_txq_t *etp,
77 __in_ecount(ndescs) efx_buffer_t *eb,
78 __in unsigned int ndescs,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
81
82 static void
83 siena_tx_qpush(
84 __in efx_txq_t *etp,
85 __in unsigned int added,
86 __in unsigned int pushed);
87
88 static __checkReturn efx_rc_t
89 siena_tx_qpace(
90 __in efx_txq_t *etp,
91 __in unsigned int ns);
92
93 static __checkReturn efx_rc_t
94 siena_tx_qflush(
95 __in efx_txq_t *etp);
96
97 static void
98 siena_tx_qenable(
99 __in efx_txq_t *etp);
100
101 __checkReturn efx_rc_t
102 siena_tx_qdesc_post(
103 __in efx_txq_t *etp,
104 __in_ecount(ndescs) efx_desc_t *ed,
105 __in unsigned int ndescs,
106 __in unsigned int completed,
107 __inout unsigned int *addedp);
108
109 void
110 siena_tx_qdesc_dma_create(
111 __in efx_txq_t *etp,
112 __in efsys_dma_addr_t addr,
113 __in size_t size,
114 __in boolean_t eop,
115 __out efx_desc_t *edp);
116
117 #if EFSYS_OPT_QSTATS
118 static void
119 siena_tx_qstats_update(
120 __in efx_txq_t *etp,
121 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
122 #endif
123
124 #endif /* EFSYS_OPT_SIENA */
125
126 #if EFSYS_OPT_SIENA
127 static const efx_tx_ops_t __efx_tx_siena_ops = {
128 siena_tx_init, /* etxo_init */
129 siena_tx_fini, /* etxo_fini */
130 siena_tx_qcreate, /* etxo_qcreate */
131 siena_tx_qdestroy, /* etxo_qdestroy */
132 siena_tx_qpost, /* etxo_qpost */
133 siena_tx_qpush, /* etxo_qpush */
134 siena_tx_qpace, /* etxo_qpace */
135 siena_tx_qflush, /* etxo_qflush */
136 siena_tx_qenable, /* etxo_qenable */
137 NULL, /* etxo_qpio_enable */
138 NULL, /* etxo_qpio_disable */
139 NULL, /* etxo_qpio_write */
140 NULL, /* etxo_qpio_post */
141 siena_tx_qdesc_post, /* etxo_qdesc_post */
142 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
143 NULL, /* etxo_qdesc_tso_create */
144 NULL, /* etxo_qdesc_tso2_create */
145 NULL, /* etxo_qdesc_vlantci_create */
146 NULL, /* etxo_qdesc_checksum_create */
147 #if EFSYS_OPT_QSTATS
148 siena_tx_qstats_update, /* etxo_qstats_update */
149 #endif
150 };
151 #endif /* EFSYS_OPT_SIENA */
152
153 #if EFSYS_OPT_HUNTINGTON
154 static const efx_tx_ops_t __efx_tx_hunt_ops = {
155 ef10_tx_init, /* etxo_init */
156 ef10_tx_fini, /* etxo_fini */
157 ef10_tx_qcreate, /* etxo_qcreate */
158 ef10_tx_qdestroy, /* etxo_qdestroy */
159 ef10_tx_qpost, /* etxo_qpost */
160 ef10_tx_qpush, /* etxo_qpush */
161 ef10_tx_qpace, /* etxo_qpace */
162 ef10_tx_qflush, /* etxo_qflush */
163 ef10_tx_qenable, /* etxo_qenable */
164 ef10_tx_qpio_enable, /* etxo_qpio_enable */
165 ef10_tx_qpio_disable, /* etxo_qpio_disable */
166 ef10_tx_qpio_write, /* etxo_qpio_write */
167 ef10_tx_qpio_post, /* etxo_qpio_post */
168 ef10_tx_qdesc_post, /* etxo_qdesc_post */
169 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
170 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
171 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
172 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
173 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
174 #if EFSYS_OPT_QSTATS
175 ef10_tx_qstats_update, /* etxo_qstats_update */
176 #endif
177 };
178 #endif /* EFSYS_OPT_HUNTINGTON */
179
180 #if EFSYS_OPT_MEDFORD
181 static const efx_tx_ops_t __efx_tx_medford_ops = {
182 ef10_tx_init, /* etxo_init */
183 ef10_tx_fini, /* etxo_fini */
184 ef10_tx_qcreate, /* etxo_qcreate */
185 ef10_tx_qdestroy, /* etxo_qdestroy */
186 ef10_tx_qpost, /* etxo_qpost */
187 ef10_tx_qpush, /* etxo_qpush */
188 ef10_tx_qpace, /* etxo_qpace */
189 ef10_tx_qflush, /* etxo_qflush */
190 ef10_tx_qenable, /* etxo_qenable */
191 ef10_tx_qpio_enable, /* etxo_qpio_enable */
192 ef10_tx_qpio_disable, /* etxo_qpio_disable */
193 ef10_tx_qpio_write, /* etxo_qpio_write */
194 ef10_tx_qpio_post, /* etxo_qpio_post */
195 ef10_tx_qdesc_post, /* etxo_qdesc_post */
196 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
197 NULL, /* etxo_qdesc_tso_create */
198 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
199 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
200 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
201 #if EFSYS_OPT_QSTATS
202 ef10_tx_qstats_update, /* etxo_qstats_update */
203 #endif
204 };
205 #endif /* EFSYS_OPT_MEDFORD */
206
207 #if EFSYS_OPT_MEDFORD2
208 static const efx_tx_ops_t __efx_tx_medford2_ops = {
209 ef10_tx_init, /* etxo_init */
210 ef10_tx_fini, /* etxo_fini */
211 ef10_tx_qcreate, /* etxo_qcreate */
212 ef10_tx_qdestroy, /* etxo_qdestroy */
213 ef10_tx_qpost, /* etxo_qpost */
214 ef10_tx_qpush, /* etxo_qpush */
215 ef10_tx_qpace, /* etxo_qpace */
216 ef10_tx_qflush, /* etxo_qflush */
217 ef10_tx_qenable, /* etxo_qenable */
218 ef10_tx_qpio_enable, /* etxo_qpio_enable */
219 ef10_tx_qpio_disable, /* etxo_qpio_disable */
220 ef10_tx_qpio_write, /* etxo_qpio_write */
221 ef10_tx_qpio_post, /* etxo_qpio_post */
222 ef10_tx_qdesc_post, /* etxo_qdesc_post */
223 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
224 NULL, /* etxo_qdesc_tso_create */
225 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
226 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
227 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
228 #if EFSYS_OPT_QSTATS
229 ef10_tx_qstats_update, /* etxo_qstats_update */
230 #endif
231 };
232 #endif /* EFSYS_OPT_MEDFORD2 */
233
234 __checkReturn efx_rc_t
efx_tx_init(__in efx_nic_t * enp)235 efx_tx_init(
236 __in efx_nic_t *enp)
237 {
238 const efx_tx_ops_t *etxop;
239 efx_rc_t rc;
240
241 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
242 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
243
244 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
245 rc = EINVAL;
246 goto fail1;
247 }
248
249 if (enp->en_mod_flags & EFX_MOD_TX) {
250 rc = EINVAL;
251 goto fail2;
252 }
253
254 switch (enp->en_family) {
255 #if EFSYS_OPT_SIENA
256 case EFX_FAMILY_SIENA:
257 etxop = &__efx_tx_siena_ops;
258 break;
259 #endif /* EFSYS_OPT_SIENA */
260
261 #if EFSYS_OPT_HUNTINGTON
262 case EFX_FAMILY_HUNTINGTON:
263 etxop = &__efx_tx_hunt_ops;
264 break;
265 #endif /* EFSYS_OPT_HUNTINGTON */
266
267 #if EFSYS_OPT_MEDFORD
268 case EFX_FAMILY_MEDFORD:
269 etxop = &__efx_tx_medford_ops;
270 break;
271 #endif /* EFSYS_OPT_MEDFORD */
272
273 #if EFSYS_OPT_MEDFORD2
274 case EFX_FAMILY_MEDFORD2:
275 etxop = &__efx_tx_medford2_ops;
276 break;
277 #endif /* EFSYS_OPT_MEDFORD2 */
278
279 default:
280 EFSYS_ASSERT(0);
281 rc = ENOTSUP;
282 goto fail3;
283 }
284
285 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
286
287 if ((rc = etxop->etxo_init(enp)) != 0)
288 goto fail4;
289
290 enp->en_etxop = etxop;
291 enp->en_mod_flags |= EFX_MOD_TX;
292 return (0);
293
294 fail4:
295 EFSYS_PROBE(fail4);
296 fail3:
297 EFSYS_PROBE(fail3);
298 fail2:
299 EFSYS_PROBE(fail2);
300 fail1:
301 EFSYS_PROBE1(fail1, efx_rc_t, rc);
302
303 enp->en_etxop = NULL;
304 enp->en_mod_flags &= ~EFX_MOD_TX;
305 return (rc);
306 }
307
308 void
efx_tx_fini(__in efx_nic_t * enp)309 efx_tx_fini(
310 __in efx_nic_t *enp)
311 {
312 const efx_tx_ops_t *etxop = enp->en_etxop;
313
314 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
315 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
316 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
317 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
318
319 etxop->etxo_fini(enp);
320
321 enp->en_etxop = NULL;
322 enp->en_mod_flags &= ~EFX_MOD_TX;
323 }
324
325 __checkReturn efx_rc_t
efx_tx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in uint16_t flags,__in efx_evq_t * eep,__deref_out efx_txq_t ** etpp,__out unsigned int * addedp)326 efx_tx_qcreate(
327 __in efx_nic_t *enp,
328 __in unsigned int index,
329 __in unsigned int label,
330 __in efsys_mem_t *esmp,
331 __in size_t ndescs,
332 __in uint32_t id,
333 __in uint16_t flags,
334 __in efx_evq_t *eep,
335 __deref_out efx_txq_t **etpp,
336 __out unsigned int *addedp)
337 {
338 const efx_tx_ops_t *etxop = enp->en_etxop;
339 efx_txq_t *etp;
340 efx_rc_t rc;
341
342 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
343 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
344
345 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <,
346 enp->en_nic_cfg.enc_txq_limit);
347
348 /* Allocate an TXQ object */
349 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
350
351 if (etp == NULL) {
352 rc = ENOMEM;
353 goto fail1;
354 }
355
356 etp->et_magic = EFX_TXQ_MAGIC;
357 etp->et_enp = enp;
358 etp->et_index = index;
359 etp->et_mask = ndescs - 1;
360 etp->et_esmp = esmp;
361
362 /* Initial descriptor index may be modified by etxo_qcreate */
363 *addedp = 0;
364
365 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
366 ndescs, id, flags, eep, etp, addedp)) != 0)
367 goto fail2;
368
369 enp->en_tx_qcount++;
370 *etpp = etp;
371
372 return (0);
373
374 fail2:
375 EFSYS_PROBE(fail2);
376 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
377 fail1:
378 EFSYS_PROBE1(fail1, efx_rc_t, rc);
379 return (rc);
380 }
381
382 void
efx_tx_qdestroy(__in efx_txq_t * etp)383 efx_tx_qdestroy(
384 __in efx_txq_t *etp)
385 {
386 efx_nic_t *enp = etp->et_enp;
387 const efx_tx_ops_t *etxop = enp->en_etxop;
388
389 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
390
391 EFSYS_ASSERT(enp->en_tx_qcount != 0);
392 --enp->en_tx_qcount;
393
394 etxop->etxo_qdestroy(etp);
395
396 /* Free the TXQ object */
397 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
398 }
399
400 __checkReturn efx_rc_t
efx_tx_qpost(__in efx_txq_t * etp,__in_ecount (ndescs)efx_buffer_t * eb,__in unsigned int ndescs,__in unsigned int completed,__inout unsigned int * addedp)401 efx_tx_qpost(
402 __in efx_txq_t *etp,
403 __in_ecount(ndescs) efx_buffer_t *eb,
404 __in unsigned int ndescs,
405 __in unsigned int completed,
406 __inout unsigned int *addedp)
407 {
408 efx_nic_t *enp = etp->et_enp;
409 const efx_tx_ops_t *etxop = enp->en_etxop;
410 efx_rc_t rc;
411
412 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
413
414 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0)
415 goto fail1;
416
417 return (0);
418
419 fail1:
420 EFSYS_PROBE1(fail1, efx_rc_t, rc);
421 return (rc);
422 }
423
424 void
efx_tx_qpush(__in efx_txq_t * etp,__in unsigned int added,__in unsigned int pushed)425 efx_tx_qpush(
426 __in efx_txq_t *etp,
427 __in unsigned int added,
428 __in unsigned int pushed)
429 {
430 efx_nic_t *enp = etp->et_enp;
431 const efx_tx_ops_t *etxop = enp->en_etxop;
432
433 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
434
435 etxop->etxo_qpush(etp, added, pushed);
436 }
437
438 __checkReturn efx_rc_t
efx_tx_qpace(__in efx_txq_t * etp,__in unsigned int ns)439 efx_tx_qpace(
440 __in efx_txq_t *etp,
441 __in unsigned int ns)
442 {
443 efx_nic_t *enp = etp->et_enp;
444 const efx_tx_ops_t *etxop = enp->en_etxop;
445 efx_rc_t rc;
446
447 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
448
449 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
450 goto fail1;
451
452 return (0);
453
454 fail1:
455 EFSYS_PROBE1(fail1, efx_rc_t, rc);
456 return (rc);
457 }
458
459 __checkReturn efx_rc_t
efx_tx_qflush(__in efx_txq_t * etp)460 efx_tx_qflush(
461 __in efx_txq_t *etp)
462 {
463 efx_nic_t *enp = etp->et_enp;
464 const efx_tx_ops_t *etxop = enp->en_etxop;
465 efx_rc_t rc;
466
467 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
468
469 if ((rc = etxop->etxo_qflush(etp)) != 0)
470 goto fail1;
471
472 return (0);
473
474 fail1:
475 EFSYS_PROBE1(fail1, efx_rc_t, rc);
476 return (rc);
477 }
478
479 void
efx_tx_qenable(__in efx_txq_t * etp)480 efx_tx_qenable(
481 __in efx_txq_t *etp)
482 {
483 efx_nic_t *enp = etp->et_enp;
484 const efx_tx_ops_t *etxop = enp->en_etxop;
485
486 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
487
488 etxop->etxo_qenable(etp);
489 }
490
491 __checkReturn efx_rc_t
efx_tx_qpio_enable(__in efx_txq_t * etp)492 efx_tx_qpio_enable(
493 __in efx_txq_t *etp)
494 {
495 efx_nic_t *enp = etp->et_enp;
496 const efx_tx_ops_t *etxop = enp->en_etxop;
497 efx_rc_t rc;
498
499 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
500
501 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
502 rc = ENOTSUP;
503 goto fail1;
504 }
505 if (etxop->etxo_qpio_enable == NULL) {
506 rc = ENOTSUP;
507 goto fail2;
508 }
509 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
510 goto fail3;
511
512 return (0);
513
514 fail3:
515 EFSYS_PROBE(fail3);
516 fail2:
517 EFSYS_PROBE(fail2);
518 fail1:
519 EFSYS_PROBE1(fail1, efx_rc_t, rc);
520 return (rc);
521 }
522
523 void
efx_tx_qpio_disable(__in efx_txq_t * etp)524 efx_tx_qpio_disable(
525 __in efx_txq_t *etp)
526 {
527 efx_nic_t *enp = etp->et_enp;
528 const efx_tx_ops_t *etxop = enp->en_etxop;
529
530 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
531
532 if (etxop->etxo_qpio_disable != NULL)
533 etxop->etxo_qpio_disable(etp);
534 }
535
536 __checkReturn efx_rc_t
efx_tx_qpio_write(__in efx_txq_t * etp,__in_ecount (buf_length)uint8_t * buffer,__in size_t buf_length,__in size_t pio_buf_offset)537 efx_tx_qpio_write(
538 __in efx_txq_t *etp,
539 __in_ecount(buf_length) uint8_t *buffer,
540 __in size_t buf_length,
541 __in size_t pio_buf_offset)
542 {
543 efx_nic_t *enp = etp->et_enp;
544 const efx_tx_ops_t *etxop = enp->en_etxop;
545 efx_rc_t rc;
546
547 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
548
549 if (etxop->etxo_qpio_write != NULL) {
550 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
551 pio_buf_offset)) != 0)
552 goto fail1;
553 return (0);
554 }
555
556 return (ENOTSUP);
557
558 fail1:
559 EFSYS_PROBE1(fail1, efx_rc_t, rc);
560 return (rc);
561 }
562
563 __checkReturn efx_rc_t
efx_tx_qpio_post(__in efx_txq_t * etp,__in size_t pkt_length,__in unsigned int completed,__inout unsigned int * addedp)564 efx_tx_qpio_post(
565 __in efx_txq_t *etp,
566 __in size_t pkt_length,
567 __in unsigned int completed,
568 __inout unsigned int *addedp)
569 {
570 efx_nic_t *enp = etp->et_enp;
571 const efx_tx_ops_t *etxop = enp->en_etxop;
572 efx_rc_t rc;
573
574 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
575
576 if (etxop->etxo_qpio_post != NULL) {
577 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
578 addedp)) != 0)
579 goto fail1;
580 return (0);
581 }
582
583 return (ENOTSUP);
584
585 fail1:
586 EFSYS_PROBE1(fail1, efx_rc_t, rc);
587 return (rc);
588 }
589
590 __checkReturn efx_rc_t
efx_tx_qdesc_post(__in efx_txq_t * etp,__in_ecount (ndescs)efx_desc_t * ed,__in unsigned int ndescs,__in unsigned int completed,__inout unsigned int * addedp)591 efx_tx_qdesc_post(
592 __in efx_txq_t *etp,
593 __in_ecount(ndescs) efx_desc_t *ed,
594 __in unsigned int ndescs,
595 __in unsigned int completed,
596 __inout unsigned int *addedp)
597 {
598 efx_nic_t *enp = etp->et_enp;
599 const efx_tx_ops_t *etxop = enp->en_etxop;
600
601 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
602
603 return (etxop->etxo_qdesc_post(etp, ed, ndescs, completed, addedp));
604 }
605
606 void
efx_tx_qdesc_dma_create(__in efx_txq_t * etp,__in efsys_dma_addr_t addr,__in size_t size,__in boolean_t eop,__out efx_desc_t * edp)607 efx_tx_qdesc_dma_create(
608 __in efx_txq_t *etp,
609 __in efsys_dma_addr_t addr,
610 __in size_t size,
611 __in boolean_t eop,
612 __out efx_desc_t *edp)
613 {
614 efx_nic_t *enp = etp->et_enp;
615 const efx_tx_ops_t *etxop = enp->en_etxop;
616
617 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
618 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
619
620 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
621 }
622
623 void
efx_tx_qdesc_tso_create(__in efx_txq_t * etp,__in uint16_t ipv4_id,__in uint32_t tcp_seq,__in uint8_t tcp_flags,__out efx_desc_t * edp)624 efx_tx_qdesc_tso_create(
625 __in efx_txq_t *etp,
626 __in uint16_t ipv4_id,
627 __in uint32_t tcp_seq,
628 __in uint8_t tcp_flags,
629 __out efx_desc_t *edp)
630 {
631 efx_nic_t *enp = etp->et_enp;
632 const efx_tx_ops_t *etxop = enp->en_etxop;
633
634 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
635 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
636
637 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
638 }
639
640 void
efx_tx_qdesc_tso2_create(__in efx_txq_t * etp,__in uint16_t ipv4_id,__in uint16_t outer_ipv4_id,__in uint32_t tcp_seq,__in uint16_t mss,__out_ecount (count)efx_desc_t * edp,__in int count)641 efx_tx_qdesc_tso2_create(
642 __in efx_txq_t *etp,
643 __in uint16_t ipv4_id,
644 __in uint16_t outer_ipv4_id,
645 __in uint32_t tcp_seq,
646 __in uint16_t mss,
647 __out_ecount(count) efx_desc_t *edp,
648 __in int count)
649 {
650 efx_nic_t *enp = etp->et_enp;
651 const efx_tx_ops_t *etxop = enp->en_etxop;
652
653 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
654 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
655
656 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, outer_ipv4_id,
657 tcp_seq, mss, edp, count);
658 }
659
660 void
efx_tx_qdesc_vlantci_create(__in efx_txq_t * etp,__in uint16_t tci,__out efx_desc_t * edp)661 efx_tx_qdesc_vlantci_create(
662 __in efx_txq_t *etp,
663 __in uint16_t tci,
664 __out efx_desc_t *edp)
665 {
666 efx_nic_t *enp = etp->et_enp;
667 const efx_tx_ops_t *etxop = enp->en_etxop;
668
669 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
670 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
671
672 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
673 }
674
675 void
efx_tx_qdesc_checksum_create(__in efx_txq_t * etp,__in uint16_t flags,__out efx_desc_t * edp)676 efx_tx_qdesc_checksum_create(
677 __in efx_txq_t *etp,
678 __in uint16_t flags,
679 __out efx_desc_t *edp)
680 {
681 efx_nic_t *enp = etp->et_enp;
682 const efx_tx_ops_t *etxop = enp->en_etxop;
683
684 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
685 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL);
686
687 etxop->etxo_qdesc_checksum_create(etp, flags, edp);
688 }
689
690 #if EFSYS_OPT_QSTATS
691 void
efx_tx_qstats_update(__in efx_txq_t * etp,__inout_ecount (TX_NQSTATS)efsys_stat_t * stat)692 efx_tx_qstats_update(
693 __in efx_txq_t *etp,
694 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
695 {
696 efx_nic_t *enp = etp->et_enp;
697 const efx_tx_ops_t *etxop = enp->en_etxop;
698
699 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
700
701 etxop->etxo_qstats_update(etp, stat);
702 }
703 #endif
704
705 #if EFSYS_OPT_SIENA
706
707 static __checkReturn efx_rc_t
siena_tx_init(__in efx_nic_t * enp)708 siena_tx_init(
709 __in efx_nic_t *enp)
710 {
711 efx_oword_t oword;
712
713 /*
714 * Disable the timer-based TX DMA backoff and allow TX DMA to be
715 * controlled by the RX FIFO fill level (although always allow a
716 * minimal trickle).
717 */
718 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
719 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
720 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
721 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
722 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
723 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
724 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
725 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
726
727 /*
728 * Filter all packets less than 14 bytes to avoid parsing
729 * errors.
730 */
731 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
732 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
733
734 /*
735 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
736 * descriptors (which is bad).
737 */
738 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
739 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
740 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
741
742 return (0);
743 }
744
745 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
746 do { \
747 unsigned int id; \
748 size_t offset; \
749 efx_qword_t qword; \
750 \
751 id = (_added)++ & (_etp)->et_mask; \
752 offset = id * sizeof (efx_qword_t); \
753 \
754 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
755 unsigned int, id, efsys_dma_addr_t, (_addr), \
756 size_t, (_size), boolean_t, (_eop)); \
757 \
758 EFX_POPULATE_QWORD_4(qword, \
759 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
760 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
761 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
762 (uint32_t)((_addr) & 0xffffffff), \
763 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
764 (uint32_t)((_addr) >> 32)); \
765 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
766 \
767 _NOTE(CONSTANTCONDITION) \
768 } while (B_FALSE)
769
770 static __checkReturn efx_rc_t
siena_tx_qpost(__in efx_txq_t * etp,__in_ecount (ndescs)efx_buffer_t * eb,__in unsigned int ndescs,__in unsigned int completed,__inout unsigned int * addedp)771 siena_tx_qpost(
772 __in efx_txq_t *etp,
773 __in_ecount(ndescs) efx_buffer_t *eb,
774 __in unsigned int ndescs,
775 __in unsigned int completed,
776 __inout unsigned int *addedp)
777 {
778 unsigned int added = *addedp;
779 unsigned int i;
780
781 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
782 return (ENOSPC);
783
784 for (i = 0; i < ndescs; i++) {
785 efx_buffer_t *ebp = &eb[i];
786 efsys_dma_addr_t start = ebp->eb_addr;
787 size_t size = ebp->eb_size;
788 efsys_dma_addr_t end = start + size;
789
790 /*
791 * Fragments must not span 4k boundaries.
792 * Here it is a stricter requirement than the maximum length.
793 */
794 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, start + 1,
795 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
796
797 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
798 }
799
800 EFX_TX_QSTAT_INCR(etp, TX_POST);
801
802 *addedp = added;
803 return (0);
804 }
805
806 static void
siena_tx_qpush(__in efx_txq_t * etp,__in unsigned int added,__in unsigned int pushed)807 siena_tx_qpush(
808 __in efx_txq_t *etp,
809 __in unsigned int added,
810 __in unsigned int pushed)
811 {
812 efx_nic_t *enp = etp->et_enp;
813 uint32_t wptr;
814 efx_dword_t dword;
815 efx_oword_t oword;
816
817 /* Push the populated descriptors out */
818 wptr = added & etp->et_mask;
819
820 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
821
822 /* Only write the third DWORD */
823 EFX_POPULATE_DWORD_1(dword,
824 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
825
826 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
827 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
828 wptr, pushed & etp->et_mask);
829 EFSYS_PIO_WRITE_BARRIER();
830 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
831 etp->et_index, &dword, B_FALSE);
832 }
833
834 #define EFX_MAX_PACE_VALUE 20
835 #define EFX_TX_PACE_CLOCK_BASE 104
836
837 static __checkReturn efx_rc_t
siena_tx_qpace(__in efx_txq_t * etp,__in unsigned int ns)838 siena_tx_qpace(
839 __in efx_txq_t *etp,
840 __in unsigned int ns)
841 {
842 efx_nic_t *enp = etp->et_enp;
843 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
844 efx_oword_t oword;
845 unsigned int pace_val;
846 unsigned int timer_period;
847 efx_rc_t rc;
848
849 if (ns == 0) {
850 pace_val = 0;
851 } else {
852 /*
853 * The pace_val to write into the table is s.t
854 * ns <= timer_period * (2 ^ pace_val)
855 */
856 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult;
857 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
858 if ((timer_period << pace_val) >= ns)
859 break;
860 }
861 }
862 if (pace_val > EFX_MAX_PACE_VALUE) {
863 rc = EINVAL;
864 goto fail1;
865 }
866
867 /* Update the pacing table */
868 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
869 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
870 &oword, B_TRUE);
871
872 return (0);
873
874 fail1:
875 EFSYS_PROBE1(fail1, efx_rc_t, rc);
876
877 return (rc);
878 }
879
880 static __checkReturn efx_rc_t
siena_tx_qflush(__in efx_txq_t * etp)881 siena_tx_qflush(
882 __in efx_txq_t *etp)
883 {
884 efx_nic_t *enp = etp->et_enp;
885 efx_oword_t oword;
886 uint32_t label;
887
888 efx_tx_qpace(etp, 0);
889
890 label = etp->et_index;
891
892 /* Flush the queue */
893 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
894 FRF_AZ_TX_FLUSH_DESCQ, label);
895 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
896
897 return (0);
898 }
899
900 static void
siena_tx_qenable(__in efx_txq_t * etp)901 siena_tx_qenable(
902 __in efx_txq_t *etp)
903 {
904 efx_nic_t *enp = etp->et_enp;
905 efx_oword_t oword;
906
907 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
908 etp->et_index, &oword, B_TRUE);
909
910 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
911 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
912 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
913 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
914 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
915
916 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
917 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
918 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
919
920 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
921 etp->et_index, &oword, B_TRUE);
922 }
923
924 static __checkReturn efx_rc_t
siena_tx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in uint16_t flags,__in efx_evq_t * eep,__in efx_txq_t * etp,__out unsigned int * addedp)925 siena_tx_qcreate(
926 __in efx_nic_t *enp,
927 __in unsigned int index,
928 __in unsigned int label,
929 __in efsys_mem_t *esmp,
930 __in size_t ndescs,
931 __in uint32_t id,
932 __in uint16_t flags,
933 __in efx_evq_t *eep,
934 __in efx_txq_t *etp,
935 __out unsigned int *addedp)
936 {
937 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
938 efx_oword_t oword;
939 uint32_t size;
940 uint16_t inner_csum;
941 efx_rc_t rc;
942
943 _NOTE(ARGUNUSED(esmp))
944
945 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
946 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
947 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
948
949 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
950 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
951
952 if (!ISP2(ndescs) ||
953 (ndescs < EFX_TXQ_MINNDESCS) || (ndescs > EFX_EVQ_MAXNEVS)) {
954 rc = EINVAL;
955 goto fail1;
956 }
957 if (index >= encp->enc_txq_limit) {
958 rc = EINVAL;
959 goto fail2;
960 }
961 for (size = 0;
962 (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS);
963 size++)
964 if ((1 << size) == (int)(ndescs / EFX_TXQ_MINNDESCS))
965 break;
966 if (id + (1 << size) >= encp->enc_buftbl_limit) {
967 rc = EINVAL;
968 goto fail3;
969 }
970
971 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP;
972 if ((flags & inner_csum) != 0) {
973 rc = EINVAL;
974 goto fail4;
975 }
976
977 /* Set up the new descriptor queue */
978 *addedp = 0;
979
980 EFX_POPULATE_OWORD_6(oword,
981 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
982 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
983 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
984 FRF_AZ_TX_DESCQ_LABEL, label,
985 FRF_AZ_TX_DESCQ_SIZE, size,
986 FRF_AZ_TX_DESCQ_TYPE, 0);
987
988 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
989 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
990 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
991 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
992 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
993
994 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
995 etp->et_index, &oword, B_TRUE);
996
997 return (0);
998
999 fail4:
1000 EFSYS_PROBE(fail4);
1001 fail3:
1002 EFSYS_PROBE(fail3);
1003 fail2:
1004 EFSYS_PROBE(fail2);
1005 fail1:
1006 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1007
1008 return (rc);
1009 }
1010
1011 __checkReturn efx_rc_t
siena_tx_qdesc_post(__in efx_txq_t * etp,__in_ecount (ndescs)efx_desc_t * ed,__in unsigned int ndescs,__in unsigned int completed,__inout unsigned int * addedp)1012 siena_tx_qdesc_post(
1013 __in efx_txq_t *etp,
1014 __in_ecount(ndescs) efx_desc_t *ed,
1015 __in unsigned int ndescs,
1016 __in unsigned int completed,
1017 __inout unsigned int *addedp)
1018 {
1019 unsigned int added = *addedp;
1020 unsigned int i;
1021 efx_rc_t rc;
1022
1023 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
1024 rc = ENOSPC;
1025 goto fail1;
1026 }
1027
1028 for (i = 0; i < ndescs; i++) {
1029 efx_desc_t *edp = &ed[i];
1030 unsigned int id;
1031 size_t offset;
1032
1033 id = added++ & etp->et_mask;
1034 offset = id * sizeof (efx_desc_t);
1035
1036 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
1037 }
1038
1039 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
1040 unsigned int, added, unsigned int, ndescs);
1041
1042 EFX_TX_QSTAT_INCR(etp, TX_POST);
1043
1044 *addedp = added;
1045 return (0);
1046
1047 fail1:
1048 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1049 return (rc);
1050 }
1051
1052 void
siena_tx_qdesc_dma_create(__in efx_txq_t * etp,__in efsys_dma_addr_t addr,__in size_t size,__in boolean_t eop,__out efx_desc_t * edp)1053 siena_tx_qdesc_dma_create(
1054 __in efx_txq_t *etp,
1055 __in efsys_dma_addr_t addr,
1056 __in size_t size,
1057 __in boolean_t eop,
1058 __out efx_desc_t *edp)
1059 {
1060 /*
1061 * Fragments must not span 4k boundaries.
1062 * Here it is a stricter requirement than the maximum length.
1063 */
1064 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, addr + 1,
1065 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1066
1067 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1068 efsys_dma_addr_t, addr,
1069 size_t, size, boolean_t, eop);
1070
1071 EFX_POPULATE_QWORD_4(edp->ed_eq,
1072 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1073 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1074 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1075 (uint32_t)(addr & 0xffffffff),
1076 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1077 (uint32_t)(addr >> 32));
1078 }
1079
1080 #endif /* EFSYS_OPT_SIENA */
1081
1082 #if EFSYS_OPT_QSTATS
1083 #if EFSYS_OPT_NAMES
1084 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1085 static const char * const __efx_tx_qstat_name[] = {
1086 "post",
1087 "post_pio",
1088 };
1089 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1090
1091 const char *
efx_tx_qstat_name(__in efx_nic_t * enp,__in unsigned int id)1092 efx_tx_qstat_name(
1093 __in efx_nic_t *enp,
1094 __in unsigned int id)
1095 {
1096 _NOTE(ARGUNUSED(enp))
1097 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1098 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1099
1100 return (__efx_tx_qstat_name[id]);
1101 }
1102 #endif /* EFSYS_OPT_NAMES */
1103 #endif /* EFSYS_OPT_QSTATS */
1104
1105 #if EFSYS_OPT_SIENA
1106
1107 #if EFSYS_OPT_QSTATS
1108 static void
siena_tx_qstats_update(__in efx_txq_t * etp,__inout_ecount (TX_NQSTATS)efsys_stat_t * stat)1109 siena_tx_qstats_update(
1110 __in efx_txq_t *etp,
1111 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1112 {
1113 unsigned int id;
1114
1115 for (id = 0; id < TX_NQSTATS; id++) {
1116 efsys_stat_t *essp = &stat[id];
1117
1118 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1119 etp->et_stat[id] = 0;
1120 }
1121 }
1122 #endif /* EFSYS_OPT_QSTATS */
1123
1124 static void
siena_tx_qdestroy(__in efx_txq_t * etp)1125 siena_tx_qdestroy(
1126 __in efx_txq_t *etp)
1127 {
1128 efx_nic_t *enp = etp->et_enp;
1129 efx_oword_t oword;
1130
1131 /* Purge descriptor queue */
1132 EFX_ZERO_OWORD(oword);
1133
1134 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1135 etp->et_index, &oword, B_TRUE);
1136 }
1137
1138 static void
siena_tx_fini(__in efx_nic_t * enp)1139 siena_tx_fini(
1140 __in efx_nic_t *enp)
1141 {
1142 _NOTE(ARGUNUSED(enp))
1143 }
1144
1145 #endif /* EFSYS_OPT_SIENA */
1146