1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2010-2016 Solarflare Communications Inc.
5 * All rights reserved.
6 *
7 * This software was developed in part by Philip Paeps under contract for
8 * Solarflare Communications, Inc.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright notice,
14 * this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright notice,
16 * this list of conditions and the following disclaimer in the documentation
17 * and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 *
31 * The views and conclusions contained in the software and documentation are
32 * those of the authors and should not be interpreted as representing official
33 * policies, either expressed or implied, of the FreeBSD Project.
34 */
35
36 #include <sys/param.h>
37 #include <sys/kernel.h>
38 #include <sys/malloc.h>
39 #include <sys/param.h>
40 #include <sys/queue.h>
41 #include <sys/systm.h>
42 #include <sys/taskqueue.h>
43
44 #include "common/efx.h"
45
46 #include "sfxge.h"
47
48 static void
sfxge_ev_qcomplete(struct sfxge_evq * evq,boolean_t eop)49 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop)
50 {
51 struct sfxge_softc *sc;
52 unsigned int index;
53 struct sfxge_rxq *rxq;
54 struct sfxge_txq *txq;
55
56 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
57
58 sc = evq->sc;
59 index = evq->index;
60 rxq = sc->rxq[index];
61
62 if ((txq = evq->txq) != NULL) {
63 evq->txq = NULL;
64 evq->txqs = &(evq->txq);
65
66 do {
67 struct sfxge_txq *next;
68
69 next = txq->next;
70 txq->next = NULL;
71
72 KASSERT(txq->evq_index == index,
73 ("txq->evq_index != index"));
74
75 if (txq->pending != txq->completed)
76 sfxge_tx_qcomplete(txq, evq);
77
78 txq = next;
79 } while (txq != NULL);
80 }
81
82 if (rxq->pending != rxq->completed)
83 sfxge_rx_qcomplete(rxq, eop);
84 }
85
86 static struct sfxge_rxq *
sfxge_get_rxq_by_label(struct sfxge_evq * evq,uint32_t label)87 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label)
88 {
89 struct sfxge_rxq *rxq;
90
91 KASSERT(label == 0, ("unexpected rxq label != 0"));
92
93 rxq = evq->sc->rxq[evq->index];
94
95 KASSERT(rxq != NULL, ("rxq == NULL"));
96 KASSERT(evq->index == rxq->index, ("evq->index != rxq->index"));
97
98 return (rxq);
99 }
100
101 static boolean_t
sfxge_ev_rx(void * arg,uint32_t label,uint32_t id,uint32_t size,uint16_t flags)102 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
103 uint16_t flags)
104 {
105 struct sfxge_evq *evq;
106 struct sfxge_softc *sc;
107 struct sfxge_rxq *rxq;
108 unsigned int stop;
109 unsigned int delta;
110 struct sfxge_rx_sw_desc *rx_desc;
111
112 evq = arg;
113 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
114
115 sc = evq->sc;
116
117 if (evq->exception)
118 goto done;
119
120 rxq = sfxge_get_rxq_by_label(evq, label);
121 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED))
122 goto done;
123
124 stop = (id + 1) & rxq->ptr_mask;
125 id = rxq->pending & rxq->ptr_mask;
126 delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop);
127 rxq->pending += delta;
128
129 if (delta != 1) {
130 if ((delta <= 0) ||
131 (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) {
132 evq->exception = B_TRUE;
133
134 device_printf(sc->dev, "RX completion out of order"
135 " (id=%#x delta=%u flags=%#x); resetting\n",
136 id, delta, flags);
137 sfxge_schedule_reset(sc);
138
139 goto done;
140 }
141 }
142
143 rx_desc = &rxq->queue[id];
144
145 prefetch_read_many(rx_desc->mbuf);
146
147 for (; id != stop; id = (id + 1) & rxq->ptr_mask) {
148 rx_desc = &rxq->queue[id];
149 KASSERT(rx_desc->flags == EFX_DISCARD,
150 ("rx_desc->flags != EFX_DISCARD"));
151 rx_desc->flags = flags;
152
153 KASSERT(size < (1 << 16), ("size > (1 << 16)"));
154 rx_desc->size = (uint16_t)size;
155 }
156
157 evq->rx_done++;
158
159 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH)
160 sfxge_ev_qcomplete(evq, B_FALSE);
161
162 done:
163 return (evq->rx_done >= SFXGE_EV_BATCH);
164 }
165
166 static boolean_t
sfxge_ev_exception(void * arg,uint32_t code,uint32_t data)167 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
168 {
169 struct sfxge_evq *evq;
170 struct sfxge_softc *sc;
171
172 evq = (struct sfxge_evq *)arg;
173 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
174
175 sc = evq->sc;
176
177 DBGPRINT(sc->dev, "[%d] %s", evq->index,
178 (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" :
179 (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" :
180 (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" :
181 (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" :
182 (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" :
183 (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" :
184 (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" :
185 (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" :
186 (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" :
187 "UNKNOWN");
188
189 evq->exception = B_TRUE;
190
191 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
192 device_printf(sc->dev,
193 "hardware exception (code=%u); resetting\n",
194 code);
195 sfxge_schedule_reset(sc);
196 }
197
198 return (B_FALSE);
199 }
200
201 static boolean_t
sfxge_ev_rxq_flush_done(void * arg,uint32_t rxq_index)202 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
203 {
204 struct sfxge_evq *evq;
205 struct sfxge_softc *sc;
206 struct sfxge_rxq *rxq;
207 unsigned int index;
208 uint16_t magic;
209
210 evq = (struct sfxge_evq *)arg;
211 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
212
213 sc = evq->sc;
214 rxq = sc->rxq[rxq_index];
215
216 KASSERT(rxq != NULL, ("rxq == NULL"));
217
218 /* Resend a software event on the correct queue */
219 index = rxq->index;
220 if (index == evq->index) {
221 sfxge_rx_qflush_done(rxq);
222 return (B_FALSE);
223 }
224
225 evq = sc->evq[index];
226 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq);
227
228 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
229 ("evq not started"));
230 efx_ev_qpost(evq->common, magic);
231
232 return (B_FALSE);
233 }
234
235 static boolean_t
sfxge_ev_rxq_flush_failed(void * arg,uint32_t rxq_index)236 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
237 {
238 struct sfxge_evq *evq;
239 struct sfxge_softc *sc;
240 struct sfxge_rxq *rxq;
241 unsigned int index;
242 uint16_t magic;
243
244 evq = (struct sfxge_evq *)arg;
245 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
246
247 sc = evq->sc;
248 rxq = sc->rxq[rxq_index];
249
250 KASSERT(rxq != NULL, ("rxq == NULL"));
251
252 /* Resend a software event on the correct queue */
253 index = rxq->index;
254 evq = sc->evq[index];
255 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq);
256
257 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
258 ("evq not started"));
259 efx_ev_qpost(evq->common, magic);
260
261 return (B_FALSE);
262 }
263
264 static struct sfxge_txq *
sfxge_get_txq_by_label(struct sfxge_evq * evq,enum sfxge_txq_type label)265 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label)
266 {
267 unsigned int index;
268
269 KASSERT((evq->sc->txq_dynamic_cksum_toggle_supported) ? (label == 0) :
270 ((evq->index == 0 && label < SFXGE_TXQ_NTYPES) ||
271 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM)),
272 ("unexpected txq label"));
273
274 index = (evq->index == 0) ?
275 label : (evq->index - 1 + SFXGE_EVQ0_N_TXQ(evq->sc));
276 return (evq->sc->txq[index]);
277 }
278
279 static boolean_t
sfxge_ev_tx(void * arg,uint32_t label,uint32_t id)280 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
281 {
282 struct sfxge_evq *evq;
283 struct sfxge_txq *txq;
284 unsigned int stop;
285 unsigned int delta;
286
287 evq = (struct sfxge_evq *)arg;
288 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
289
290 txq = sfxge_get_txq_by_label(evq, label);
291
292 KASSERT(txq != NULL, ("txq == NULL"));
293 KASSERT(evq->index == txq->evq_index,
294 ("evq->index != txq->evq_index"));
295
296 if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED))
297 goto done;
298
299 stop = (id + 1) & txq->ptr_mask;
300 id = txq->pending & txq->ptr_mask;
301
302 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop);
303 txq->pending += delta;
304
305 evq->tx_done++;
306
307 if (txq->next == NULL &&
308 evq->txqs != &(txq->next)) {
309 *(evq->txqs) = txq;
310 evq->txqs = &(txq->next);
311 }
312
313 if (txq->pending - txq->completed >= SFXGE_TX_BATCH)
314 sfxge_tx_qcomplete(txq, evq);
315
316 done:
317 return (evq->tx_done >= SFXGE_EV_BATCH);
318 }
319
320 static boolean_t
sfxge_ev_txq_flush_done(void * arg,uint32_t txq_index)321 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
322 {
323 struct sfxge_evq *evq;
324 struct sfxge_softc *sc;
325 struct sfxge_txq *txq;
326 uint16_t magic;
327
328 evq = (struct sfxge_evq *)arg;
329 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
330
331 sc = evq->sc;
332 txq = sc->txq[txq_index];
333
334 KASSERT(txq != NULL, ("txq == NULL"));
335 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED,
336 ("txq not initialized"));
337
338 if (txq->evq_index == evq->index) {
339 sfxge_tx_qflush_done(txq);
340 return (B_FALSE);
341 }
342
343 /* Resend a software event on the correct queue */
344 evq = sc->evq[txq->evq_index];
345 magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq);
346
347 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
348 ("evq not started"));
349 efx_ev_qpost(evq->common, magic);
350
351 return (B_FALSE);
352 }
353
354 static boolean_t
sfxge_ev_software(void * arg,uint16_t magic)355 sfxge_ev_software(void *arg, uint16_t magic)
356 {
357 struct sfxge_evq *evq;
358 unsigned int label;
359
360 evq = (struct sfxge_evq *)arg;
361 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
362
363 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
364 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
365
366 switch (magic) {
367 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE):
368 sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label));
369 break;
370
371 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED):
372 sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label));
373 break;
374
375 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL):
376 sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label));
377 break;
378
379 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): {
380 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label);
381
382 KASSERT(txq != NULL, ("txq == NULL"));
383 KASSERT(evq->index == txq->evq_index,
384 ("evq->index != txq->evq_index"));
385
386 sfxge_tx_qflush_done(txq);
387 break;
388 }
389 default:
390 break;
391 }
392
393 return (B_FALSE);
394 }
395
396 static boolean_t
sfxge_ev_sram(void * arg,uint32_t code)397 sfxge_ev_sram(void *arg, uint32_t code)
398 {
399 (void)arg;
400 (void)code;
401
402 switch (code) {
403 case EFX_SRAM_UPDATE:
404 EFSYS_PROBE(sram_update);
405 break;
406
407 case EFX_SRAM_CLEAR:
408 EFSYS_PROBE(sram_clear);
409 break;
410
411 case EFX_SRAM_ILLEGAL_CLEAR:
412 EFSYS_PROBE(sram_illegal_clear);
413 break;
414
415 default:
416 KASSERT(B_FALSE, ("Impossible SRAM event"));
417 break;
418 }
419
420 return (B_FALSE);
421 }
422
423 static boolean_t
sfxge_ev_timer(void * arg,uint32_t index)424 sfxge_ev_timer(void *arg, uint32_t index)
425 {
426 (void)arg;
427 (void)index;
428
429 return (B_FALSE);
430 }
431
432 static boolean_t
sfxge_ev_wake_up(void * arg,uint32_t index)433 sfxge_ev_wake_up(void *arg, uint32_t index)
434 {
435 (void)arg;
436 (void)index;
437
438 return (B_FALSE);
439 }
440
441 #if EFSYS_OPT_QSTATS
442
443 static void
sfxge_evq_stat_update(struct sfxge_evq * evq)444 sfxge_evq_stat_update(struct sfxge_evq *evq)
445 {
446 clock_t now;
447
448 SFXGE_EVQ_LOCK(evq);
449
450 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTED))
451 goto out;
452
453 now = ticks;
454 if ((unsigned int)(now - evq->stats_update_time) < (unsigned int)hz)
455 goto out;
456
457 evq->stats_update_time = now;
458 efx_ev_qstats_update(evq->common, evq->stats);
459
460 out:
461 SFXGE_EVQ_UNLOCK(evq);
462 }
463
464 static int
sfxge_evq_stat_handler(SYSCTL_HANDLER_ARGS)465 sfxge_evq_stat_handler(SYSCTL_HANDLER_ARGS)
466 {
467 struct sfxge_evq *evq = arg1;
468 struct sfxge_softc *sc = evq->sc;
469 unsigned int id = arg2;
470
471 SFXGE_ADAPTER_LOCK(sc);
472
473 sfxge_evq_stat_update(evq);
474
475 SFXGE_ADAPTER_UNLOCK(sc);
476
477 return (SYSCTL_OUT(req, &evq->stats[id], sizeof(evq->stats[id])));
478 }
479
480 static int
sfxge_evq_stat_init(struct sfxge_evq * evq)481 sfxge_evq_stat_init(struct sfxge_evq *evq)
482 {
483 struct sfxge_softc *sc = evq->sc;
484 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
485 char name[16];
486 struct sysctl_oid *evq_stats_node;
487 unsigned int id;
488
489 snprintf(name, sizeof(name), "%u", evq->index);
490 evq_stats_node = SYSCTL_ADD_NODE(ctx,
491 SYSCTL_CHILDREN(sc->evqs_stats_node), OID_AUTO, name,
492 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "");
493 if (evq_stats_node == NULL)
494 return (ENOMEM);
495
496 for (id = 0; id < EV_NQSTATS; id++) {
497 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(evq_stats_node),
498 OID_AUTO, efx_ev_qstat_name(sc->enp, id),
499 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, evq, id,
500 sfxge_evq_stat_handler, "Q", "");
501 }
502
503 return (0);
504 }
505
506 static void
sfxge_ev_stat_update(struct sfxge_softc * sc)507 sfxge_ev_stat_update(struct sfxge_softc *sc)
508 {
509 struct sfxge_evq *evq;
510 unsigned int index;
511 clock_t now;
512 unsigned int id;
513
514 SFXGE_ADAPTER_LOCK(sc);
515
516 now = ticks;
517 if ((unsigned int)(now - sc->ev_stats_update_time) < (unsigned int)hz)
518 goto out;
519
520 sc->ev_stats_update_time = now;
521
522 memset(sc->ev_stats, 0, sizeof(sc->ev_stats));
523
524 /* Update and add event counts from each event queue in turn */
525 for (index = 0; index < sc->evq_count; index++) {
526 evq = sc->evq[index];
527 sfxge_evq_stat_update(evq);
528 for (id = 0; id < EV_NQSTATS; id++)
529 sc->ev_stats[id] += evq->stats[id];
530 }
531 out:
532 SFXGE_ADAPTER_UNLOCK(sc);
533 }
534
535 static int
sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)536 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS)
537 {
538 struct sfxge_softc *sc = arg1;
539 unsigned int id = arg2;
540
541 sfxge_ev_stat_update(sc);
542
543 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])));
544 }
545
546 static void
sfxge_ev_stat_init(struct sfxge_softc * sc)547 sfxge_ev_stat_init(struct sfxge_softc *sc)
548 {
549 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev);
550 struct sysctl_oid_list *stat_list;
551 unsigned int id;
552 char name[40];
553
554 stat_list = SYSCTL_CHILDREN(sc->stats_node);
555
556 for (id = 0; id < EV_NQSTATS; id++) {
557 snprintf(name, sizeof(name), "ev_%s",
558 efx_ev_qstat_name(sc->enp, id));
559 SYSCTL_ADD_PROC(ctx, stat_list, OID_AUTO, name,
560 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE,
561 sc, id, sfxge_ev_stat_handler, "Q", "");
562 }
563 }
564
565 #endif /* EFSYS_OPT_QSTATS */
566
567 static void
sfxge_ev_qmoderate(struct sfxge_softc * sc,unsigned int idx,unsigned int us)568 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us)
569 {
570 struct sfxge_evq *evq;
571 efx_evq_t *eep;
572
573 evq = sc->evq[idx];
574 eep = evq->common;
575
576 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
577 ("evq->init_state != SFXGE_EVQ_STARTED"));
578
579 (void)efx_ev_qmoderate(eep, us);
580 }
581
582 static int
sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)583 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS)
584 {
585 struct sfxge_softc *sc = arg1;
586 struct sfxge_intr *intr = &sc->intr;
587 unsigned int moderation;
588 int error;
589 unsigned int index;
590
591 SFXGE_ADAPTER_LOCK(sc);
592
593 if (req->newptr != NULL) {
594 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation)))
595 != 0)
596 goto out;
597
598 /* We may not be calling efx_ev_qmoderate() now,
599 * so we have to range-check the value ourselves.
600 */
601 if (moderation >
602 efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) {
603 error = EINVAL;
604 goto out;
605 }
606
607 sc->ev_moderation = moderation;
608 if (intr->state == SFXGE_INTR_STARTED) {
609 for (index = 0; index < sc->evq_count; index++)
610 sfxge_ev_qmoderate(sc, index, moderation);
611 }
612 } else {
613 error = SYSCTL_OUT(req, &sc->ev_moderation,
614 sizeof(sc->ev_moderation));
615 }
616
617 out:
618 SFXGE_ADAPTER_UNLOCK(sc);
619
620 return (error);
621 }
622
623 static boolean_t
sfxge_ev_initialized(void * arg)624 sfxge_ev_initialized(void *arg)
625 {
626 struct sfxge_evq *evq;
627
628 evq = (struct sfxge_evq *)arg;
629 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
630
631 /* Init done events may be duplicated on 7xxx */
632 KASSERT(evq->init_state == SFXGE_EVQ_STARTING ||
633 evq->init_state == SFXGE_EVQ_STARTED,
634 ("evq not starting"));
635
636 evq->init_state = SFXGE_EVQ_STARTED;
637
638 return (0);
639 }
640
641 static boolean_t
sfxge_ev_link_change(void * arg,efx_link_mode_t link_mode)642 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode)
643 {
644 struct sfxge_evq *evq;
645 struct sfxge_softc *sc;
646
647 evq = (struct sfxge_evq *)arg;
648 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq);
649
650 sc = evq->sc;
651
652 sfxge_mac_link_update(sc, link_mode);
653
654 return (0);
655 }
656
657 static const efx_ev_callbacks_t sfxge_ev_callbacks = {
658 .eec_initialized = sfxge_ev_initialized,
659 .eec_rx = sfxge_ev_rx,
660 .eec_tx = sfxge_ev_tx,
661 .eec_exception = sfxge_ev_exception,
662 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done,
663 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed,
664 .eec_txq_flush_done = sfxge_ev_txq_flush_done,
665 .eec_software = sfxge_ev_software,
666 .eec_sram = sfxge_ev_sram,
667 .eec_wake_up = sfxge_ev_wake_up,
668 .eec_timer = sfxge_ev_timer,
669 .eec_link_change = sfxge_ev_link_change,
670 };
671
672 int
sfxge_ev_qpoll(struct sfxge_evq * evq)673 sfxge_ev_qpoll(struct sfxge_evq *evq)
674 {
675 int rc;
676
677 SFXGE_EVQ_LOCK(evq);
678
679 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING &&
680 evq->init_state != SFXGE_EVQ_STARTED)) {
681 rc = EINVAL;
682 goto fail;
683 }
684
685 /* Synchronize the DMA memory for reading */
686 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map,
687 BUS_DMASYNC_POSTREAD);
688
689 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0"));
690 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0"));
691 KASSERT(evq->txq == NULL, ("evq->txq != NULL"));
692 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
693
694 /* Poll the queue */
695 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq);
696
697 evq->rx_done = 0;
698 evq->tx_done = 0;
699
700 /* Perform any pending completion processing */
701 sfxge_ev_qcomplete(evq, B_TRUE);
702
703 /* Re-prime the event queue for interrupts */
704 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
705 goto fail;
706
707 SFXGE_EVQ_UNLOCK(evq);
708
709 return (0);
710
711 fail:
712 SFXGE_EVQ_UNLOCK(evq);
713 return (rc);
714 }
715
716 static void
sfxge_ev_qstop(struct sfxge_softc * sc,unsigned int index)717 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index)
718 {
719 struct sfxge_evq *evq;
720
721 evq = sc->evq[index];
722
723 KASSERT(evq->init_state == SFXGE_EVQ_STARTED,
724 ("evq->init_state != SFXGE_EVQ_STARTED"));
725
726 SFXGE_EVQ_LOCK(evq);
727 evq->init_state = SFXGE_EVQ_INITIALIZED;
728 evq->read_ptr = 0;
729 evq->exception = B_FALSE;
730
731 #if EFSYS_OPT_QSTATS
732 /* Add event counts before discarding the common evq state */
733 efx_ev_qstats_update(evq->common, evq->stats);
734 #endif
735
736 efx_ev_qdestroy(evq->common);
737 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
738 EFX_EVQ_NBUFS(evq->entries));
739 SFXGE_EVQ_UNLOCK(evq);
740 }
741
742 static int
sfxge_ev_qstart(struct sfxge_softc * sc,unsigned int index)743 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index)
744 {
745 struct sfxge_evq *evq;
746 efsys_mem_t *esmp;
747 int count;
748 int rc;
749
750 evq = sc->evq[index];
751 esmp = &evq->mem;
752
753 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
754 ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
755
756 /* Clear all events. */
757 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries));
758
759 /* Program the buffer table. */
760 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp,
761 EFX_EVQ_NBUFS(evq->entries))) != 0)
762 return (rc);
763
764 /* Create the common code event queue. */
765 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries,
766 evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO,
767 &evq->common)) != 0)
768 goto fail;
769
770 SFXGE_EVQ_LOCK(evq);
771
772 /* Prime the event queue for interrupts */
773 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0)
774 goto fail2;
775
776 evq->init_state = SFXGE_EVQ_STARTING;
777
778 SFXGE_EVQ_UNLOCK(evq);
779
780 /* Wait for the initialization event */
781 count = 0;
782 do {
783 /* Pause for 100 ms */
784 pause("sfxge evq init", hz / 10);
785
786 /* Check to see if the test event has been processed */
787 if (evq->init_state == SFXGE_EVQ_STARTED)
788 goto done;
789
790 } while (++count < 20);
791
792 rc = ETIMEDOUT;
793 goto fail3;
794
795 done:
796 return (0);
797
798 fail3:
799 SFXGE_EVQ_LOCK(evq);
800 evq->init_state = SFXGE_EVQ_INITIALIZED;
801 fail2:
802 SFXGE_EVQ_UNLOCK(evq);
803 efx_ev_qdestroy(evq->common);
804 fail:
805 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id,
806 EFX_EVQ_NBUFS(evq->entries));
807
808 return (rc);
809 }
810
811 void
sfxge_ev_stop(struct sfxge_softc * sc)812 sfxge_ev_stop(struct sfxge_softc *sc)
813 {
814 struct sfxge_intr *intr __diagused;
815 efx_nic_t *enp;
816 int index;
817
818 intr = &sc->intr;
819 enp = sc->enp;
820
821 KASSERT(intr->state == SFXGE_INTR_STARTED,
822 ("Interrupts not started"));
823
824 /* Stop the event queue(s) */
825 index = sc->evq_count;
826 while (--index >= 0)
827 sfxge_ev_qstop(sc, index);
828
829 /* Tear down the event module */
830 efx_ev_fini(enp);
831 }
832
833 int
sfxge_ev_start(struct sfxge_softc * sc)834 sfxge_ev_start(struct sfxge_softc *sc)
835 {
836 struct sfxge_intr *intr __diagused;
837 int index;
838 int rc;
839
840 intr = &sc->intr;
841
842 KASSERT(intr->state == SFXGE_INTR_STARTED,
843 ("intr->state != SFXGE_INTR_STARTED"));
844
845 /* Initialize the event module */
846 if ((rc = efx_ev_init(sc->enp)) != 0)
847 return (rc);
848
849 /* Start the event queues */
850 for (index = 0; index < sc->evq_count; index++) {
851 if ((rc = sfxge_ev_qstart(sc, index)) != 0)
852 goto fail;
853 }
854
855 return (0);
856
857 fail:
858 /* Stop the event queue(s) */
859 while (--index >= 0)
860 sfxge_ev_qstop(sc, index);
861
862 /* Tear down the event module */
863 efx_ev_fini(sc->enp);
864
865 return (rc);
866 }
867
868 static void
sfxge_ev_qfini(struct sfxge_softc * sc,unsigned int index)869 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index)
870 {
871 struct sfxge_evq *evq;
872
873 evq = sc->evq[index];
874
875 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED,
876 ("evq->init_state != SFXGE_EVQ_INITIALIZED"));
877 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq"));
878
879 sfxge_dma_free(&evq->mem);
880
881 sc->evq[index] = NULL;
882
883 SFXGE_EVQ_LOCK_DESTROY(evq);
884
885 free(evq, M_SFXGE);
886 }
887
888 static int
sfxge_ev_qinit(struct sfxge_softc * sc,unsigned int index)889 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index)
890 {
891 struct sfxge_evq *evq;
892 efsys_mem_t *esmp;
893 int rc;
894
895 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX"));
896
897 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK);
898 evq->sc = sc;
899 evq->index = index;
900 sc->evq[index] = evq;
901 esmp = &evq->mem;
902
903 /* Build an event queue with room for one event per tx and rx buffer,
904 * plus some extra for link state events and MCDI completions.
905 * There are three tx queues in the first event queue and one in
906 * other.
907 */
908 if (index == 0)
909 evq->entries =
910 ROUNDUP_POW_OF_TWO(sc->rxq_entries +
911 3 * sc->txq_entries +
912 128);
913 else
914 evq->entries =
915 ROUNDUP_POW_OF_TWO(sc->rxq_entries +
916 sc->txq_entries +
917 128);
918
919 /* Initialise TX completion list */
920 evq->txqs = &evq->txq;
921
922 /* Allocate DMA space. */
923 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0)
924 return (rc);
925
926 /* Allocate buffer table entries. */
927 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries),
928 &evq->buf_base_id);
929
930 SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index);
931
932 evq->init_state = SFXGE_EVQ_INITIALIZED;
933
934 #if EFSYS_OPT_QSTATS
935 rc = sfxge_evq_stat_init(evq);
936 if (rc != 0)
937 goto fail_evq_stat_init;
938 #endif
939
940 return (0);
941
942 #if EFSYS_OPT_QSTATS
943 fail_evq_stat_init:
944 evq->init_state = SFXGE_EVQ_UNINITIALIZED;
945 SFXGE_EVQ_LOCK_DESTROY(evq);
946 sfxge_dma_free(esmp);
947 sc->evq[index] = NULL;
948 free(evq, M_SFXGE);
949
950 return (rc);
951 #endif
952 }
953
954 void
sfxge_ev_fini(struct sfxge_softc * sc)955 sfxge_ev_fini(struct sfxge_softc *sc)
956 {
957 struct sfxge_intr *intr __diagused;
958 int index;
959
960 intr = &sc->intr;
961
962 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
963 ("intr->state != SFXGE_INTR_INITIALIZED"));
964
965 sc->ev_moderation = 0;
966
967 /* Tear down the event queue(s). */
968 index = sc->evq_count;
969 while (--index >= 0)
970 sfxge_ev_qfini(sc, index);
971
972 sc->evq_count = 0;
973 }
974
975 int
sfxge_ev_init(struct sfxge_softc * sc)976 sfxge_ev_init(struct sfxge_softc *sc)
977 {
978 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev);
979 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev);
980 struct sfxge_intr *intr;
981 int index;
982 int rc;
983
984 intr = &sc->intr;
985
986 sc->evq_count = intr->n_alloc;
987
988 KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
989 ("intr->state != SFXGE_INTR_INITIALIZED"));
990
991 /* Set default interrupt moderation; add a sysctl to
992 * read and change it.
993 */
994 sc->ev_moderation = SFXGE_MODERATION;
995 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree),
996 OID_AUTO, "int_mod", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
997 sc, 0, sfxge_int_mod_handler, "IU",
998 "sfxge interrupt moderation (us)");
999
1000 #if EFSYS_OPT_QSTATS
1001 sc->evqs_stats_node = SYSCTL_ADD_NODE(
1002 device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(sc->stats_node),
1003 OID_AUTO, "evq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
1004 "Event queues stats");
1005 if (sc->evqs_stats_node == NULL) {
1006 rc = ENOMEM;
1007 goto fail_evqs_stats_node;
1008 }
1009 #endif
1010
1011 /*
1012 * Initialize the event queue(s) - one per interrupt.
1013 */
1014 for (index = 0; index < sc->evq_count; index++) {
1015 if ((rc = sfxge_ev_qinit(sc, index)) != 0)
1016 goto fail;
1017 }
1018
1019 #if EFSYS_OPT_QSTATS
1020 sfxge_ev_stat_init(sc);
1021 #endif
1022
1023 return (0);
1024
1025 fail:
1026 while (--index >= 0)
1027 sfxge_ev_qfini(sc, index);
1028
1029 #if EFSYS_OPT_QSTATS
1030 fail_evqs_stats_node:
1031 #endif
1032 sc->evq_count = 0;
1033 return (rc);
1034 }
1035