1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2022 Scott Long
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include "opt_thunderbolt.h"
30
31 /* PCIe interface for Thunderbolt Native Host Interface (nhi) */
32 #include <sys/types.h>
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
36 #include <sys/module.h>
37 #include <sys/bus.h>
38 #include <sys/conf.h>
39 #include <sys/malloc.h>
40 #include <sys/queue.h>
41 #include <sys/sysctl.h>
42 #include <sys/lock.h>
43 #include <sys/mutex.h>
44 #include <sys/taskqueue.h>
45 #include <sys/gsb_crc32.h>
46 #include <sys/endian.h>
47 #include <vm/vm.h>
48 #include <vm/pmap.h>
49
50 #include <machine/bus.h>
51 #include <machine/stdarg.h>
52
53 #include <dev/thunderbolt/nhi_reg.h>
54 #include <dev/thunderbolt/nhi_var.h>
55 #include <dev/thunderbolt/tb_reg.h>
56 #include <dev/thunderbolt/tb_var.h>
57 #include <dev/thunderbolt/tb_debug.h>
58 #include <dev/thunderbolt/hcm_var.h>
59 #include <dev/thunderbolt/tbcfg_reg.h>
60 #include <dev/thunderbolt/router_var.h>
61 #include <dev/thunderbolt/tb_dev.h>
62 #include "tb_if.h"
63
64 static int nhi_alloc_ring(struct nhi_softc *, int, int, int,
65 struct nhi_ring_pair **);
66 static void nhi_free_ring(struct nhi_ring_pair *);
67 static void nhi_free_rings(struct nhi_softc *);
68 static int nhi_configure_ring(struct nhi_softc *, struct nhi_ring_pair *);
69 static int nhi_activate_ring(struct nhi_ring_pair *);
70 static int nhi_deactivate_ring(struct nhi_ring_pair *);
71 static int nhi_alloc_ring0(struct nhi_softc *);
72 static void nhi_free_ring0(struct nhi_softc *);
73 static void nhi_fill_rx_ring(struct nhi_softc *, struct nhi_ring_pair *);
74 static int nhi_init(struct nhi_softc *);
75 static void nhi_post_init(void *);
76 static int nhi_tx_enqueue(struct nhi_ring_pair *, struct nhi_cmd_frame *);
77 static int nhi_setup_sysctl(struct nhi_softc *);
78
79 SYSCTL_NODE(_hw, OID_AUTO, nhi, CTLFLAG_RD, 0, "NHI Driver Parameters");
80
81 MALLOC_DEFINE(M_NHI, "nhi", "nhi driver memory");
82
83 #ifndef NHI_DEBUG_LEVEL
84 #define NHI_DEBUG_LEVEL 0
85 #endif
86
87 void
nhi_get_tunables(struct nhi_softc * sc)88 nhi_get_tunables(struct nhi_softc *sc)
89 {
90 devclass_t dc;
91 device_t ufp;
92 char tmpstr[80], oid[80];
93 u_int val;
94
95 /* Set local defaults */
96 sc->debug = NHI_DEBUG_LEVEL;
97 sc->max_ring_count = NHI_DEFAULT_NUM_RINGS;
98
99 /* Inherit setting from the upstream thunderbolt switch node */
100 val = TB_GET_DEBUG(sc->dev, &sc->debug);
101 if (val != 0) {
102 dc = devclass_find("tbolt");
103 if (dc != NULL) {
104 ufp = devclass_get_device(dc, device_get_unit(sc->dev));
105 if (ufp != NULL)
106 TB_GET_DEBUG(ufp, &sc->debug);
107 } else {
108 if (TUNABLE_STR_FETCH("hw.tbolt.debug_level", oid,
109 80) != 0)
110 tb_parse_debug(&sc->debug, oid);
111 }
112 }
113
114 /*
115 * Grab global variables. Allow nhi debug flags to override
116 * thunderbolt debug flags, if present.
117 */
118 bzero(oid, 80);
119 if (TUNABLE_STR_FETCH("hw.nhi.debug_level", oid, 80) != 0)
120 tb_parse_debug(&sc->debug, oid);
121 if (TUNABLE_INT_FETCH("hw.nhi.max_rings", &val) != 0) {
122 val = min(val, NHI_MAX_NUM_RINGS);
123 sc->max_ring_count = max(val, 1);
124 }
125
126 /* Grab instance variables */
127 bzero(oid, 80);
128 snprintf(tmpstr, sizeof(tmpstr), "dev.nhi.%d.debug_level",
129 device_get_unit(sc->dev));
130 if (TUNABLE_STR_FETCH(tmpstr, oid, 80) != 0)
131 tb_parse_debug(&sc->debug, oid);
132 snprintf(tmpstr, sizeof(tmpstr), "dev.nhi.%d.max_rings",
133 device_get_unit(sc->dev));
134 if (TUNABLE_INT_FETCH(tmpstr, &val) != 0) {
135 val = min(val, NHI_MAX_NUM_RINGS);
136 sc->max_ring_count = max(val, 1);
137 }
138
139 return;
140 }
141
142 struct nhi_cmd_frame *
nhi_alloc_tx_frame(struct nhi_ring_pair * r)143 nhi_alloc_tx_frame(struct nhi_ring_pair *r)
144 {
145 struct nhi_cmd_frame *cmd;
146
147 mtx_lock(&r->mtx);
148 cmd = nhi_alloc_tx_frame_locked(r);
149 mtx_unlock(&r->mtx);
150
151 return (cmd);
152 }
153
154 void
nhi_free_tx_frame(struct nhi_ring_pair * r,struct nhi_cmd_frame * cmd)155 nhi_free_tx_frame(struct nhi_ring_pair *r, struct nhi_cmd_frame *cmd)
156 {
157 mtx_lock(&r->mtx);
158 nhi_free_tx_frame_locked(r, cmd);
159 mtx_unlock(&r->mtx);
160 }
161
162 /*
163 * Push a command and data dword through the mailbox to the firmware.
164 * Response is either good, error, or timeout. Commands that return data
165 * do so by reading OUTMAILDATA.
166 */
167 int
nhi_inmail_cmd(struct nhi_softc * sc,uint32_t cmd,uint32_t data)168 nhi_inmail_cmd(struct nhi_softc *sc, uint32_t cmd, uint32_t data)
169 {
170 uint32_t val;
171 u_int error, timeout;
172
173 mtx_lock(&sc->nhi_mtx);
174 /*
175 * XXX Should a defer/reschedule happen here, or is it not worth
176 * worrying about?
177 */
178 if (sc->hwflags & NHI_MBOX_BUSY) {
179 mtx_unlock(&sc->nhi_mtx);
180 tb_debug(sc, DBG_MBOX, "Driver busy with mailbox\n");
181 return (EBUSY);
182 }
183 sc->hwflags |= NHI_MBOX_BUSY;
184
185 val = nhi_read_reg(sc, TBT_INMAILCMD);
186 tb_debug(sc, DBG_MBOX|DBG_FULL, "Reading INMAILCMD= 0x%08x\n", val);
187 if (val & INMAILCMD_ERROR)
188 tb_debug(sc, DBG_MBOX, "Error already set in INMAILCMD\n");
189 if (val & INMAILCMD_OPREQ) {
190 mtx_unlock(&sc->nhi_mtx);
191 tb_debug(sc, DBG_MBOX,
192 "INMAILCMD request already in progress\n");
193 return (EBUSY);
194 }
195
196 nhi_write_reg(sc, TBT_INMAILDATA, data);
197 nhi_write_reg(sc, TBT_INMAILCMD, cmd | INMAILCMD_OPREQ);
198
199 /* Poll at 1s intervals */
200 timeout = NHI_MAILBOX_TIMEOUT;
201 while (timeout--) {
202 DELAY(1000000);
203 val = nhi_read_reg(sc, TBT_INMAILCMD);
204 tb_debug(sc, DBG_MBOX|DBG_EXTRA,
205 "Polling INMAILCMD= 0x%08x\n", val);
206 if ((val & INMAILCMD_OPREQ) == 0)
207 break;
208 }
209 sc->hwflags &= ~NHI_MBOX_BUSY;
210 mtx_unlock(&sc->nhi_mtx);
211
212 error = 0;
213 if (val & INMAILCMD_OPREQ) {
214 tb_printf(sc, "Timeout waiting for mailbox\n");
215 error = ETIMEDOUT;
216 }
217 if (val & INMAILCMD_ERROR) {
218 tb_printf(sc, "Firmware reports error in mailbox\n");
219 error = EINVAL;
220 }
221
222 return (error);
223 }
224
225 /*
226 * Pull command status and data from the firmware mailbox.
227 */
228 int
nhi_outmail_cmd(struct nhi_softc * sc,uint32_t * val)229 nhi_outmail_cmd(struct nhi_softc *sc, uint32_t *val)
230 {
231
232 if (val == NULL)
233 return (EINVAL);
234 *val = nhi_read_reg(sc, TBT_OUTMAILCMD);
235 return (0);
236 }
237
238 int
nhi_attach(struct nhi_softc * sc)239 nhi_attach(struct nhi_softc *sc)
240 {
241 uint32_t val;
242 int error = 0;
243
244 if ((error = nhi_setup_sysctl(sc)) != 0)
245 return (error);
246
247 mtx_init(&sc->nhi_mtx, "nhimtx", "NHI Control Mutex", MTX_DEF);
248
249 /*
250 * Get the number of TX/RX paths. This sizes some of the register
251 * arrays during allocation and initialization. USB4 spec says that
252 * the max is 21.
253 */
254 val = GET_HOST_CAPS_PATHS(nhi_read_reg(sc, NHI_HOST_CAPS));
255 tb_debug(sc, DBG_INIT|DBG_NOISY, "Total Paths= %d\n", val);
256 if (val == 0 || val > 21) {
257 tb_printf(sc, "WARN: unexpected number of paths: %d\n", val);
258 /* return (ENXIO); */
259 }
260 sc->path_count = val;
261
262 SLIST_INIT(&sc->ring_list);
263
264 error = nhi_pci_configure_interrupts(sc);
265 if (error == 0)
266 error = nhi_alloc_ring0(sc);
267 if (error == 0) {
268 nhi_configure_ring(sc, sc->ring0);
269 nhi_activate_ring(sc->ring0);
270 nhi_fill_rx_ring(sc, sc->ring0);
271 }
272
273 if (error == 0)
274 error = tbdev_add_interface(sc);
275
276 error = hcm_attach(sc);
277
278 if (error == 0)
279 error = nhi_init(sc);
280
281 return (error);
282 }
283
284 int
nhi_detach(struct nhi_softc * sc)285 nhi_detach(struct nhi_softc *sc)
286 {
287 hcm_detach(sc);
288
289 if (sc->root_rsc != NULL)
290 tb_router_detach(sc->root_rsc);
291
292 tbdev_remove_interface(sc);
293
294 nhi_pci_disable_interrupts(sc);
295 nhi_pci_free_interrupts(sc);
296
297 nhi_free_ring0(sc);
298
299 /* XXX Should the rings be marked as !VALID in the descriptors? */
300 nhi_free_rings(sc);
301
302 mtx_destroy(&sc->nhi_mtx);
303
304 return (0);
305 }
306
307 static void
nhi_memaddr_cb(void * arg,bus_dma_segment_t * segs,int nsegs,int error)308 nhi_memaddr_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
309 {
310 bus_addr_t *addr;
311
312 addr = arg;
313 if (error == 0 && nsegs == 1) {
314 *addr = segs[0].ds_addr;
315 } else
316 *addr = 0;
317 }
318
319 static int
nhi_alloc_ring(struct nhi_softc * sc,int ringnum,int tx_depth,int rx_depth,struct nhi_ring_pair ** rp)320 nhi_alloc_ring(struct nhi_softc *sc, int ringnum, int tx_depth, int rx_depth,
321 struct nhi_ring_pair **rp)
322 {
323 bus_dma_template_t t;
324 bus_addr_t ring_busaddr;
325 struct nhi_ring_pair *r;
326 int ring_size, error;
327 u_int rxring_len, txring_len;
328 char *ring;
329
330 if (ringnum >= sc->max_ring_count) {
331 tb_debug(sc, DBG_INIT, "Tried to allocate ring number %d\n",
332 ringnum);
333 return (EINVAL);
334 }
335
336 /* Allocate the ring structure and the RX ring tacker together. */
337 rxring_len = rx_depth * sizeof(void *);
338 txring_len = tx_depth * sizeof(void *);
339 r = malloc(sizeof(struct nhi_ring_pair) + rxring_len + txring_len,
340 M_NHI, M_NOWAIT|M_ZERO);
341 if (r == NULL) {
342 tb_printf(sc, "ERROR: Cannot allocate ring memory\n");
343 return (ENOMEM);
344 }
345
346 r->sc = sc;
347 TAILQ_INIT(&r->tx_head);
348 TAILQ_INIT(&r->rx_head);
349 r->ring_num = ringnum;
350 r->tx_ring_depth = tx_depth;
351 r->tx_ring_mask = tx_depth - 1;
352 r->rx_ring_depth = rx_depth;
353 r->rx_ring_mask = rx_depth - 1;
354 r->rx_pici_reg = NHI_RX_RING_PICI + ringnum * 16;
355 r->tx_pici_reg = NHI_TX_RING_PICI + ringnum * 16;
356 r->rx_cmd_ring = (struct nhi_cmd_frame **)((uint8_t *)r + sizeof (*r));
357 r->tx_cmd_ring = (struct nhi_cmd_frame **)((uint8_t *)r->rx_cmd_ring +
358 rxring_len);
359
360 snprintf(r->name, NHI_RING_NAMELEN, "nhiring%d\n", ringnum);
361 mtx_init(&r->mtx, r->name, "NHI Ring Lock", MTX_DEF);
362 tb_debug(sc, DBG_INIT | DBG_FULL, "Allocated ring context at %p, "
363 "mutex %p\n", r, &r->mtx);
364
365 /* Allocate the RX and TX buffer descriptor rings */
366 ring_size = sizeof(struct nhi_tx_buffer_desc) * r->tx_ring_depth;
367 ring_size += sizeof(struct nhi_rx_buffer_desc) * r->rx_ring_depth;
368 tb_debug(sc, DBG_INIT | DBG_FULL, "Ring %d ring_size= %d\n",
369 ringnum, ring_size);
370
371 bus_dma_template_init(&t, sc->parent_dmat);
372 t.alignment = 4;
373 t.maxsize = t.maxsegsize = ring_size;
374 t.nsegments = 1;
375 if ((error = bus_dma_template_tag(&t, &r->ring_dmat)) != 0) {
376 tb_printf(sc, "Cannot allocate ring %d DMA tag: %d\n",
377 ringnum, error);
378 return (ENOMEM);
379 }
380 if (bus_dmamem_alloc(r->ring_dmat, (void **)&ring, BUS_DMA_NOWAIT,
381 &r->ring_map)) {
382 tb_printf(sc, "Cannot allocate ring memory\n");
383 return (ENOMEM);
384 }
385 bzero(ring, ring_size);
386 bus_dmamap_load(r->ring_dmat, r->ring_map, ring, ring_size,
387 nhi_memaddr_cb, &ring_busaddr, 0);
388
389 r->ring = ring;
390
391 r->tx_ring = (union nhi_ring_desc *)(ring);
392 r->tx_ring_busaddr = ring_busaddr;
393 ring += sizeof(struct nhi_tx_buffer_desc) * r->tx_ring_depth;
394 ring_busaddr += sizeof(struct nhi_tx_buffer_desc) * r->tx_ring_depth;
395
396 r->rx_ring = (union nhi_ring_desc *)(ring);
397 r->rx_ring_busaddr = ring_busaddr;
398
399 tb_debug(sc, DBG_INIT | DBG_EXTRA, "Ring %d: RX %p [0x%jx] "
400 "TX %p [0x%jx]\n", ringnum, r->tx_ring, r->tx_ring_busaddr,
401 r->rx_ring, r->rx_ring_busaddr);
402
403 *rp = r;
404 return (0);
405 }
406
407 static void
nhi_free_ring(struct nhi_ring_pair * r)408 nhi_free_ring(struct nhi_ring_pair *r)
409 {
410
411 tb_debug(r->sc, DBG_INIT, "Freeing ring %d resources\n", r->ring_num);
412 nhi_deactivate_ring(r);
413
414 if (r->tx_ring_busaddr != 0) {
415 bus_dmamap_unload(r->ring_dmat, r->ring_map);
416 r->tx_ring_busaddr = 0;
417 }
418 if (r->ring != NULL) {
419 bus_dmamem_free(r->ring_dmat, r->ring, r->ring_map);
420 r->ring = NULL;
421 }
422 if (r->ring_dmat != NULL) {
423 bus_dma_tag_destroy(r->ring_dmat);
424 r->ring_dmat = NULL;
425 }
426 mtx_destroy(&r->mtx);
427 }
428
429 static void
nhi_free_rings(struct nhi_softc * sc)430 nhi_free_rings(struct nhi_softc *sc)
431 {
432 struct nhi_ring_pair *r;
433
434 while ((r = SLIST_FIRST(&sc->ring_list)) != NULL) {
435 nhi_free_ring(r);
436 mtx_lock(&sc->nhi_mtx);
437 SLIST_REMOVE_HEAD(&sc->ring_list, ring_link);
438 mtx_unlock(&sc->nhi_mtx);
439 free(r, M_NHI);
440 }
441
442 return;
443 }
444
445 static int
nhi_configure_ring(struct nhi_softc * sc,struct nhi_ring_pair * ring)446 nhi_configure_ring(struct nhi_softc *sc, struct nhi_ring_pair *ring)
447 {
448 bus_addr_t busaddr;
449 uint32_t val;
450 int idx;
451
452 idx = ring->ring_num * 16;
453
454 /* Program the TX ring address and size */
455 busaddr = ring->tx_ring_busaddr;
456 nhi_write_reg(sc, NHI_TX_RING_ADDR_LO + idx, busaddr & 0xffffffff);
457 nhi_write_reg(sc, NHI_TX_RING_ADDR_HI + idx, busaddr >> 32);
458 nhi_write_reg(sc, NHI_TX_RING_SIZE + idx, ring->tx_ring_depth);
459 nhi_write_reg(sc, NHI_TX_RING_TABLE_TIMESTAMP + idx, 0x0);
460 tb_debug(sc, DBG_INIT, "TX Ring %d TX_RING_SIZE= 0x%x\n",
461 ring->ring_num, ring->tx_ring_depth);
462
463 /* Program the RX ring address and size */
464 busaddr = ring->rx_ring_busaddr;
465 val = (ring->rx_buffer_size << 16) | ring->rx_ring_depth;
466 nhi_write_reg(sc, NHI_RX_RING_ADDR_LO + idx, busaddr & 0xffffffff);
467 nhi_write_reg(sc, NHI_RX_RING_ADDR_HI + idx, busaddr >> 32);
468 nhi_write_reg(sc, NHI_RX_RING_SIZE + idx, val);
469 nhi_write_reg(sc, NHI_RX_RING_TABLE_BASE1 + idx, 0xffffffff);
470 tb_debug(sc, DBG_INIT, "RX Ring %d RX_RING_SIZE= 0x%x\n",
471 ring->ring_num, val);
472
473 return (0);
474 }
475
476 static int
nhi_activate_ring(struct nhi_ring_pair * ring)477 nhi_activate_ring(struct nhi_ring_pair *ring)
478 {
479 struct nhi_softc *sc = ring->sc;
480 int idx;
481
482 nhi_pci_enable_interrupt(ring);
483
484 idx = ring->ring_num * 32;
485 tb_debug(sc, DBG_INIT, "Activating ring %d at idx %d\n",
486 ring->ring_num, idx);
487 nhi_write_reg(sc, NHI_TX_RING_TABLE_BASE0 + idx,
488 TX_TABLE_RAW | TX_TABLE_VALID);
489 nhi_write_reg(sc, NHI_RX_RING_TABLE_BASE0 + idx,
490 RX_TABLE_RAW | RX_TABLE_VALID);
491
492 return (0);
493 }
494
495 static int
nhi_deactivate_ring(struct nhi_ring_pair * r)496 nhi_deactivate_ring(struct nhi_ring_pair *r)
497 {
498 struct nhi_softc *sc = r->sc;
499 int idx;
500
501 idx = r->ring_num * 32;
502 tb_debug(sc, DBG_INIT, "Deactiving ring %d at idx %d\n",
503 r->ring_num, idx);
504 nhi_write_reg(sc, NHI_TX_RING_TABLE_BASE0 + idx, 0);
505 nhi_write_reg(sc, NHI_RX_RING_TABLE_BASE0 + idx, 0);
506
507 idx = r->ring_num * 16;
508 tb_debug(sc, DBG_INIT, "Setting ring %d sizes to 0\n", r->ring_num);
509 nhi_write_reg(sc, NHI_TX_RING_SIZE + idx, 0);
510 nhi_write_reg(sc, NHI_RX_RING_SIZE + idx, 0);
511
512 return (0);
513 }
514
515 static int
nhi_alloc_ring0(struct nhi_softc * sc)516 nhi_alloc_ring0(struct nhi_softc *sc)
517 {
518 bus_addr_t frames_busaddr;
519 bus_dma_template_t t;
520 struct nhi_intr_tracker *trkr;
521 struct nhi_ring_pair *r;
522 struct nhi_cmd_frame *cmd;
523 char *frames;
524 int error, size, i;
525
526 if ((error = nhi_alloc_ring(sc, 0, NHI_RING0_TX_DEPTH,
527 NHI_RING0_RX_DEPTH, &r)) != 0) {
528 tb_printf(sc, "Error allocating control ring\n");
529 return (error);
530 }
531
532 r->rx_buffer_size = NHI_RING0_FRAME_SIZE;/* Control packets are small */
533
534 /* Allocate the RX and TX buffers that are used for Ring0 comms */
535 size = r->tx_ring_depth * NHI_RING0_FRAME_SIZE;
536 size += r->rx_ring_depth * NHI_RING0_FRAME_SIZE;
537
538 bus_dma_template_init(&t, sc->parent_dmat);
539 t.maxsize = t.maxsegsize = size;
540 t.nsegments = 1;
541 if (bus_dma_template_tag(&t, &sc->ring0_dmat)) {
542 tb_printf(sc, "Error allocating control ring buffer tag\n");
543 return (ENOMEM);
544 }
545
546 if (bus_dmamem_alloc(sc->ring0_dmat, (void **)&frames, BUS_DMA_NOWAIT,
547 &sc->ring0_map) != 0) {
548 tb_printf(sc, "Error allocating control ring memory\n");
549 return (ENOMEM);
550 }
551 bzero(frames, size);
552 bus_dmamap_load(sc->ring0_dmat, sc->ring0_map, frames, size,
553 nhi_memaddr_cb, &frames_busaddr, 0);
554 sc->ring0_frames_busaddr = frames_busaddr;
555 sc->ring0_frames = frames;
556
557 /* Allocate the driver command trackers */
558 sc->ring0_cmds = malloc(sizeof(struct nhi_cmd_frame) *
559 (r->tx_ring_depth + r->rx_ring_depth), M_NHI, M_NOWAIT | M_ZERO);
560 if (sc->ring0_cmds == NULL)
561 return (ENOMEM);
562
563 /* Initialize the RX frames so they can be used */
564 mtx_lock(&r->mtx);
565 for (i = 0; i < r->rx_ring_depth; i++) {
566 cmd = &sc->ring0_cmds[i];
567 cmd->data = (uint32_t *)(frames + NHI_RING0_FRAME_SIZE * i);
568 cmd->data_busaddr = frames_busaddr + NHI_RING0_FRAME_SIZE * i;
569 cmd->flags = CMD_MAPPED;
570 cmd->idx = i;
571 TAILQ_INSERT_TAIL(&r->rx_head, cmd, cm_link);
572 }
573
574 /* Initialize the TX frames */
575 for ( ; i < r->tx_ring_depth + r->rx_ring_depth - 1; i++) {
576 cmd = &sc->ring0_cmds[i];
577 cmd->data = (uint32_t *)(frames + NHI_RING0_FRAME_SIZE * i);
578 cmd->data_busaddr = frames_busaddr + NHI_RING0_FRAME_SIZE * i;
579 cmd->flags = CMD_MAPPED;
580 cmd->idx = i;
581 nhi_free_tx_frame_locked(r, cmd);
582 }
583 mtx_unlock(&r->mtx);
584
585 /* Do a 1:1 mapping of rings to interrupt vectors. */
586 /* XXX Should be abstracted */
587 trkr = &sc->intr_trackers[0];
588 trkr->ring = r;
589 r->tracker = trkr;
590
591 /* XXX Should be an array */
592 sc->ring0 = r;
593 SLIST_INSERT_HEAD(&sc->ring_list, r, ring_link);
594
595 return (0);
596 }
597
598 static void
nhi_free_ring0(struct nhi_softc * sc)599 nhi_free_ring0(struct nhi_softc *sc)
600 {
601 if (sc->ring0_cmds != NULL) {
602 free(sc->ring0_cmds, M_NHI);
603 sc->ring0_cmds = NULL;
604 }
605
606 if (sc->ring0_frames_busaddr != 0) {
607 bus_dmamap_unload(sc->ring0_dmat, sc->ring0_map);
608 sc->ring0_frames_busaddr = 0;
609 }
610
611 if (sc->ring0_frames != NULL) {
612 bus_dmamem_free(sc->ring0_dmat, sc->ring0_frames,
613 sc->ring0_map);
614 sc->ring0_frames = NULL;
615 }
616
617 if (sc->ring0_dmat != NULL)
618 bus_dma_tag_destroy(sc->ring0_dmat);
619
620 return;
621 }
622
623 static void
nhi_fill_rx_ring(struct nhi_softc * sc,struct nhi_ring_pair * rp)624 nhi_fill_rx_ring(struct nhi_softc *sc, struct nhi_ring_pair *rp)
625 {
626 struct nhi_cmd_frame *cmd;
627 struct nhi_rx_buffer_desc *desc;
628 u_int ci;
629
630 /* Assume that we never grow or shrink the ring population */
631 rp->rx_ci = ci = 0;
632 rp->rx_pi = 0;
633
634 do {
635 cmd = TAILQ_FIRST(&rp->rx_head);
636 if (cmd == NULL)
637 break;
638 TAILQ_REMOVE(&rp->rx_head, cmd, cm_link);
639 desc = &rp->rx_ring[ci].rx;
640 if ((cmd->flags & CMD_MAPPED) == 0)
641 panic("Need rx buffer mapping code");
642
643 desc->addr_lo = cmd->data_busaddr & 0xffffffff;
644 desc->addr_hi = (cmd->data_busaddr >> 32) & 0xffffffff;
645 desc->offset = 0;
646 desc->flags = RX_BUFFER_DESC_RS | RX_BUFFER_DESC_IE;
647 rp->rx_ci = ci;
648 rp->rx_cmd_ring[ci] = cmd;
649 tb_debug(sc, DBG_RXQ | DBG_FULL,
650 "Updating ring%d ci= %d cmd= %p, busaddr= 0x%jx\n",
651 rp->ring_num, ci, cmd, cmd->data_busaddr);
652
653 ci = (rp->rx_ci + 1) & rp->rx_ring_mask;
654 } while (ci != rp->rx_pi);
655
656 /* Update the CI in one shot */
657 tb_debug(sc, DBG_RXQ, "Writing RX CI= %d\n", rp->rx_ci);
658 nhi_write_reg(sc, rp->rx_pici_reg, rp->rx_ci);
659
660 return;
661 }
662
663 static int
nhi_init(struct nhi_softc * sc)664 nhi_init(struct nhi_softc *sc)
665 {
666 tb_route_t root_route = {0x0, 0x0};
667 uint32_t val;
668 int error;
669
670 tb_debug(sc, DBG_INIT, "Initializing NHI\n");
671
672 /* Set interrupt Auto-ACK */
673 val = nhi_read_reg(sc, NHI_DMA_MISC);
674 tb_debug(sc, DBG_INIT|DBG_FULL, "Read NHI_DMA_MISC= 0x%08x\n", val);
675 val |= DMA_MISC_INT_AUTOCLEAR;
676 tb_debug(sc, DBG_INIT, "Setting interrupt auto-ACK, 0x%08x\n", val);
677 nhi_write_reg(sc, NHI_DMA_MISC, val);
678
679 /*
680 * Attach the router to the root thunderbolt bridge now that the DMA
681 * channel is configured and ready.
682 * The root router always has a route of 0x0...0, so set it statically
683 * here.
684 */
685 if ((error = tb_router_attach_root(sc, root_route)) != 0)
686 tb_printf(sc, "tb_router_attach_root() error."
687 " The driver should be loaded at boot\n");
688
689 if (error == 0) {
690 sc->ich.ich_func = nhi_post_init;
691 sc->ich.ich_arg = sc;
692 error = config_intrhook_establish(&sc->ich);
693 if (error)
694 tb_printf(sc, "Failed to establish config hook\n");
695 }
696
697 return (error);
698 }
699
700 static void
nhi_post_init(void * arg)701 nhi_post_init(void *arg)
702 {
703 struct nhi_softc *sc;
704 uint8_t *u;
705 int error;
706
707 sc = (struct nhi_softc *)arg;
708 tb_debug(sc, DBG_INIT | DBG_EXTRA, "nhi_post_init\n");
709
710 bzero(sc->lc_uuid, 16);
711 error = tb_config_get_lc_uuid(sc->root_rsc, sc->lc_uuid);
712 if (error == 0) {
713 u = sc->lc_uuid;
714 tb_printf(sc, "Root Router LC UUID: %02x%02x%02x%02x-"
715 "%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x\n",
716 u[15], u[14], u[13], u[12], u[11], u[10], u[9], u[8], u[7],
717 u[6], u[5], u[4], u[3], u[2], u[1], u[0]);
718 } else
719 tb_printf(sc, "Error finding LC registers: %d\n", error);
720
721 u = sc->uuid;
722 tb_printf(sc, "Root Router UUID: %02x%02x%02x%02x-"
723 "%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x\n",
724 u[15], u[14], u[13], u[12], u[11], u[10], u[9], u[8], u[7],
725 u[6], u[5], u[4], u[3], u[2], u[1], u[0]);
726
727 config_intrhook_disestablish(&sc->ich);
728 }
729
730 static int
nhi_tx_enqueue(struct nhi_ring_pair * r,struct nhi_cmd_frame * cmd)731 nhi_tx_enqueue(struct nhi_ring_pair *r, struct nhi_cmd_frame *cmd)
732 {
733 struct nhi_softc *sc;
734 struct nhi_tx_buffer_desc *desc;
735 uint16_t pi;
736
737 sc = r->sc;
738
739 /* A length of 0 means 4096. Can't have longer lengths */
740 if (cmd->req_len > TX_BUFFER_DESC_LEN_MASK + 1) {
741 tb_debug(sc, DBG_TXQ, "Error: TX frame too big\n");
742 return (EINVAL);
743 }
744 cmd->req_len &= TX_BUFFER_DESC_LEN_MASK;
745
746 mtx_lock(&r->mtx);
747 desc = &r->tx_ring[r->tx_pi].tx;
748 pi = (r->tx_pi + 1) & r->tx_ring_mask;
749 if (pi == r->tx_ci) {
750 mtx_unlock(&r->mtx);
751 return (EBUSY);
752 }
753 r->tx_cmd_ring[r->tx_pi] = cmd;
754 r->tx_pi = pi;
755
756 desc->addr_lo = htole32(cmd->data_busaddr & 0xffffffff);
757 desc->addr_hi = htole32(cmd->data_busaddr >> 32);
758 desc->eof_len = htole16((cmd->pdf << TX_BUFFER_DESC_EOF_SHIFT) |
759 cmd->req_len);
760 desc->flags_sof = cmd->pdf | TX_BUFFER_DESC_IE | TX_BUFFER_DESC_RS;
761 desc->offset = 0;
762 desc->payload_time = 0;
763
764 tb_debug(sc, DBG_TXQ, "enqueue TXdescIdx= %d cmdidx= %d len= %d, "
765 "busaddr= 0x%jx\n", r->tx_pi, cmd->idx, cmd->req_len,
766 cmd->data_busaddr);
767
768 nhi_write_reg(sc, r->tx_pici_reg, pi << TX_RING_PI_SHIFT | r->tx_ci);
769 mtx_unlock(&r->mtx);
770 return (0);
771 }
772
773 /*
774 * No scheduling happens for now. Ring0 scheduling is done in the TB
775 * layer.
776 */
777 int
nhi_tx_schedule(struct nhi_ring_pair * r,struct nhi_cmd_frame * cmd)778 nhi_tx_schedule(struct nhi_ring_pair *r, struct nhi_cmd_frame *cmd)
779 {
780 int error;
781
782 error = nhi_tx_enqueue(r, cmd);
783 if (error == EBUSY)
784 nhi_write_reg(r->sc, r->tx_pici_reg, r->tx_pi << TX_RING_PI_SHIFT | r->tx_ci);
785 return (error);
786 }
787
788 int
nhi_tx_synchronous(struct nhi_ring_pair * r,struct nhi_cmd_frame * cmd)789 nhi_tx_synchronous(struct nhi_ring_pair *r, struct nhi_cmd_frame *cmd)
790 {
791 struct nhi_softc *sc __diagused;
792 int error, count;
793
794 sc = r->sc;
795
796 if ((error = nhi_tx_schedule(r, cmd)) != 0)
797 return (error);
798
799 if (cmd->flags & CMD_POLLED) {
800 error = 0;
801 count = cmd->timeout * 100;
802
803 /* Enter the loop at least once */
804 while ((count-- > 0) && (cmd->flags & CMD_REQ_COMPLETE) == 0) {
805 DELAY(10000);
806 rmb();
807 nhi_intr(r->tracker);
808 }
809 } else {
810 error = msleep(cmd, &r->mtx, PCATCH, "nhi_tx", cmd->timeout);
811 if ((error == 0) && (cmd->flags & CMD_REQ_COMPLETE) != 0)
812 error = EWOULDBLOCK;
813 }
814
815 if ((cmd->flags & CMD_REQ_COMPLETE) == 0)
816 error = ETIMEDOUT;
817
818 tb_debug(sc, DBG_TXQ|DBG_FULL, "tx_synchronous done waiting, "
819 "err= %d, TX_COMPLETE= %d\n", error,
820 !!(cmd->flags & CMD_REQ_COMPLETE));
821
822 if (error == ERESTART) {
823 tb_printf(sc, "TX command interrupted\n");
824 } else if ((error == EWOULDBLOCK) || (error == ETIMEDOUT)) {
825 tb_printf(sc, "TX command timed out\n");
826 } else if (error != 0) {
827 tb_printf(sc, "TX command failed error= %d\n", error);
828 }
829
830 return (error);
831 }
832
833 static int
nhi_tx_complete(struct nhi_ring_pair * r,struct nhi_tx_buffer_desc * desc,struct nhi_cmd_frame * cmd)834 nhi_tx_complete(struct nhi_ring_pair *r, struct nhi_tx_buffer_desc *desc,
835 struct nhi_cmd_frame *cmd)
836 {
837 struct nhi_softc *sc __maybe_unused;
838 struct nhi_pdf_dispatch *txpdf;
839 u_int sof;
840
841 sc = r->sc;
842 sof = desc->flags_sof & TX_BUFFER_DESC_SOF_MASK;
843 tb_debug(sc, DBG_TXQ, "Recovered TX pdf= %s cmdidx= %d flags= 0x%x\n",
844 tb_get_string(sof, nhi_frame_pdf), cmd->idx, desc->flags_sof);
845
846 if ((desc->flags_sof & TX_BUFFER_DESC_DONE) == 0)
847 tb_debug(sc, DBG_TXQ,
848 "warning, TX descriptor DONE flag not set\n");
849
850 /* XXX Atomics */
851 cmd->flags |= CMD_REQ_COMPLETE;
852
853 txpdf = &r->tracker->txpdf[sof];
854 if (txpdf->cb != NULL) {
855 tb_debug(sc, DBG_INTR|DBG_TXQ, "Calling PDF TX callback\n");
856 txpdf->cb(txpdf->context, (union nhi_ring_desc *)desc, cmd);
857 return (0);
858 }
859
860 tb_debug(sc, DBG_TXQ, "Unhandled TX complete %s\n",
861 tb_get_string(sof, nhi_frame_pdf));
862 nhi_free_tx_frame(r, cmd);
863
864 return (0);
865 }
866
867 static int
nhi_rx_complete(struct nhi_ring_pair * r,struct nhi_rx_post_desc * desc,struct nhi_cmd_frame * cmd)868 nhi_rx_complete(struct nhi_ring_pair *r, struct nhi_rx_post_desc *desc,
869 struct nhi_cmd_frame *cmd)
870 {
871 struct nhi_softc *sc __maybe_unused;
872 struct nhi_pdf_dispatch *rxpdf;
873 u_int eof;
874 u_int len __maybe_unused;
875
876 sc = r->sc;
877 eof = desc->eof_len >> RX_BUFFER_DESC_EOF_SHIFT;
878 len = desc->eof_len & RX_BUFFER_DESC_LEN_MASK;
879 tb_debug(sc, DBG_INTR|DBG_RXQ,
880 "Recovered RX pdf= %s len= %d cmdidx= %d, busaddr= 0x%jx\n",
881 tb_get_string(eof, nhi_frame_pdf), len, cmd->idx,
882 cmd->data_busaddr);
883
884 rxpdf = &r->tracker->rxpdf[eof];
885 if (rxpdf->cb != NULL) {
886 tb_debug(sc, DBG_INTR|DBG_RXQ, "Calling PDF RX callback\n");
887 rxpdf->cb(rxpdf->context, (union nhi_ring_desc *)desc, cmd);
888 return (0);
889 }
890
891 tb_debug(sc, DBG_INTR, "Unhandled RX frame %s\n",
892 tb_get_string(eof, nhi_frame_pdf));
893
894 return (0);
895 }
896
897 int
nhi_register_pdf(struct nhi_ring_pair * rp,struct nhi_dispatch * tx,struct nhi_dispatch * rx)898 nhi_register_pdf(struct nhi_ring_pair *rp, struct nhi_dispatch *tx,
899 struct nhi_dispatch *rx)
900 {
901 struct nhi_intr_tracker *trkr;
902 struct nhi_pdf_dispatch *slot;
903
904 KASSERT(rp != NULL, ("ring_pair is null\n"));
905 tb_debug(rp->sc, DBG_INTR|DBG_EXTRA, "nhi_register_pdf called\n");
906
907 trkr = rp->tracker;
908 if (trkr == NULL) {
909 tb_debug(rp->sc, DBG_INTR, "Invalid tracker\n");
910 return (EINVAL);
911 }
912
913 tb_debug(rp->sc, DBG_INTR|DBG_EXTRA, "Registering TX interrupts\n");
914 if (tx != NULL) {
915 while (tx->cb != NULL) {
916 if ((tx->pdf < 0) || (tx->pdf > 15))
917 return (EINVAL);
918 slot = &trkr->txpdf[tx->pdf];
919 if (slot->cb != NULL) {
920 tb_debug(rp->sc, DBG_INTR,
921 "Attempted to register busy callback\n");
922 return (EBUSY);
923 }
924 slot->cb = tx->cb;
925 slot->context = tx->context;
926 tb_debug(rp->sc, DBG_INTR,
927 "Registered TX callback for PDF %d\n", tx->pdf);
928 tx++;
929 }
930 }
931
932 tb_debug(rp->sc, DBG_INTR|DBG_EXTRA, "Registering RX interrupts\n");
933 if (rx != NULL) {
934 while (rx->cb != NULL) {
935 if ((rx->pdf < 0) || (rx->pdf > 15))
936 return (EINVAL);
937 slot = &trkr->rxpdf[rx->pdf];
938 if (slot->cb != NULL) {
939 tb_debug(rp->sc, DBG_INTR,
940 "Attempted to register busy callback\n");
941 return (EBUSY);
942 }
943 slot->cb = rx->cb;
944 slot->context = rx->context;
945 tb_debug(rp->sc, DBG_INTR,
946 "Registered RX callback for PDF %d\n", rx->pdf);
947 rx++;
948 }
949 }
950
951 return (0);
952 }
953
954 int
nhi_deregister_pdf(struct nhi_ring_pair * rp,struct nhi_dispatch * tx,struct nhi_dispatch * rx)955 nhi_deregister_pdf(struct nhi_ring_pair *rp, struct nhi_dispatch *tx,
956 struct nhi_dispatch *rx)
957 {
958 struct nhi_intr_tracker *trkr;
959 struct nhi_pdf_dispatch *slot;
960
961 tb_debug(rp->sc, DBG_INTR|DBG_EXTRA, "nhi_register_pdf called\n");
962
963 trkr = rp->tracker;
964
965 if (tx != NULL) {
966 while (tx->cb != NULL) {
967 if ((tx->pdf < 0) || (tx->pdf > 15))
968 return (EINVAL);
969 slot = &trkr->txpdf[tx->pdf];
970 slot->cb = NULL;
971 slot->context = NULL;
972 tx++;
973 }
974 }
975
976 if (rx != NULL) {
977 while (rx->cb != NULL) {
978 if ((rx->pdf < 0) || (rx->pdf > 15))
979 return (EINVAL);
980 slot = &trkr->rxpdf[rx->pdf];
981 slot->cb = NULL;
982 slot->context = NULL;
983 rx++;
984 }
985 }
986
987 return (0);
988 }
989
990 /*
991 * The CI and PI indexes are not read from the hardware. We track them in
992 * software, so we know where in the ring to start a scan on an interrupt.
993 * All we have to do is check for the appropriate Done bit in the next
994 * descriptor, and we know if we have reached the last descriptor that the
995 * hardware touched. This technique saves at least 2 MEMIO reads per
996 * interrupt.
997 */
998 void
nhi_intr(void * data)999 nhi_intr(void *data)
1000 {
1001 union nhi_ring_desc *rxd;
1002 struct nhi_cmd_frame *cmd;
1003 struct nhi_intr_tracker *trkr = data;
1004 struct nhi_softc *sc;
1005 struct nhi_ring_pair *r;
1006 struct nhi_tx_buffer_desc *txd;
1007 uint32_t val, old_ci;
1008 u_int count;
1009
1010 sc = trkr->sc;
1011
1012 tb_debug(sc, DBG_INTR|DBG_FULL, "Interrupt @ vector %d\n",
1013 trkr->vector);
1014 if ((r = trkr->ring) == NULL)
1015 return;
1016
1017 /*
1018 * Process TX completions from the adapter. Only go through
1019 * the ring once to prevent unbounded looping.
1020 */
1021 count = r->tx_ring_depth;
1022 while (count-- > 0) {
1023 txd = &r->tx_ring[r->tx_ci].tx;
1024 if ((txd->flags_sof & TX_BUFFER_DESC_DONE) == 0)
1025 break;
1026 cmd = r->tx_cmd_ring[r->tx_ci];
1027 tb_debug(sc, DBG_INTR|DBG_TXQ|DBG_FULL,
1028 "Found tx cmdidx= %d cmd= %p\n", r->tx_ci, cmd);
1029
1030 /* Pass the completion up the stack */
1031 nhi_tx_complete(r, txd, cmd);
1032
1033 /*
1034 * Advance to the next item in the ring via the cached
1035 * copy of the CI. Clear the flags so we can detect
1036 * a new done condition the next time the ring wraps
1037 * around. Anything higher up the stack that needs this
1038 * field should have already copied it.
1039 *
1040 * XXX is a memory barrier needed?
1041 */
1042 txd->flags_sof = 0;
1043 r->tx_ci = (r->tx_ci + 1) & r->tx_ring_mask;
1044 }
1045
1046 /* Process RX packets from the adapter */
1047 count = r->rx_ring_depth;
1048 old_ci = r->rx_ci;
1049
1050 while (count-- > 0) {
1051 tb_debug(sc, DBG_INTR|DBG_RXQ|DBG_FULL,
1052 "Checking RX descriptor at %d\n", r->rx_pi);
1053
1054 /* Look up RX descriptor and cmd */
1055 rxd = &r->rx_ring[r->rx_pi];
1056 tb_debug(sc, DBG_INTR|DBG_RXQ|DBG_FULL,
1057 "rx desc len= 0x%04x flags= 0x%04x\n", rxd->rxpost.eof_len,
1058 rxd->rxpost.flags_sof);
1059 if ((rxd->rxpost.flags_sof & RX_BUFFER_DESC_DONE) == 0)
1060 break;
1061 cmd = r->rx_cmd_ring[r->rx_pi];
1062 tb_debug(sc, DBG_INTR|DBG_RXQ|DBG_FULL,
1063 "Found rx cmdidx= %d cmd= %p\n", r->rx_pi, cmd);
1064
1065 /*
1066 * Pass the RX frame up the stack. RX frames are re-used
1067 * in-place, so their contents must be copied before this
1068 * function returns.
1069 *
1070 * XXX Rings other than Ring0 might want to have a different
1071 * re-use and re-populate policy
1072 */
1073 nhi_rx_complete(r, &rxd->rxpost, cmd);
1074
1075 /*
1076 * Advance the CI and move forward to the next item in the
1077 * ring via our cached copy of the PI. Clear out the
1078 * length field so we can detect a new RX frame when the
1079 * ring wraps around. Reset the flags of the descriptor.
1080 */
1081 rxd->rxpost.eof_len = 0;
1082 rxd->rx.flags = RX_BUFFER_DESC_RS | RX_BUFFER_DESC_IE;
1083 r->rx_ci = (r->rx_ci + 1) & r->rx_ring_mask;
1084 r->rx_pi = (r->rx_pi + 1) & r->rx_ring_mask;
1085 }
1086
1087 /*
1088 * Tell the firmware about the new RX CI
1089 *
1090 * XXX There's a chance this will overwrite an update to the PI.
1091 * Is that OK? We keep our own copy of the PI and never read it from
1092 * hardware. However, will overwriting it result in a missed
1093 * interrupt?
1094 */
1095 if (r->rx_ci != old_ci) {
1096 val = r->rx_pi << RX_RING_PI_SHIFT | r->rx_ci;
1097 tb_debug(sc, DBG_INTR | DBG_RXQ,
1098 "Writing new RX PICI= 0x%08x\n", val);
1099 nhi_write_reg(sc, r->rx_pici_reg, val);
1100 }
1101 }
1102
1103 static int
nhi_setup_sysctl(struct nhi_softc * sc)1104 nhi_setup_sysctl(struct nhi_softc *sc)
1105 {
1106 struct sysctl_ctx_list *ctx = NULL;
1107 struct sysctl_oid *tree = NULL;
1108
1109 ctx = device_get_sysctl_ctx(sc->dev);
1110 if (ctx != NULL)
1111 tree = device_get_sysctl_tree(sc->dev);
1112
1113 /*
1114 * Not being able to create sysctls is going to hamper other
1115 * parts of the driver.
1116 */
1117 if (tree == NULL) {
1118 tb_printf(sc, "Error: cannot create sysctl nodes\n");
1119 return (EINVAL);
1120 }
1121 sc->sysctl_tree = tree;
1122 sc->sysctl_ctx = ctx;
1123
1124 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
1125 OID_AUTO, "debug_level", CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_MPSAFE,
1126 &sc->debug, 0, tb_debug_sysctl, "A", "Thunderbolt debug level");
1127 SYSCTL_ADD_U16(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
1128 "max_rings", CTLFLAG_RD, &sc->max_ring_count, 0,
1129 "Max number of rings available");
1130
1131 return (0);
1132 }
1133