xref: /freebsd/sys/dev/ioat/ioat.c (revision 39ee7a7a6bdd1557b1c3532abf60d139798ac88b)
1 /*-
2  * Copyright (C) 2012 Intel Corporation
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/bus.h>
33 #include <sys/conf.h>
34 #include <sys/ioccom.h>
35 #include <sys/kernel.h>
36 #include <sys/lock.h>
37 #include <sys/malloc.h>
38 #include <sys/module.h>
39 #include <sys/mutex.h>
40 #include <sys/rman.h>
41 #include <sys/sysctl.h>
42 #include <sys/time.h>
43 #include <dev/pci/pcireg.h>
44 #include <dev/pci/pcivar.h>
45 #include <machine/bus.h>
46 #include <machine/resource.h>
47 #include <machine/stdarg.h>
48 
49 #include "ioat.h"
50 #include "ioat_hw.h"
51 #include "ioat_internal.h"
52 
53 #define	IOAT_INTR_TIMO	(hz / 10)
54 #define	IOAT_REFLK	(&ioat->submit_lock)
55 
56 static int ioat_probe(device_t device);
57 static int ioat_attach(device_t device);
58 static int ioat_detach(device_t device);
59 static int ioat_setup_intr(struct ioat_softc *ioat);
60 static int ioat_teardown_intr(struct ioat_softc *ioat);
61 static int ioat3_attach(device_t device);
62 static int ioat_start_channel(struct ioat_softc *ioat);
63 static int ioat_map_pci_bar(struct ioat_softc *ioat);
64 static void ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg,
65     int error);
66 static void ioat_interrupt_handler(void *arg);
67 static boolean_t ioat_model_resets_msix(struct ioat_softc *ioat);
68 static void ioat_process_events(struct ioat_softc *ioat);
69 static inline uint32_t ioat_get_active(struct ioat_softc *ioat);
70 static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat);
71 static void ioat_free_ring(struct ioat_softc *, uint32_t size,
72     struct ioat_descriptor **);
73 static void ioat_free_ring_entry(struct ioat_softc *ioat,
74     struct ioat_descriptor *desc);
75 static struct ioat_descriptor *ioat_alloc_ring_entry(struct ioat_softc *,
76     int mflags);
77 static int ioat_reserve_space(struct ioat_softc *, uint32_t, int mflags);
78 static struct ioat_descriptor *ioat_get_ring_entry(struct ioat_softc *ioat,
79     uint32_t index);
80 static struct ioat_descriptor **ioat_prealloc_ring(struct ioat_softc *,
81     uint32_t size, boolean_t need_dscr, int mflags);
82 static int ring_grow(struct ioat_softc *, uint32_t oldorder,
83     struct ioat_descriptor **);
84 static int ring_shrink(struct ioat_softc *, uint32_t oldorder,
85     struct ioat_descriptor **);
86 static void ioat_timer_callback(void *arg);
87 static void dump_descriptor(void *hw_desc);
88 static void ioat_submit_single(struct ioat_softc *ioat);
89 static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg,
90     int error);
91 static int ioat_reset_hw(struct ioat_softc *ioat);
92 static void ioat_setup_sysctl(device_t device);
93 static int sysctl_handle_reset(SYSCTL_HANDLER_ARGS);
94 static inline struct ioat_softc *ioat_get(struct ioat_softc *,
95     enum ioat_ref_kind);
96 static inline void ioat_put(struct ioat_softc *, enum ioat_ref_kind);
97 static inline void ioat_putn(struct ioat_softc *, uint32_t,
98     enum ioat_ref_kind);
99 static void ioat_drain(struct ioat_softc *);
100 
101 #define	ioat_log_message(v, ...) do {					\
102 	if ((v) <= g_ioat_debug_level) {				\
103 		device_printf(ioat->device, __VA_ARGS__);		\
104 	}								\
105 } while (0)
106 
107 MALLOC_DEFINE(M_IOAT, "ioat", "ioat driver memory allocations");
108 SYSCTL_NODE(_hw, OID_AUTO, ioat, CTLFLAG_RD, 0, "ioat node");
109 
110 static int g_force_legacy_interrupts;
111 SYSCTL_INT(_hw_ioat, OID_AUTO, force_legacy_interrupts, CTLFLAG_RDTUN,
112     &g_force_legacy_interrupts, 0, "Set to non-zero to force MSI-X disabled");
113 
114 int g_ioat_debug_level = 0;
115 SYSCTL_INT(_hw_ioat, OID_AUTO, debug_level, CTLFLAG_RWTUN, &g_ioat_debug_level,
116     0, "Set log level (0-3) for ioat(4). Higher is more verbose.");
117 
118 /*
119  * OS <-> Driver interface structures
120  */
121 static device_method_t ioat_pci_methods[] = {
122 	/* Device interface */
123 	DEVMETHOD(device_probe,     ioat_probe),
124 	DEVMETHOD(device_attach,    ioat_attach),
125 	DEVMETHOD(device_detach,    ioat_detach),
126 	{ 0, 0 }
127 };
128 
129 static driver_t ioat_pci_driver = {
130 	"ioat",
131 	ioat_pci_methods,
132 	sizeof(struct ioat_softc),
133 };
134 
135 static devclass_t ioat_devclass;
136 DRIVER_MODULE(ioat, pci, ioat_pci_driver, ioat_devclass, 0, 0);
137 
138 /*
139  * Private data structures
140  */
141 static struct ioat_softc *ioat_channel[IOAT_MAX_CHANNELS];
142 static int ioat_channel_index = 0;
143 SYSCTL_INT(_hw_ioat, OID_AUTO, channels, CTLFLAG_RD, &ioat_channel_index, 0,
144     "Number of IOAT channels attached");
145 
146 static struct _pcsid
147 {
148 	u_int32_t   type;
149 	const char  *desc;
150 } pci_ids[] = {
151 	{ 0x34308086, "TBG IOAT Ch0" },
152 	{ 0x34318086, "TBG IOAT Ch1" },
153 	{ 0x34328086, "TBG IOAT Ch2" },
154 	{ 0x34338086, "TBG IOAT Ch3" },
155 	{ 0x34298086, "TBG IOAT Ch4" },
156 	{ 0x342a8086, "TBG IOAT Ch5" },
157 	{ 0x342b8086, "TBG IOAT Ch6" },
158 	{ 0x342c8086, "TBG IOAT Ch7" },
159 
160 	{ 0x37108086, "JSF IOAT Ch0" },
161 	{ 0x37118086, "JSF IOAT Ch1" },
162 	{ 0x37128086, "JSF IOAT Ch2" },
163 	{ 0x37138086, "JSF IOAT Ch3" },
164 	{ 0x37148086, "JSF IOAT Ch4" },
165 	{ 0x37158086, "JSF IOAT Ch5" },
166 	{ 0x37168086, "JSF IOAT Ch6" },
167 	{ 0x37178086, "JSF IOAT Ch7" },
168 	{ 0x37188086, "JSF IOAT Ch0 (RAID)" },
169 	{ 0x37198086, "JSF IOAT Ch1 (RAID)" },
170 
171 	{ 0x3c208086, "SNB IOAT Ch0" },
172 	{ 0x3c218086, "SNB IOAT Ch1" },
173 	{ 0x3c228086, "SNB IOAT Ch2" },
174 	{ 0x3c238086, "SNB IOAT Ch3" },
175 	{ 0x3c248086, "SNB IOAT Ch4" },
176 	{ 0x3c258086, "SNB IOAT Ch5" },
177 	{ 0x3c268086, "SNB IOAT Ch6" },
178 	{ 0x3c278086, "SNB IOAT Ch7" },
179 	{ 0x3c2e8086, "SNB IOAT Ch0 (RAID)" },
180 	{ 0x3c2f8086, "SNB IOAT Ch1 (RAID)" },
181 
182 	{ 0x0e208086, "IVB IOAT Ch0" },
183 	{ 0x0e218086, "IVB IOAT Ch1" },
184 	{ 0x0e228086, "IVB IOAT Ch2" },
185 	{ 0x0e238086, "IVB IOAT Ch3" },
186 	{ 0x0e248086, "IVB IOAT Ch4" },
187 	{ 0x0e258086, "IVB IOAT Ch5" },
188 	{ 0x0e268086, "IVB IOAT Ch6" },
189 	{ 0x0e278086, "IVB IOAT Ch7" },
190 	{ 0x0e2e8086, "IVB IOAT Ch0 (RAID)" },
191 	{ 0x0e2f8086, "IVB IOAT Ch1 (RAID)" },
192 
193 	{ 0x2f208086, "HSW IOAT Ch0" },
194 	{ 0x2f218086, "HSW IOAT Ch1" },
195 	{ 0x2f228086, "HSW IOAT Ch2" },
196 	{ 0x2f238086, "HSW IOAT Ch3" },
197 	{ 0x2f248086, "HSW IOAT Ch4" },
198 	{ 0x2f258086, "HSW IOAT Ch5" },
199 	{ 0x2f268086, "HSW IOAT Ch6" },
200 	{ 0x2f278086, "HSW IOAT Ch7" },
201 	{ 0x2f2e8086, "HSW IOAT Ch0 (RAID)" },
202 	{ 0x2f2f8086, "HSW IOAT Ch1 (RAID)" },
203 
204 	{ 0x0c508086, "BWD IOAT Ch0" },
205 	{ 0x0c518086, "BWD IOAT Ch1" },
206 	{ 0x0c528086, "BWD IOAT Ch2" },
207 	{ 0x0c538086, "BWD IOAT Ch3" },
208 
209 	{ 0x6f508086, "BDXDE IOAT Ch0" },
210 	{ 0x6f518086, "BDXDE IOAT Ch1" },
211 	{ 0x6f528086, "BDXDE IOAT Ch2" },
212 	{ 0x6f538086, "BDXDE IOAT Ch3" },
213 
214 	{ 0x00000000, NULL           }
215 };
216 
217 /*
218  * OS <-> Driver linkage functions
219  */
220 static int
221 ioat_probe(device_t device)
222 {
223 	struct _pcsid *ep;
224 	u_int32_t type;
225 
226 	type = pci_get_devid(device);
227 	for (ep = pci_ids; ep->type; ep++) {
228 		if (ep->type == type) {
229 			device_set_desc(device, ep->desc);
230 			return (0);
231 		}
232 	}
233 	return (ENXIO);
234 }
235 
236 static int
237 ioat_attach(device_t device)
238 {
239 	struct ioat_softc *ioat;
240 	int error;
241 
242 	ioat = DEVICE2SOFTC(device);
243 	ioat->device = device;
244 
245 	error = ioat_map_pci_bar(ioat);
246 	if (error != 0)
247 		goto err;
248 
249 	ioat->version = ioat_read_cbver(ioat);
250 	if (ioat->version < IOAT_VER_3_0) {
251 		error = ENODEV;
252 		goto err;
253 	}
254 
255 	error = ioat3_attach(device);
256 	if (error != 0)
257 		goto err;
258 
259 	error = pci_enable_busmaster(device);
260 	if (error != 0)
261 		goto err;
262 
263 	error = ioat_setup_intr(ioat);
264 	if (error != 0)
265 		goto err;
266 
267 	error = ioat_reset_hw(ioat);
268 	if (error != 0)
269 		goto err;
270 
271 	ioat_process_events(ioat);
272 	ioat_setup_sysctl(device);
273 
274 	ioat_channel[ioat_channel_index++] = ioat;
275 	ioat_test_attach();
276 
277 err:
278 	if (error != 0)
279 		ioat_detach(device);
280 	return (error);
281 }
282 
283 static int
284 ioat_detach(device_t device)
285 {
286 	struct ioat_softc *ioat;
287 
288 	ioat = DEVICE2SOFTC(device);
289 
290 	ioat_test_detach();
291 	ioat_drain(ioat);
292 
293 	ioat_teardown_intr(ioat);
294 	callout_drain(&ioat->timer);
295 
296 	pci_disable_busmaster(device);
297 
298 	if (ioat->pci_resource != NULL)
299 		bus_release_resource(device, SYS_RES_MEMORY,
300 		    ioat->pci_resource_id, ioat->pci_resource);
301 
302 	if (ioat->ring != NULL)
303 		ioat_free_ring(ioat, 1 << ioat->ring_size_order, ioat->ring);
304 
305 	if (ioat->comp_update != NULL) {
306 		bus_dmamap_unload(ioat->comp_update_tag, ioat->comp_update_map);
307 		bus_dmamem_free(ioat->comp_update_tag, ioat->comp_update,
308 		    ioat->comp_update_map);
309 		bus_dma_tag_destroy(ioat->comp_update_tag);
310 	}
311 
312 	bus_dma_tag_destroy(ioat->hw_desc_tag);
313 
314 	return (0);
315 }
316 
317 static int
318 ioat_teardown_intr(struct ioat_softc *ioat)
319 {
320 
321 	if (ioat->tag != NULL)
322 		bus_teardown_intr(ioat->device, ioat->res, ioat->tag);
323 
324 	if (ioat->res != NULL)
325 		bus_release_resource(ioat->device, SYS_RES_IRQ,
326 		    rman_get_rid(ioat->res), ioat->res);
327 
328 	pci_release_msi(ioat->device);
329 	return (0);
330 }
331 
332 static int
333 ioat_start_channel(struct ioat_softc *ioat)
334 {
335 	uint64_t status;
336 	uint32_t chanerr;
337 	int i;
338 
339 	ioat_acquire(&ioat->dmaengine);
340 	ioat_null(&ioat->dmaengine, NULL, NULL, 0);
341 	ioat_release(&ioat->dmaengine);
342 
343 	for (i = 0; i < 100; i++) {
344 		DELAY(1);
345 		status = ioat_get_chansts(ioat);
346 		if (is_ioat_idle(status))
347 			return (0);
348 	}
349 
350 	chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
351 	ioat_log_message(0, "could not start channel: "
352 	    "status = %#jx error = %b\n", (uintmax_t)status, (int)chanerr,
353 	    IOAT_CHANERR_STR);
354 	return (ENXIO);
355 }
356 
357 /*
358  * Initialize Hardware
359  */
360 static int
361 ioat3_attach(device_t device)
362 {
363 	struct ioat_softc *ioat;
364 	struct ioat_descriptor **ring;
365 	struct ioat_descriptor *next;
366 	struct ioat_dma_hw_descriptor *dma_hw_desc;
367 	uint32_t capabilities;
368 	int i, num_descriptors;
369 	int error;
370 	uint8_t xfercap;
371 
372 	error = 0;
373 	ioat = DEVICE2SOFTC(device);
374 	capabilities = ioat_read_dmacapability(ioat);
375 
376 	xfercap = ioat_read_xfercap(ioat);
377 	ioat->max_xfer_size = 1 << xfercap;
378 
379 	/* TODO: need to check DCA here if we ever do XOR/PQ */
380 
381 	mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF);
382 	mtx_init(&ioat->cleanup_lock, "ioat_process_events", NULL, MTX_DEF);
383 	callout_init(&ioat->timer, 1);
384 
385 	ioat->is_resize_pending = FALSE;
386 	ioat->is_completion_pending = FALSE;
387 	ioat->is_reset_pending = FALSE;
388 	ioat->is_channel_running = FALSE;
389 
390 	bus_dma_tag_create(bus_get_dma_tag(ioat->device), sizeof(uint64_t), 0x0,
391 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
392 	    sizeof(uint64_t), 1, sizeof(uint64_t), 0, NULL, NULL,
393 	    &ioat->comp_update_tag);
394 
395 	error = bus_dmamem_alloc(ioat->comp_update_tag,
396 	    (void **)&ioat->comp_update, BUS_DMA_ZERO, &ioat->comp_update_map);
397 	if (ioat->comp_update == NULL)
398 		return (ENOMEM);
399 
400 	error = bus_dmamap_load(ioat->comp_update_tag, ioat->comp_update_map,
401 	    ioat->comp_update, sizeof(uint64_t), ioat_comp_update_map, ioat,
402 	    0);
403 	if (error != 0)
404 		return (error);
405 
406 	ioat->ring_size_order = IOAT_MIN_ORDER;
407 
408 	num_descriptors = 1 << ioat->ring_size_order;
409 
410 	bus_dma_tag_create(bus_get_dma_tag(ioat->device), 0x40, 0x0,
411 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
412 	    sizeof(struct ioat_dma_hw_descriptor), 1,
413 	    sizeof(struct ioat_dma_hw_descriptor), 0, NULL, NULL,
414 	    &ioat->hw_desc_tag);
415 
416 	ioat->ring = malloc(num_descriptors * sizeof(*ring), M_IOAT,
417 	    M_ZERO | M_WAITOK);
418 	if (ioat->ring == NULL)
419 		return (ENOMEM);
420 
421 	ring = ioat->ring;
422 	for (i = 0; i < num_descriptors; i++) {
423 		ring[i] = ioat_alloc_ring_entry(ioat, M_WAITOK);
424 		if (ring[i] == NULL)
425 			return (ENOMEM);
426 
427 		ring[i]->id = i;
428 	}
429 
430 	for (i = 0; i < num_descriptors - 1; i++) {
431 		next = ring[i + 1];
432 		dma_hw_desc = ring[i]->u.dma;
433 
434 		dma_hw_desc->next = next->hw_desc_bus_addr;
435 	}
436 
437 	ring[i]->u.dma->next = ring[0]->hw_desc_bus_addr;
438 
439 	ioat->head = ioat->hw_head = 0;
440 	ioat->tail = 0;
441 	ioat->last_seen = 0;
442 	return (0);
443 }
444 
445 static int
446 ioat_map_pci_bar(struct ioat_softc *ioat)
447 {
448 
449 	ioat->pci_resource_id = PCIR_BAR(0);
450 	ioat->pci_resource = bus_alloc_resource_any(ioat->device,
451 	    SYS_RES_MEMORY, &ioat->pci_resource_id, RF_ACTIVE);
452 
453 	if (ioat->pci_resource == NULL) {
454 		ioat_log_message(0, "unable to allocate pci resource\n");
455 		return (ENODEV);
456 	}
457 
458 	ioat->pci_bus_tag = rman_get_bustag(ioat->pci_resource);
459 	ioat->pci_bus_handle = rman_get_bushandle(ioat->pci_resource);
460 	return (0);
461 }
462 
463 static void
464 ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
465 {
466 	struct ioat_softc *ioat = arg;
467 
468 	KASSERT(error == 0, ("%s: error:%d", __func__, error));
469 	ioat->comp_update_bus_addr = seg[0].ds_addr;
470 }
471 
472 static void
473 ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
474 {
475 	bus_addr_t *baddr;
476 
477 	KASSERT(error == 0, ("%s: error:%d", __func__, error));
478 	baddr = arg;
479 	*baddr = segs->ds_addr;
480 }
481 
482 /*
483  * Interrupt setup and handlers
484  */
485 static int
486 ioat_setup_intr(struct ioat_softc *ioat)
487 {
488 	uint32_t num_vectors;
489 	int error;
490 	boolean_t use_msix;
491 	boolean_t force_legacy_interrupts;
492 
493 	use_msix = FALSE;
494 	force_legacy_interrupts = FALSE;
495 
496 	if (!g_force_legacy_interrupts && pci_msix_count(ioat->device) >= 1) {
497 		num_vectors = 1;
498 		pci_alloc_msix(ioat->device, &num_vectors);
499 		if (num_vectors == 1)
500 			use_msix = TRUE;
501 	}
502 
503 	if (use_msix) {
504 		ioat->rid = 1;
505 		ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ,
506 		    &ioat->rid, RF_ACTIVE);
507 	} else {
508 		ioat->rid = 0;
509 		ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ,
510 		    &ioat->rid, RF_SHAREABLE | RF_ACTIVE);
511 	}
512 	if (ioat->res == NULL) {
513 		ioat_log_message(0, "bus_alloc_resource failed\n");
514 		return (ENOMEM);
515 	}
516 
517 	ioat->tag = NULL;
518 	error = bus_setup_intr(ioat->device, ioat->res, INTR_MPSAFE |
519 	    INTR_TYPE_MISC, NULL, ioat_interrupt_handler, ioat, &ioat->tag);
520 	if (error != 0) {
521 		ioat_log_message(0, "bus_setup_intr failed\n");
522 		return (error);
523 	}
524 
525 	ioat_write_intrctrl(ioat, IOAT_INTRCTRL_MASTER_INT_EN);
526 	return (0);
527 }
528 
529 static boolean_t
530 ioat_model_resets_msix(struct ioat_softc *ioat)
531 {
532 	u_int32_t pciid;
533 
534 	pciid = pci_get_devid(ioat->device);
535 	switch (pciid) {
536 		/* BWD: */
537 	case 0x0c508086:
538 	case 0x0c518086:
539 	case 0x0c528086:
540 	case 0x0c538086:
541 		/* BDXDE: */
542 	case 0x6f508086:
543 	case 0x6f518086:
544 	case 0x6f528086:
545 	case 0x6f538086:
546 		return (TRUE);
547 	}
548 
549 	return (FALSE);
550 }
551 
552 static void
553 ioat_interrupt_handler(void *arg)
554 {
555 	struct ioat_softc *ioat = arg;
556 
557 	ioat_process_events(ioat);
558 }
559 
560 static void
561 ioat_process_events(struct ioat_softc *ioat)
562 {
563 	struct ioat_descriptor *desc;
564 	struct bus_dmadesc *dmadesc;
565 	uint64_t comp_update, status;
566 	uint32_t completed;
567 
568 	mtx_lock(&ioat->cleanup_lock);
569 
570 	completed = 0;
571 	comp_update = *ioat->comp_update;
572 	status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK;
573 
574 	CTR0(KTR_IOAT, __func__);
575 
576 	if (status == ioat->last_seen)
577 		goto out;
578 
579 	while (1) {
580 		desc = ioat_get_ring_entry(ioat, ioat->tail);
581 		dmadesc = &desc->bus_dmadesc;
582 		CTR1(KTR_IOAT, "completing desc %d", ioat->tail);
583 
584 		if (dmadesc->callback_fn)
585 			(*dmadesc->callback_fn)(dmadesc->callback_arg);
586 
587 		completed++;
588 		ioat->tail++;
589 		if (desc->hw_desc_bus_addr == status)
590 			break;
591 	}
592 
593 	ioat->last_seen = desc->hw_desc_bus_addr;
594 
595 	if (ioat->head == ioat->tail) {
596 		ioat->is_completion_pending = FALSE;
597 		callout_reset(&ioat->timer, IOAT_INTR_TIMO,
598 		    ioat_timer_callback, ioat);
599 	}
600 
601 out:
602 	ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
603 	mtx_unlock(&ioat->cleanup_lock);
604 
605 	ioat_putn(ioat, completed, IOAT_ACTIVE_DESCR_REF);
606 	wakeup(&ioat->tail);
607 }
608 
609 /*
610  * User API functions
611  */
612 bus_dmaengine_t
613 ioat_get_dmaengine(uint32_t index)
614 {
615 
616 	if (index >= ioat_channel_index)
617 		return (NULL);
618 	return (&ioat_get(ioat_channel[index], IOAT_DMAENGINE_REF)->dmaengine);
619 }
620 
621 void
622 ioat_put_dmaengine(bus_dmaengine_t dmaengine)
623 {
624 	struct ioat_softc *ioat;
625 
626 	ioat = to_ioat_softc(dmaengine);
627 	ioat_put(ioat, IOAT_DMAENGINE_REF);
628 }
629 
630 void
631 ioat_acquire(bus_dmaengine_t dmaengine)
632 {
633 	struct ioat_softc *ioat;
634 
635 	ioat = to_ioat_softc(dmaengine);
636 	mtx_lock(&ioat->submit_lock);
637 	CTR0(KTR_IOAT, __func__);
638 }
639 
640 void
641 ioat_release(bus_dmaengine_t dmaengine)
642 {
643 	struct ioat_softc *ioat;
644 
645 	ioat = to_ioat_softc(dmaengine);
646 	CTR0(KTR_IOAT, __func__);
647 	ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, (uint16_t)ioat->hw_head);
648 	mtx_unlock(&ioat->submit_lock);
649 }
650 
651 static struct ioat_descriptor *
652 ioat_op_generic(struct ioat_softc *ioat, uint8_t op,
653     uint32_t size, uint64_t src, uint64_t dst,
654     bus_dmaengine_callback_t callback_fn, void *callback_arg,
655     uint32_t flags)
656 {
657 	struct ioat_generic_hw_descriptor *hw_desc;
658 	struct ioat_descriptor *desc;
659 	int mflags;
660 
661 	mtx_assert(&ioat->submit_lock, MA_OWNED);
662 
663 	KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x",
664 		flags & ~DMA_ALL_FLAGS));
665 	if ((flags & DMA_NO_WAIT) != 0)
666 		mflags = M_NOWAIT;
667 	else
668 		mflags = M_WAITOK;
669 
670 	if (size > ioat->max_xfer_size) {
671 		ioat_log_message(0, "%s: max_xfer_size = %d, requested = %u\n",
672 		    __func__, ioat->max_xfer_size, (unsigned)size);
673 		return (NULL);
674 	}
675 
676 	if (ioat_reserve_space(ioat, 1, mflags) != 0)
677 		return (NULL);
678 
679 	desc = ioat_get_ring_entry(ioat, ioat->head);
680 	hw_desc = desc->u.generic;
681 
682 	hw_desc->u.control_raw = 0;
683 	hw_desc->u.control_generic.op = op;
684 	hw_desc->u.control_generic.completion_update = 1;
685 
686 	if ((flags & DMA_INT_EN) != 0)
687 		hw_desc->u.control_generic.int_enable = 1;
688 
689 	hw_desc->size = size;
690 	hw_desc->src_addr = src;
691 	hw_desc->dest_addr = dst;
692 
693 	desc->bus_dmadesc.callback_fn = callback_fn;
694 	desc->bus_dmadesc.callback_arg = callback_arg;
695 	return (desc);
696 }
697 
698 struct bus_dmadesc *
699 ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_callback_t callback_fn,
700     void *callback_arg, uint32_t flags)
701 {
702 	struct ioat_dma_hw_descriptor *hw_desc;
703 	struct ioat_descriptor *desc;
704 	struct ioat_softc *ioat;
705 
706 	CTR0(KTR_IOAT, __func__);
707 	ioat = to_ioat_softc(dmaengine);
708 
709 	desc = ioat_op_generic(ioat, IOAT_OP_COPY, 8, 0, 0, callback_fn,
710 	    callback_arg, flags);
711 	if (desc == NULL)
712 		return (NULL);
713 
714 	hw_desc = desc->u.dma;
715 	hw_desc->u.control.null = 1;
716 	ioat_submit_single(ioat);
717 	return (&desc->bus_dmadesc);
718 }
719 
720 struct bus_dmadesc *
721 ioat_copy(bus_dmaengine_t dmaengine, bus_addr_t dst,
722     bus_addr_t src, bus_size_t len, bus_dmaengine_callback_t callback_fn,
723     void *callback_arg, uint32_t flags)
724 {
725 	struct ioat_dma_hw_descriptor *hw_desc;
726 	struct ioat_descriptor *desc;
727 	struct ioat_softc *ioat;
728 
729 	CTR0(KTR_IOAT, __func__);
730 	ioat = to_ioat_softc(dmaengine);
731 
732 	if (((src | dst) & (0xffffull << 48)) != 0) {
733 		ioat_log_message(0, "%s: High 16 bits of src/dst invalid\n",
734 		    __func__);
735 		return (NULL);
736 	}
737 
738 	desc = ioat_op_generic(ioat, IOAT_OP_COPY, len, src, dst, callback_fn,
739 	    callback_arg, flags);
740 	if (desc == NULL)
741 		return (NULL);
742 
743 	hw_desc = desc->u.dma;
744 	if (g_ioat_debug_level >= 3)
745 		dump_descriptor(hw_desc);
746 
747 	ioat_submit_single(ioat);
748 	return (&desc->bus_dmadesc);
749 }
750 
751 struct bus_dmadesc *
752 ioat_blockfill(bus_dmaengine_t dmaengine, bus_addr_t dst, uint64_t fillpattern,
753     bus_size_t len, bus_dmaengine_callback_t callback_fn, void *callback_arg,
754     uint32_t flags)
755 {
756 	struct ioat_fill_hw_descriptor *hw_desc;
757 	struct ioat_descriptor *desc;
758 	struct ioat_softc *ioat;
759 
760 	CTR0(KTR_IOAT, __func__);
761 	ioat = to_ioat_softc(dmaengine);
762 
763 	if ((dst & (0xffffull << 48)) != 0) {
764 		ioat_log_message(0, "%s: High 16 bits of dst invalid\n",
765 		    __func__);
766 		return (NULL);
767 	}
768 
769 	desc = ioat_op_generic(ioat, IOAT_OP_FILL, len, fillpattern, dst,
770 	    callback_fn, callback_arg, flags);
771 	if (desc == NULL)
772 		return (NULL);
773 
774 	hw_desc = desc->u.fill;
775 	if (g_ioat_debug_level >= 3)
776 		dump_descriptor(hw_desc);
777 
778 	ioat_submit_single(ioat);
779 	return (&desc->bus_dmadesc);
780 }
781 
782 /*
783  * Ring Management
784  */
785 static inline uint32_t
786 ioat_get_active(struct ioat_softc *ioat)
787 {
788 
789 	return ((ioat->head - ioat->tail) & ((1 << ioat->ring_size_order) - 1));
790 }
791 
792 static inline uint32_t
793 ioat_get_ring_space(struct ioat_softc *ioat)
794 {
795 
796 	return ((1 << ioat->ring_size_order) - ioat_get_active(ioat) - 1);
797 }
798 
799 static struct ioat_descriptor *
800 ioat_alloc_ring_entry(struct ioat_softc *ioat, int mflags)
801 {
802 	struct ioat_generic_hw_descriptor *hw_desc;
803 	struct ioat_descriptor *desc;
804 	int error, busdmaflag;
805 
806 	error = ENOMEM;
807 	hw_desc = NULL;
808 
809 	if ((mflags & M_WAITOK) != 0)
810 		busdmaflag = BUS_DMA_WAITOK;
811 	else
812 		busdmaflag = BUS_DMA_NOWAIT;
813 
814 	desc = malloc(sizeof(*desc), M_IOAT, mflags);
815 	if (desc == NULL)
816 		goto out;
817 
818 	bus_dmamem_alloc(ioat->hw_desc_tag, (void **)&hw_desc,
819 	    BUS_DMA_ZERO | busdmaflag, &ioat->hw_desc_map);
820 	if (hw_desc == NULL)
821 		goto out;
822 
823 	desc->u.generic = hw_desc;
824 
825 	error = bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc,
826 	    sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr,
827 	    busdmaflag);
828 	if (error)
829 		goto out;
830 
831 out:
832 	if (error) {
833 		ioat_free_ring_entry(ioat, desc);
834 		return (NULL);
835 	}
836 	return (desc);
837 }
838 
839 static void
840 ioat_free_ring_entry(struct ioat_softc *ioat, struct ioat_descriptor *desc)
841 {
842 
843 	if (desc == NULL)
844 		return;
845 
846 	if (desc->u.generic)
847 		bus_dmamem_free(ioat->hw_desc_tag, desc->u.generic,
848 		    ioat->hw_desc_map);
849 	free(desc, M_IOAT);
850 }
851 
852 /*
853  * Reserves space in this IOAT descriptor ring by ensuring enough slots remain
854  * for 'num_descs'.
855  *
856  * If mflags contains M_WAITOK, blocks until enough space is available.
857  *
858  * Returns zero on success, or an errno on error.  If num_descs is beyond the
859  * maximum ring size, returns EINVAl; if allocation would block and mflags
860  * contains M_NOWAIT, returns EAGAIN.
861  *
862  * Must be called with the submit_lock held; returns with the lock held.  The
863  * lock may be dropped to allocate the ring.
864  *
865  * (The submit_lock is needed to add any entries to the ring, so callers are
866  * assured enough room is available.)
867  */
868 static int
869 ioat_reserve_space(struct ioat_softc *ioat, uint32_t num_descs, int mflags)
870 {
871 	struct ioat_descriptor **new_ring;
872 	uint32_t order;
873 	int error;
874 
875 	mtx_assert(&ioat->submit_lock, MA_OWNED);
876 	error = 0;
877 
878 	if (num_descs < 1 || num_descs > (1 << IOAT_MAX_ORDER)) {
879 		error = EINVAL;
880 		goto out;
881 	}
882 
883 	for (;;) {
884 		if (ioat_get_ring_space(ioat) >= num_descs)
885 			goto out;
886 
887 		order = ioat->ring_size_order;
888 		if (ioat->is_resize_pending || order == IOAT_MAX_ORDER) {
889 			if ((mflags & M_WAITOK) != 0) {
890 				msleep(&ioat->tail, &ioat->submit_lock, 0,
891 				    "ioat_rsz", 0);
892 				continue;
893 			}
894 
895 			error = EAGAIN;
896 			break;
897 		}
898 
899 		ioat->is_resize_pending = TRUE;
900 		for (;;) {
901 			mtx_unlock(&ioat->submit_lock);
902 
903 			new_ring = ioat_prealloc_ring(ioat, 1 << (order + 1),
904 			    TRUE, mflags);
905 
906 			mtx_lock(&ioat->submit_lock);
907 			KASSERT(ioat->ring_size_order == order,
908 			    ("is_resize_pending should protect order"));
909 
910 			if (new_ring == NULL) {
911 				KASSERT((mflags & M_WAITOK) == 0,
912 				    ("allocation failed"));
913 				error = EAGAIN;
914 				break;
915 			}
916 
917 			error = ring_grow(ioat, order, new_ring);
918 			if (error == 0)
919 				break;
920 		}
921 		ioat->is_resize_pending = FALSE;
922 		wakeup(&ioat->tail);
923 		if (error)
924 			break;
925 	}
926 
927 out:
928 	mtx_assert(&ioat->submit_lock, MA_OWNED);
929 	return (error);
930 }
931 
932 static struct ioat_descriptor **
933 ioat_prealloc_ring(struct ioat_softc *ioat, uint32_t size, boolean_t need_dscr,
934     int mflags)
935 {
936 	struct ioat_descriptor **ring;
937 	uint32_t i;
938 	int error;
939 
940 	KASSERT(size > 0 && powerof2(size), ("bogus size"));
941 
942 	ring = malloc(size * sizeof(*ring), M_IOAT, M_ZERO | mflags);
943 	if (ring == NULL)
944 		return (NULL);
945 
946 	if (need_dscr) {
947 		error = ENOMEM;
948 		for (i = size / 2; i < size; i++) {
949 			ring[i] = ioat_alloc_ring_entry(ioat, mflags);
950 			if (ring[i] == NULL)
951 				goto out;
952 			ring[i]->id = i;
953 		}
954 	}
955 	error = 0;
956 
957 out:
958 	if (error != 0 && ring != NULL) {
959 		ioat_free_ring(ioat, size, ring);
960 		ring = NULL;
961 	}
962 	return (ring);
963 }
964 
965 static void
966 ioat_free_ring(struct ioat_softc *ioat, uint32_t size,
967     struct ioat_descriptor **ring)
968 {
969 	uint32_t i;
970 
971 	for (i = 0; i < size; i++) {
972 		if (ring[i] != NULL)
973 			ioat_free_ring_entry(ioat, ring[i]);
974 	}
975 	free(ring, M_IOAT);
976 }
977 
978 static struct ioat_descriptor *
979 ioat_get_ring_entry(struct ioat_softc *ioat, uint32_t index)
980 {
981 
982 	return (ioat->ring[index % (1 << ioat->ring_size_order)]);
983 }
984 
985 static int
986 ring_grow(struct ioat_softc *ioat, uint32_t oldorder,
987     struct ioat_descriptor **newring)
988 {
989 	struct ioat_descriptor *tmp, *next;
990 	struct ioat_dma_hw_descriptor *hw;
991 	uint32_t oldsize, newsize, head, tail, i, end;
992 	int error;
993 
994 	CTR0(KTR_IOAT, __func__);
995 
996 	mtx_assert(&ioat->submit_lock, MA_OWNED);
997 
998 	if (oldorder != ioat->ring_size_order || oldorder >= IOAT_MAX_ORDER) {
999 		error = EINVAL;
1000 		goto out;
1001 	}
1002 
1003 	oldsize = (1 << oldorder);
1004 	newsize = (1 << (oldorder + 1));
1005 
1006 	mtx_lock(&ioat->cleanup_lock);
1007 
1008 	head = ioat->head & (oldsize - 1);
1009 	tail = ioat->tail & (oldsize - 1);
1010 
1011 	/* Copy old descriptors to new ring */
1012 	for (i = 0; i < oldsize; i++)
1013 		newring[i] = ioat->ring[i];
1014 
1015 	/*
1016 	 * If head has wrapped but tail hasn't, we must swap some descriptors
1017 	 * around so that tail can increment directly to head.
1018 	 */
1019 	if (head < tail) {
1020 		for (i = 0; i <= head; i++) {
1021 			tmp = newring[oldsize + i];
1022 
1023 			newring[oldsize + i] = newring[i];
1024 			newring[oldsize + i]->id = oldsize + i;
1025 
1026 			newring[i] = tmp;
1027 			newring[i]->id = i;
1028 		}
1029 		head += oldsize;
1030 	}
1031 
1032 	KASSERT(head >= tail, ("invariants"));
1033 
1034 	/* Head didn't wrap; we only need to link in oldsize..newsize */
1035 	if (head < oldsize) {
1036 		i = oldsize - 1;
1037 		end = newsize;
1038 	} else {
1039 		/* Head did wrap; link newhead..newsize and 0..oldhead */
1040 		i = head;
1041 		end = newsize + (head - oldsize) + 1;
1042 	}
1043 
1044 	/*
1045 	 * Fix up hardware ring, being careful not to trample the active
1046 	 * section (tail -> head).
1047 	 */
1048 	for (; i < end; i++) {
1049 		KASSERT((i & (newsize - 1)) < tail ||
1050 		    (i & (newsize - 1)) >= head, ("trampling snake"));
1051 
1052 		next = newring[(i + 1) & (newsize - 1)];
1053 		hw = newring[i & (newsize - 1)]->u.dma;
1054 		hw->next = next->hw_desc_bus_addr;
1055 	}
1056 
1057 	free(ioat->ring, M_IOAT);
1058 	ioat->ring = newring;
1059 	ioat->ring_size_order = oldorder + 1;
1060 	ioat->tail = tail;
1061 	ioat->head = head;
1062 	error = 0;
1063 
1064 	mtx_unlock(&ioat->cleanup_lock);
1065 out:
1066 	if (error)
1067 		ioat_free_ring(ioat, (1 << (oldorder + 1)), newring);
1068 	return (error);
1069 }
1070 
1071 static int
1072 ring_shrink(struct ioat_softc *ioat, uint32_t oldorder,
1073     struct ioat_descriptor **newring)
1074 {
1075 	struct ioat_dma_hw_descriptor *hw;
1076 	struct ioat_descriptor *ent, *next;
1077 	uint32_t oldsize, newsize, current_idx, new_idx, i;
1078 	int error;
1079 
1080 	CTR0(KTR_IOAT, __func__);
1081 
1082 	mtx_assert(&ioat->submit_lock, MA_OWNED);
1083 
1084 	if (oldorder != ioat->ring_size_order || oldorder <= IOAT_MIN_ORDER) {
1085 		error = EINVAL;
1086 		goto out_unlocked;
1087 	}
1088 
1089 	oldsize = (1 << oldorder);
1090 	newsize = (1 << (oldorder - 1));
1091 
1092 	mtx_lock(&ioat->cleanup_lock);
1093 
1094 	/* Can't shrink below current active set! */
1095 	if (ioat_get_active(ioat) >= newsize) {
1096 		error = ENOMEM;
1097 		goto out;
1098 	}
1099 
1100 	/*
1101 	 * Copy current descriptors to the new ring, dropping the removed
1102 	 * descriptors.
1103 	 */
1104 	for (i = 0; i < newsize; i++) {
1105 		current_idx = (ioat->tail + i) & (oldsize - 1);
1106 		new_idx = (ioat->tail + i) & (newsize - 1);
1107 
1108 		newring[new_idx] = ioat->ring[current_idx];
1109 		newring[new_idx]->id = new_idx;
1110 	}
1111 
1112 	/* Free deleted descriptors */
1113 	for (i = newsize; i < oldsize; i++) {
1114 		ent = ioat_get_ring_entry(ioat, ioat->tail + i);
1115 		ioat_free_ring_entry(ioat, ent);
1116 	}
1117 
1118 	/* Fix up hardware ring. */
1119 	hw = newring[(ioat->tail + newsize - 1) & (newsize - 1)]->u.dma;
1120 	next = newring[(ioat->tail + newsize) & (newsize - 1)];
1121 	hw->next = next->hw_desc_bus_addr;
1122 
1123 	free(ioat->ring, M_IOAT);
1124 	ioat->ring = newring;
1125 	ioat->ring_size_order = oldorder - 1;
1126 	error = 0;
1127 
1128 out:
1129 	mtx_unlock(&ioat->cleanup_lock);
1130 out_unlocked:
1131 	if (error)
1132 		ioat_free_ring(ioat, (1 << (oldorder - 1)), newring);
1133 	return (error);
1134 }
1135 
1136 static void
1137 ioat_halted_debug(struct ioat_softc *ioat, uint32_t chanerr)
1138 {
1139 	struct ioat_descriptor *desc;
1140 
1141 	ioat_log_message(0, "Channel halted (%b)\n", (int)chanerr,
1142 	    IOAT_CHANERR_STR);
1143 	if (chanerr == 0)
1144 		return;
1145 
1146 	mtx_lock(&ioat->submit_lock);
1147 	desc = ioat_get_ring_entry(ioat, ioat->tail + 0);
1148 	dump_descriptor(desc->u.raw);
1149 
1150 	desc = ioat_get_ring_entry(ioat, ioat->tail + 1);
1151 	dump_descriptor(desc->u.raw);
1152 	mtx_unlock(&ioat->submit_lock);
1153 }
1154 
1155 static void
1156 ioat_timer_callback(void *arg)
1157 {
1158 	struct ioat_descriptor **newring;
1159 	struct ioat_softc *ioat;
1160 	uint64_t status;
1161 	uint32_t chanerr, order;
1162 
1163 	ioat = arg;
1164 	ioat_log_message(1, "%s\n", __func__);
1165 
1166 	if (ioat->is_completion_pending) {
1167 		status = ioat_get_chansts(ioat);
1168 
1169 		/*
1170 		 * When halted due to errors, check for channel programming
1171 		 * errors before advancing the completion state.
1172 		 */
1173 		if (is_ioat_halted(status)) {
1174 			chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
1175 			ioat_halted_debug(ioat, chanerr);
1176 		}
1177 		ioat_process_events(ioat);
1178 	} else {
1179 		mtx_lock(&ioat->submit_lock);
1180 		order = ioat->ring_size_order;
1181 		if (ioat->is_resize_pending || order == IOAT_MIN_ORDER) {
1182 			mtx_unlock(&ioat->submit_lock);
1183 			goto out;
1184 		}
1185 		ioat->is_resize_pending = TRUE;
1186 		mtx_unlock(&ioat->submit_lock);
1187 
1188 		newring = ioat_prealloc_ring(ioat, 1 << (order - 1), FALSE,
1189 		    M_NOWAIT);
1190 
1191 		mtx_lock(&ioat->submit_lock);
1192 		KASSERT(ioat->ring_size_order == order,
1193 		    ("resize_pending protects order"));
1194 
1195 		if (newring != NULL)
1196 			ring_shrink(ioat, order, newring);
1197 
1198 		ioat->is_resize_pending = FALSE;
1199 		mtx_unlock(&ioat->submit_lock);
1200 
1201 out:
1202 		/* Slowly scale the ring down if idle. */
1203 		if (ioat->ring_size_order > IOAT_MIN_ORDER)
1204 			callout_reset(&ioat->timer, 10 * hz,
1205 			    ioat_timer_callback, ioat);
1206 	}
1207 }
1208 
1209 /*
1210  * Support Functions
1211  */
1212 static void
1213 ioat_submit_single(struct ioat_softc *ioat)
1214 {
1215 
1216 	ioat_get(ioat, IOAT_ACTIVE_DESCR_REF);
1217 	atomic_add_rel_int(&ioat->head, 1);
1218 	atomic_add_rel_int(&ioat->hw_head, 1);
1219 
1220 	if (!ioat->is_completion_pending) {
1221 		ioat->is_completion_pending = TRUE;
1222 		callout_reset(&ioat->timer, IOAT_INTR_TIMO,
1223 		    ioat_timer_callback, ioat);
1224 	}
1225 }
1226 
1227 static int
1228 ioat_reset_hw(struct ioat_softc *ioat)
1229 {
1230 	uint64_t status;
1231 	uint32_t chanerr;
1232 	unsigned timeout;
1233 
1234 	status = ioat_get_chansts(ioat);
1235 	if (is_ioat_active(status) || is_ioat_idle(status))
1236 		ioat_suspend(ioat);
1237 
1238 	/* Wait at most 20 ms */
1239 	for (timeout = 0; (is_ioat_active(status) || is_ioat_idle(status)) &&
1240 	    timeout < 20; timeout++) {
1241 		DELAY(1000);
1242 		status = ioat_get_chansts(ioat);
1243 	}
1244 	if (timeout == 20)
1245 		return (ETIMEDOUT);
1246 
1247 	KASSERT(ioat_get_active(ioat) == 0, ("active after quiesce"));
1248 
1249 	chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
1250 	ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr);
1251 
1252 	/*
1253 	 * IOAT v3 workaround - CHANERRMSK_INT with 3E07h to masks out errors
1254 	 *  that can cause stability issues for IOAT v3.
1255 	 */
1256 	pci_write_config(ioat->device, IOAT_CFG_CHANERRMASK_INT_OFFSET, 0x3e07,
1257 	    4);
1258 	chanerr = pci_read_config(ioat->device, IOAT_CFG_CHANERR_INT_OFFSET, 4);
1259 	pci_write_config(ioat->device, IOAT_CFG_CHANERR_INT_OFFSET, chanerr, 4);
1260 
1261 	/*
1262 	 * BDXDE and BWD models reset MSI-X registers on device reset.
1263 	 * Save/restore their contents manually.
1264 	 */
1265 	if (ioat_model_resets_msix(ioat)) {
1266 		ioat_log_message(1, "device resets MSI-X registers; saving\n");
1267 		pci_save_state(ioat->device);
1268 	}
1269 
1270 	ioat_reset(ioat);
1271 
1272 	/* Wait at most 20 ms */
1273 	for (timeout = 0; ioat_reset_pending(ioat) && timeout < 20; timeout++)
1274 		DELAY(1000);
1275 	if (timeout == 20)
1276 		return (ETIMEDOUT);
1277 
1278 	if (ioat_model_resets_msix(ioat)) {
1279 		ioat_log_message(1, "device resets registers; restored\n");
1280 		pci_restore_state(ioat->device);
1281 	}
1282 
1283 	/* Reset attempts to return the hardware to "halted." */
1284 	status = ioat_get_chansts(ioat);
1285 	if (is_ioat_active(status) || is_ioat_idle(status)) {
1286 		/* So this really shouldn't happen... */
1287 		ioat_log_message(0, "Device is active after a reset?\n");
1288 		ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
1289 		return (0);
1290 	}
1291 
1292 	chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
1293 	ioat_halted_debug(ioat, chanerr);
1294 	if (chanerr != 0)
1295 		return (EIO);
1296 
1297 	/*
1298 	 * Bring device back online after reset.  Writing CHAINADDR brings the
1299 	 * device back to active.
1300 	 *
1301 	 * The internal ring counter resets to zero, so we have to start over
1302 	 * at zero as well.
1303 	 */
1304 	ioat->tail = ioat->head = ioat->hw_head = 0;
1305 	ioat->last_seen = 0;
1306 
1307 	ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
1308 	ioat_write_chancmp(ioat, ioat->comp_update_bus_addr);
1309 	ioat_write_chainaddr(ioat, ioat->ring[0]->hw_desc_bus_addr);
1310 	return (ioat_start_channel(ioat));
1311 }
1312 
1313 static int
1314 sysctl_handle_reset(SYSCTL_HANDLER_ARGS)
1315 {
1316 	struct ioat_softc *ioat;
1317 	int error, arg;
1318 
1319 	ioat = arg1;
1320 
1321 	arg = 0;
1322 	error = SYSCTL_OUT(req, &arg, sizeof(arg));
1323 	if (error != 0 || req->newptr == NULL)
1324 		return (error);
1325 
1326 	error = SYSCTL_IN(req, &arg, sizeof(arg));
1327 	if (error != 0)
1328 		return (error);
1329 
1330 	if (arg != 0)
1331 		error = ioat_reset_hw(ioat);
1332 
1333 	return (error);
1334 }
1335 
1336 static void
1337 dump_descriptor(void *hw_desc)
1338 {
1339 	int i, j;
1340 
1341 	for (i = 0; i < 2; i++) {
1342 		for (j = 0; j < 8; j++)
1343 			printf("%08x ", ((uint32_t *)hw_desc)[i * 8 + j]);
1344 		printf("\n");
1345 	}
1346 }
1347 
1348 static void
1349 ioat_setup_sysctl(device_t device)
1350 {
1351 	struct sysctl_oid_list *par;
1352 	struct sysctl_ctx_list *ctx;
1353 	struct sysctl_oid *tree;
1354 	struct ioat_softc *ioat;
1355 
1356 	ioat = DEVICE2SOFTC(device);
1357 	ctx = device_get_sysctl_ctx(device);
1358 	tree = device_get_sysctl_tree(device);
1359 	par = SYSCTL_CHILDREN(tree);
1360 
1361 	SYSCTL_ADD_INT(ctx, par, OID_AUTO, "version", CTLFLAG_RD,
1362 	    &ioat->version, 0, "HW version (0xMM form)");
1363 	SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "max_xfer_size", CTLFLAG_RD,
1364 	    &ioat->max_xfer_size, 0, "HW maximum transfer size");
1365 
1366 	SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "ring_size_order", CTLFLAG_RD,
1367 	    &ioat->ring_size_order, 0, "SW descriptor ring size order");
1368 	SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "head", CTLFLAG_RD, &ioat->head, 0,
1369 	    "SW descriptor head pointer index");
1370 	SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "tail", CTLFLAG_RD, &ioat->tail, 0,
1371 	    "SW descriptor tail pointer index");
1372 	SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "hw_head", CTLFLAG_RD,
1373 	    &ioat->hw_head, 0, "HW DMACOUNT");
1374 
1375 	SYSCTL_ADD_UQUAD(ctx, par, OID_AUTO, "last_completion", CTLFLAG_RD,
1376 	    ioat->comp_update, "HW addr of last completion");
1377 
1378 	SYSCTL_ADD_INT(ctx, par, OID_AUTO, "is_resize_pending", CTLFLAG_RD,
1379 	    &ioat->is_resize_pending, 0, "resize pending");
1380 	SYSCTL_ADD_INT(ctx, par, OID_AUTO, "is_completion_pending", CTLFLAG_RD,
1381 	    &ioat->is_completion_pending, 0, "completion pending");
1382 	SYSCTL_ADD_INT(ctx, par, OID_AUTO, "is_reset_pending", CTLFLAG_RD,
1383 	    &ioat->is_reset_pending, 0, "reset pending");
1384 	SYSCTL_ADD_INT(ctx, par, OID_AUTO, "is_channel_running", CTLFLAG_RD,
1385 	    &ioat->is_channel_running, 0, "channel running");
1386 
1387 	SYSCTL_ADD_PROC(ctx, par, OID_AUTO, "force_hw_reset",
1388 	    CTLTYPE_INT | CTLFLAG_RW, ioat, 0, sysctl_handle_reset, "I",
1389 	    "Set to non-zero to reset the hardware");
1390 }
1391 
1392 static inline struct ioat_softc *
1393 ioat_get(struct ioat_softc *ioat, enum ioat_ref_kind kind)
1394 {
1395 	uint32_t old;
1396 
1397 	KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus"));
1398 
1399 	old = atomic_fetchadd_32(&ioat->refcnt, 1);
1400 	KASSERT(old < UINT32_MAX, ("refcnt overflow"));
1401 
1402 #ifdef INVARIANTS
1403 	old = atomic_fetchadd_32(&ioat->refkinds[kind], 1);
1404 	KASSERT(old < UINT32_MAX, ("refcnt kind overflow"));
1405 #endif
1406 
1407 	return (ioat);
1408 }
1409 
1410 static inline void
1411 ioat_putn(struct ioat_softc *ioat, uint32_t n, enum ioat_ref_kind kind)
1412 {
1413 	uint32_t old;
1414 
1415 	KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus"));
1416 
1417 	if (n == 0)
1418 		return;
1419 
1420 #ifdef INVARIANTS
1421 	old = atomic_fetchadd_32(&ioat->refkinds[kind], -n);
1422 	KASSERT(old >= n, ("refcnt kind underflow"));
1423 #endif
1424 
1425 	/* Skip acquiring the lock if resulting refcnt > 0. */
1426 	for (;;) {
1427 		old = ioat->refcnt;
1428 		if (old <= n)
1429 			break;
1430 		if (atomic_cmpset_32(&ioat->refcnt, old, old - n))
1431 			return;
1432 	}
1433 
1434 	mtx_lock(IOAT_REFLK);
1435 	old = atomic_fetchadd_32(&ioat->refcnt, -n);
1436 	KASSERT(old >= n, ("refcnt error"));
1437 
1438 	if (old == n)
1439 		wakeup(IOAT_REFLK);
1440 	mtx_unlock(IOAT_REFLK);
1441 }
1442 
1443 static inline void
1444 ioat_put(struct ioat_softc *ioat, enum ioat_ref_kind kind)
1445 {
1446 
1447 	ioat_putn(ioat, 1, kind);
1448 }
1449 
1450 static void
1451 ioat_drain(struct ioat_softc *ioat)
1452 {
1453 
1454 	mtx_lock(IOAT_REFLK);
1455 	while (ioat->refcnt > 0)
1456 		msleep(IOAT_REFLK, IOAT_REFLK, 0, "ioat_drain", 0);
1457 	mtx_unlock(IOAT_REFLK);
1458 }
1459