xref: /freebsd/sys/dev/ioat/ioat.c (revision e88e14b9f01d29616ccff4d3bbd84c71657d56f5)
1e974f91cSConrad Meyer /*-
2e974f91cSConrad Meyer  * Copyright (C) 2012 Intel Corporation
3e974f91cSConrad Meyer  * All rights reserved.
4e974f91cSConrad Meyer  *
5e974f91cSConrad Meyer  * Redistribution and use in source and binary forms, with or without
6e974f91cSConrad Meyer  * modification, are permitted provided that the following conditions
7e974f91cSConrad Meyer  * are met:
8e974f91cSConrad Meyer  * 1. Redistributions of source code must retain the above copyright
9e974f91cSConrad Meyer  *    notice, this list of conditions and the following disclaimer.
10e974f91cSConrad Meyer  * 2. Redistributions in binary form must reproduce the above copyright
11e974f91cSConrad Meyer  *    notice, this list of conditions and the following disclaimer in the
12e974f91cSConrad Meyer  *    documentation and/or other materials provided with the distribution.
13e974f91cSConrad Meyer  *
14e974f91cSConrad Meyer  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15e974f91cSConrad Meyer  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16e974f91cSConrad Meyer  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17e974f91cSConrad Meyer  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18e974f91cSConrad Meyer  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19e974f91cSConrad Meyer  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20e974f91cSConrad Meyer  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21e974f91cSConrad Meyer  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22e974f91cSConrad Meyer  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23e974f91cSConrad Meyer  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24e974f91cSConrad Meyer  * SUCH DAMAGE.
25e974f91cSConrad Meyer  */
26e974f91cSConrad Meyer 
27e974f91cSConrad Meyer #include <sys/cdefs.h>
28e974f91cSConrad Meyer __FBSDID("$FreeBSD$");
29e974f91cSConrad Meyer 
30e974f91cSConrad Meyer #include <sys/param.h>
31e974f91cSConrad Meyer #include <sys/systm.h>
32e974f91cSConrad Meyer #include <sys/bus.h>
33e974f91cSConrad Meyer #include <sys/conf.h>
34e974f91cSConrad Meyer #include <sys/ioccom.h>
35e974f91cSConrad Meyer #include <sys/kernel.h>
36e974f91cSConrad Meyer #include <sys/lock.h>
37e974f91cSConrad Meyer #include <sys/malloc.h>
38e974f91cSConrad Meyer #include <sys/module.h>
39e974f91cSConrad Meyer #include <sys/mutex.h>
40e974f91cSConrad Meyer #include <sys/rman.h>
41e974f91cSConrad Meyer #include <sys/sysctl.h>
42e974f91cSConrad Meyer #include <sys/time.h>
43e974f91cSConrad Meyer #include <dev/pci/pcireg.h>
44e974f91cSConrad Meyer #include <dev/pci/pcivar.h>
45e974f91cSConrad Meyer #include <machine/bus.h>
46e974f91cSConrad Meyer #include <machine/resource.h>
47e974f91cSConrad Meyer #include <machine/stdarg.h>
48e974f91cSConrad Meyer 
49e974f91cSConrad Meyer #include "ioat.h"
50e974f91cSConrad Meyer #include "ioat_hw.h"
51e974f91cSConrad Meyer #include "ioat_internal.h"
52e974f91cSConrad Meyer 
53fe720f5aSConrad Meyer #define	IOAT_INTR_TIMO	(hz / 10)
54466b3540SConrad Meyer #define	IOAT_REFLK	(&ioat->submit_lock)
55fe720f5aSConrad Meyer 
56e974f91cSConrad Meyer static int ioat_probe(device_t device);
57e974f91cSConrad Meyer static int ioat_attach(device_t device);
58e974f91cSConrad Meyer static int ioat_detach(device_t device);
594253ea50SConrad Meyer static int ioat_setup_intr(struct ioat_softc *ioat);
604253ea50SConrad Meyer static int ioat_teardown_intr(struct ioat_softc *ioat);
61e974f91cSConrad Meyer static int ioat3_attach(device_t device);
627afbb263SConrad Meyer static int ioat3_selftest(struct ioat_softc *ioat);
63e974f91cSConrad Meyer static int ioat_map_pci_bar(struct ioat_softc *ioat);
64e974f91cSConrad Meyer static void ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg,
65e974f91cSConrad Meyer     int error);
66e974f91cSConrad Meyer static void ioat_interrupt_handler(void *arg);
670d1a05d9SConrad Meyer static boolean_t ioat_model_resets_msix(struct ioat_softc *ioat);
68e974f91cSConrad Meyer static void ioat_process_events(struct ioat_softc *ioat);
69e974f91cSConrad Meyer static inline uint32_t ioat_get_active(struct ioat_softc *ioat);
70e974f91cSConrad Meyer static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat);
71e974f91cSConrad Meyer static void ioat_free_ring_entry(struct ioat_softc *ioat,
72e974f91cSConrad Meyer     struct ioat_descriptor *desc);
73e974f91cSConrad Meyer static struct ioat_descriptor *ioat_alloc_ring_entry(struct ioat_softc *ioat);
74e974f91cSConrad Meyer static int ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs);
75e974f91cSConrad Meyer static struct ioat_descriptor *ioat_get_ring_entry(struct ioat_softc *ioat,
76e974f91cSConrad Meyer     uint32_t index);
77e974f91cSConrad Meyer static boolean_t resize_ring(struct ioat_softc *ioat, int order);
78e974f91cSConrad Meyer static void ioat_timer_callback(void *arg);
79e974f91cSConrad Meyer static void dump_descriptor(void *hw_desc);
80e974f91cSConrad Meyer static void ioat_submit_single(struct ioat_softc *ioat);
81e974f91cSConrad Meyer static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg,
82e974f91cSConrad Meyer     int error);
83e974f91cSConrad Meyer static int ioat_reset_hw(struct ioat_softc *ioat);
84e974f91cSConrad Meyer static void ioat_setup_sysctl(device_t device);
85f7157235SConrad Meyer static int sysctl_handle_reset(SYSCTL_HANDLER_ARGS);
86466b3540SConrad Meyer static inline struct ioat_softc *ioat_get(struct ioat_softc *,
87466b3540SConrad Meyer     enum ioat_ref_kind);
88466b3540SConrad Meyer static inline void ioat_put(struct ioat_softc *, enum ioat_ref_kind);
89466b3540SConrad Meyer static inline void ioat_putn(struct ioat_softc *, uint32_t,
90466b3540SConrad Meyer     enum ioat_ref_kind);
91466b3540SConrad Meyer static void ioat_drain(struct ioat_softc *);
92e974f91cSConrad Meyer 
931c25420eSConrad Meyer #define	ioat_log_message(v, ...) do {					\
941c25420eSConrad Meyer 	if ((v) <= g_ioat_debug_level) {				\
951c25420eSConrad Meyer 		device_printf(ioat->device, __VA_ARGS__);		\
961c25420eSConrad Meyer 	}								\
971c25420eSConrad Meyer } while (0)
981c25420eSConrad Meyer 
99e974f91cSConrad Meyer MALLOC_DEFINE(M_IOAT, "ioat", "ioat driver memory allocations");
100e974f91cSConrad Meyer SYSCTL_NODE(_hw, OID_AUTO, ioat, CTLFLAG_RD, 0, "ioat node");
101e974f91cSConrad Meyer 
102e974f91cSConrad Meyer static int g_force_legacy_interrupts;
103e974f91cSConrad Meyer SYSCTL_INT(_hw_ioat, OID_AUTO, force_legacy_interrupts, CTLFLAG_RDTUN,
104e974f91cSConrad Meyer     &g_force_legacy_interrupts, 0, "Set to non-zero to force MSI-X disabled");
105e974f91cSConrad Meyer 
1061c25420eSConrad Meyer int g_ioat_debug_level = 0;
107e974f91cSConrad Meyer SYSCTL_INT(_hw_ioat, OID_AUTO, debug_level, CTLFLAG_RWTUN, &g_ioat_debug_level,
108e974f91cSConrad Meyer     0, "Set log level (0-3) for ioat(4). Higher is more verbose.");
109e974f91cSConrad Meyer 
110e974f91cSConrad Meyer /*
111e974f91cSConrad Meyer  * OS <-> Driver interface structures
112e974f91cSConrad Meyer  */
113e974f91cSConrad Meyer static device_method_t ioat_pci_methods[] = {
114e974f91cSConrad Meyer 	/* Device interface */
115e974f91cSConrad Meyer 	DEVMETHOD(device_probe,     ioat_probe),
116e974f91cSConrad Meyer 	DEVMETHOD(device_attach,    ioat_attach),
117e974f91cSConrad Meyer 	DEVMETHOD(device_detach,    ioat_detach),
118e974f91cSConrad Meyer 	{ 0, 0 }
119e974f91cSConrad Meyer };
120e974f91cSConrad Meyer 
121e974f91cSConrad Meyer static driver_t ioat_pci_driver = {
122e974f91cSConrad Meyer 	"ioat",
123e974f91cSConrad Meyer 	ioat_pci_methods,
124e974f91cSConrad Meyer 	sizeof(struct ioat_softc),
125e974f91cSConrad Meyer };
126e974f91cSConrad Meyer 
127e974f91cSConrad Meyer static devclass_t ioat_devclass;
128e974f91cSConrad Meyer DRIVER_MODULE(ioat, pci, ioat_pci_driver, ioat_devclass, 0, 0);
129e974f91cSConrad Meyer 
130e974f91cSConrad Meyer /*
131e974f91cSConrad Meyer  * Private data structures
132e974f91cSConrad Meyer  */
133e974f91cSConrad Meyer static struct ioat_softc *ioat_channel[IOAT_MAX_CHANNELS];
134e974f91cSConrad Meyer static int ioat_channel_index = 0;
135e974f91cSConrad Meyer SYSCTL_INT(_hw_ioat, OID_AUTO, channels, CTLFLAG_RD, &ioat_channel_index, 0,
136e974f91cSConrad Meyer     "Number of IOAT channels attached");
137e974f91cSConrad Meyer 
138e974f91cSConrad Meyer static struct _pcsid
139e974f91cSConrad Meyer {
140e974f91cSConrad Meyer 	u_int32_t   type;
141e974f91cSConrad Meyer 	const char  *desc;
142e974f91cSConrad Meyer } pci_ids[] = {
143e974f91cSConrad Meyer 	{ 0x34308086, "TBG IOAT Ch0" },
144e974f91cSConrad Meyer 	{ 0x34318086, "TBG IOAT Ch1" },
145e974f91cSConrad Meyer 	{ 0x34328086, "TBG IOAT Ch2" },
146e974f91cSConrad Meyer 	{ 0x34338086, "TBG IOAT Ch3" },
147e974f91cSConrad Meyer 	{ 0x34298086, "TBG IOAT Ch4" },
148e974f91cSConrad Meyer 	{ 0x342a8086, "TBG IOAT Ch5" },
149e974f91cSConrad Meyer 	{ 0x342b8086, "TBG IOAT Ch6" },
150e974f91cSConrad Meyer 	{ 0x342c8086, "TBG IOAT Ch7" },
151e974f91cSConrad Meyer 
152e974f91cSConrad Meyer 	{ 0x37108086, "JSF IOAT Ch0" },
153e974f91cSConrad Meyer 	{ 0x37118086, "JSF IOAT Ch1" },
154e974f91cSConrad Meyer 	{ 0x37128086, "JSF IOAT Ch2" },
155e974f91cSConrad Meyer 	{ 0x37138086, "JSF IOAT Ch3" },
156e974f91cSConrad Meyer 	{ 0x37148086, "JSF IOAT Ch4" },
157e974f91cSConrad Meyer 	{ 0x37158086, "JSF IOAT Ch5" },
158e974f91cSConrad Meyer 	{ 0x37168086, "JSF IOAT Ch6" },
159e974f91cSConrad Meyer 	{ 0x37178086, "JSF IOAT Ch7" },
160e974f91cSConrad Meyer 	{ 0x37188086, "JSF IOAT Ch0 (RAID)" },
161e974f91cSConrad Meyer 	{ 0x37198086, "JSF IOAT Ch1 (RAID)" },
162e974f91cSConrad Meyer 
163e974f91cSConrad Meyer 	{ 0x3c208086, "SNB IOAT Ch0" },
164e974f91cSConrad Meyer 	{ 0x3c218086, "SNB IOAT Ch1" },
165e974f91cSConrad Meyer 	{ 0x3c228086, "SNB IOAT Ch2" },
166e974f91cSConrad Meyer 	{ 0x3c238086, "SNB IOAT Ch3" },
167e974f91cSConrad Meyer 	{ 0x3c248086, "SNB IOAT Ch4" },
168e974f91cSConrad Meyer 	{ 0x3c258086, "SNB IOAT Ch5" },
169e974f91cSConrad Meyer 	{ 0x3c268086, "SNB IOAT Ch6" },
170e974f91cSConrad Meyer 	{ 0x3c278086, "SNB IOAT Ch7" },
171e974f91cSConrad Meyer 	{ 0x3c2e8086, "SNB IOAT Ch0 (RAID)" },
172e974f91cSConrad Meyer 	{ 0x3c2f8086, "SNB IOAT Ch1 (RAID)" },
173e974f91cSConrad Meyer 
174e974f91cSConrad Meyer 	{ 0x0e208086, "IVB IOAT Ch0" },
175e974f91cSConrad Meyer 	{ 0x0e218086, "IVB IOAT Ch1" },
176e974f91cSConrad Meyer 	{ 0x0e228086, "IVB IOAT Ch2" },
177e974f91cSConrad Meyer 	{ 0x0e238086, "IVB IOAT Ch3" },
178e974f91cSConrad Meyer 	{ 0x0e248086, "IVB IOAT Ch4" },
179e974f91cSConrad Meyer 	{ 0x0e258086, "IVB IOAT Ch5" },
180e974f91cSConrad Meyer 	{ 0x0e268086, "IVB IOAT Ch6" },
181e974f91cSConrad Meyer 	{ 0x0e278086, "IVB IOAT Ch7" },
182e974f91cSConrad Meyer 	{ 0x0e2e8086, "IVB IOAT Ch0 (RAID)" },
183e974f91cSConrad Meyer 	{ 0x0e2f8086, "IVB IOAT Ch1 (RAID)" },
184e974f91cSConrad Meyer 
185e974f91cSConrad Meyer 	{ 0x2f208086, "HSW IOAT Ch0" },
186e974f91cSConrad Meyer 	{ 0x2f218086, "HSW IOAT Ch1" },
187e974f91cSConrad Meyer 	{ 0x2f228086, "HSW IOAT Ch2" },
188e974f91cSConrad Meyer 	{ 0x2f238086, "HSW IOAT Ch3" },
189e974f91cSConrad Meyer 	{ 0x2f248086, "HSW IOAT Ch4" },
190e974f91cSConrad Meyer 	{ 0x2f258086, "HSW IOAT Ch5" },
191e974f91cSConrad Meyer 	{ 0x2f268086, "HSW IOAT Ch6" },
192e974f91cSConrad Meyer 	{ 0x2f278086, "HSW IOAT Ch7" },
193e974f91cSConrad Meyer 	{ 0x2f2e8086, "HSW IOAT Ch0 (RAID)" },
194e974f91cSConrad Meyer 	{ 0x2f2f8086, "HSW IOAT Ch1 (RAID)" },
195e974f91cSConrad Meyer 
196e974f91cSConrad Meyer 	{ 0x0c508086, "BWD IOAT Ch0" },
197e974f91cSConrad Meyer 	{ 0x0c518086, "BWD IOAT Ch1" },
198e974f91cSConrad Meyer 	{ 0x0c528086, "BWD IOAT Ch2" },
199e974f91cSConrad Meyer 	{ 0x0c538086, "BWD IOAT Ch3" },
200e974f91cSConrad Meyer 
201e974f91cSConrad Meyer 	{ 0x6f508086, "BDXDE IOAT Ch0" },
202e974f91cSConrad Meyer 	{ 0x6f518086, "BDXDE IOAT Ch1" },
203e974f91cSConrad Meyer 	{ 0x6f528086, "BDXDE IOAT Ch2" },
204e974f91cSConrad Meyer 	{ 0x6f538086, "BDXDE IOAT Ch3" },
205e974f91cSConrad Meyer 
206e974f91cSConrad Meyer 	{ 0x00000000, NULL           }
207e974f91cSConrad Meyer };
208e974f91cSConrad Meyer 
209e974f91cSConrad Meyer /*
210e974f91cSConrad Meyer  * OS <-> Driver linkage functions
211e974f91cSConrad Meyer  */
212e974f91cSConrad Meyer static int
213e974f91cSConrad Meyer ioat_probe(device_t device)
214e974f91cSConrad Meyer {
215e974f91cSConrad Meyer 	struct _pcsid *ep;
216e974f91cSConrad Meyer 	u_int32_t type;
217e974f91cSConrad Meyer 
218e974f91cSConrad Meyer 	type = pci_get_devid(device);
219e974f91cSConrad Meyer 	for (ep = pci_ids; ep->type; ep++) {
220e974f91cSConrad Meyer 		if (ep->type == type) {
221e974f91cSConrad Meyer 			device_set_desc(device, ep->desc);
222e974f91cSConrad Meyer 			return (0);
223e974f91cSConrad Meyer 		}
224e974f91cSConrad Meyer 	}
225e974f91cSConrad Meyer 	return (ENXIO);
226e974f91cSConrad Meyer }
227e974f91cSConrad Meyer 
228e974f91cSConrad Meyer static int
229e974f91cSConrad Meyer ioat_attach(device_t device)
230e974f91cSConrad Meyer {
231e974f91cSConrad Meyer 	struct ioat_softc *ioat;
232e974f91cSConrad Meyer 	int error;
233e974f91cSConrad Meyer 
234e974f91cSConrad Meyer 	ioat = DEVICE2SOFTC(device);
235e974f91cSConrad Meyer 	ioat->device = device;
236e974f91cSConrad Meyer 
237e974f91cSConrad Meyer 	error = ioat_map_pci_bar(ioat);
238e974f91cSConrad Meyer 	if (error != 0)
239e974f91cSConrad Meyer 		goto err;
240e974f91cSConrad Meyer 
241e974f91cSConrad Meyer 	ioat->version = ioat_read_cbver(ioat);
242e974f91cSConrad Meyer 	if (ioat->version < IOAT_VER_3_0) {
243e974f91cSConrad Meyer 		error = ENODEV;
244e974f91cSConrad Meyer 		goto err;
245e974f91cSConrad Meyer 	}
246e974f91cSConrad Meyer 
247e974f91cSConrad Meyer 	error = ioat3_attach(device);
248e974f91cSConrad Meyer 	if (error != 0)
249e974f91cSConrad Meyer 		goto err;
250e974f91cSConrad Meyer 
251e974f91cSConrad Meyer 	error = pci_enable_busmaster(device);
252e974f91cSConrad Meyer 	if (error != 0)
253e974f91cSConrad Meyer 		goto err;
254e974f91cSConrad Meyer 
255466b3540SConrad Meyer 	error = ioat_setup_intr(ioat);
256466b3540SConrad Meyer 	if (error != 0)
257466b3540SConrad Meyer 		goto err;
258466b3540SConrad Meyer 
2597afbb263SConrad Meyer 	error = ioat3_selftest(ioat);
2607afbb263SConrad Meyer 	if (error != 0)
261466b3540SConrad Meyer 		goto err;
2627afbb263SConrad Meyer 
2637afbb263SConrad Meyer 	ioat_process_events(ioat);
2647afbb263SConrad Meyer 	ioat_setup_sysctl(device);
2657afbb263SConrad Meyer 
266e974f91cSConrad Meyer 	ioat_channel[ioat_channel_index++] = ioat;
2677afbb263SConrad Meyer 	ioat_test_attach();
268e974f91cSConrad Meyer 
269e974f91cSConrad Meyer err:
270e974f91cSConrad Meyer 	if (error != 0)
271e974f91cSConrad Meyer 		ioat_detach(device);
272e974f91cSConrad Meyer 	return (error);
273e974f91cSConrad Meyer }
274e974f91cSConrad Meyer 
275e974f91cSConrad Meyer static int
276e974f91cSConrad Meyer ioat_detach(device_t device)
277e974f91cSConrad Meyer {
278e974f91cSConrad Meyer 	struct ioat_softc *ioat;
279e974f91cSConrad Meyer 	uint32_t i;
280e974f91cSConrad Meyer 
281e974f91cSConrad Meyer 	ioat = DEVICE2SOFTC(device);
2827afbb263SConrad Meyer 
2837afbb263SConrad Meyer 	ioat_test_detach();
284466b3540SConrad Meyer 	ioat_drain(ioat);
285fe720f5aSConrad Meyer 
286fe720f5aSConrad Meyer 	ioat_teardown_intr(ioat);
287e974f91cSConrad Meyer 	callout_drain(&ioat->timer);
288e974f91cSConrad Meyer 
289e974f91cSConrad Meyer 	pci_disable_busmaster(device);
290e974f91cSConrad Meyer 
291e974f91cSConrad Meyer 	if (ioat->pci_resource != NULL)
292e974f91cSConrad Meyer 		bus_release_resource(device, SYS_RES_MEMORY,
293e974f91cSConrad Meyer 		    ioat->pci_resource_id, ioat->pci_resource);
294e974f91cSConrad Meyer 
295e974f91cSConrad Meyer 	if (ioat->ring != NULL) {
296e974f91cSConrad Meyer 		for (i = 0; i < (1 << ioat->ring_size_order); i++)
297e974f91cSConrad Meyer 			ioat_free_ring_entry(ioat, ioat->ring[i]);
298e974f91cSConrad Meyer 		free(ioat->ring, M_IOAT);
299e974f91cSConrad Meyer 	}
300e974f91cSConrad Meyer 
301e974f91cSConrad Meyer 	if (ioat->comp_update != NULL) {
302e974f91cSConrad Meyer 		bus_dmamap_unload(ioat->comp_update_tag, ioat->comp_update_map);
303e974f91cSConrad Meyer 		bus_dmamem_free(ioat->comp_update_tag, ioat->comp_update,
304e974f91cSConrad Meyer 		    ioat->comp_update_map);
305e974f91cSConrad Meyer 		bus_dma_tag_destroy(ioat->comp_update_tag);
306e974f91cSConrad Meyer 	}
307e974f91cSConrad Meyer 
308e974f91cSConrad Meyer 	bus_dma_tag_destroy(ioat->hw_desc_tag);
309e974f91cSConrad Meyer 
3104253ea50SConrad Meyer 	return (0);
3114253ea50SConrad Meyer }
3124253ea50SConrad Meyer 
3134253ea50SConrad Meyer static int
3144253ea50SConrad Meyer ioat_teardown_intr(struct ioat_softc *ioat)
3154253ea50SConrad Meyer {
3164253ea50SConrad Meyer 
317e974f91cSConrad Meyer 	if (ioat->tag != NULL)
3184253ea50SConrad Meyer 		bus_teardown_intr(ioat->device, ioat->res, ioat->tag);
319e974f91cSConrad Meyer 
320e974f91cSConrad Meyer 	if (ioat->res != NULL)
3214253ea50SConrad Meyer 		bus_release_resource(ioat->device, SYS_RES_IRQ,
322e974f91cSConrad Meyer 		    rman_get_rid(ioat->res), ioat->res);
323e974f91cSConrad Meyer 
3244253ea50SConrad Meyer 	pci_release_msi(ioat->device);
325e974f91cSConrad Meyer 	return (0);
326e974f91cSConrad Meyer }
327e974f91cSConrad Meyer 
328e974f91cSConrad Meyer static int
329e974f91cSConrad Meyer ioat3_selftest(struct ioat_softc *ioat)
330e974f91cSConrad Meyer {
331e974f91cSConrad Meyer 	uint64_t status;
332e974f91cSConrad Meyer 	uint32_t chanerr;
333e974f91cSConrad Meyer 	int i;
334e974f91cSConrad Meyer 
335e974f91cSConrad Meyer 	ioat_acquire(&ioat->dmaengine);
336e974f91cSConrad Meyer 	ioat_null(&ioat->dmaengine, NULL, NULL, 0);
337e974f91cSConrad Meyer 	ioat_release(&ioat->dmaengine);
338e974f91cSConrad Meyer 
339e974f91cSConrad Meyer 	for (i = 0; i < 100; i++) {
340e974f91cSConrad Meyer 		DELAY(1);
341e974f91cSConrad Meyer 		status = ioat_get_chansts(ioat);
342e974f91cSConrad Meyer 		if (is_ioat_idle(status))
343e974f91cSConrad Meyer 			return (0);
344e974f91cSConrad Meyer 	}
345e974f91cSConrad Meyer 
346e974f91cSConrad Meyer 	chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
347e974f91cSConrad Meyer 	ioat_log_message(0, "could not start channel: "
348e974f91cSConrad Meyer 	    "status = %#jx error = %x\n", (uintmax_t)status, chanerr);
349e974f91cSConrad Meyer 	return (ENXIO);
350e974f91cSConrad Meyer }
351e974f91cSConrad Meyer 
352e974f91cSConrad Meyer /*
353e974f91cSConrad Meyer  * Initialize Hardware
354e974f91cSConrad Meyer  */
355e974f91cSConrad Meyer static int
356e974f91cSConrad Meyer ioat3_attach(device_t device)
357e974f91cSConrad Meyer {
358e974f91cSConrad Meyer 	struct ioat_softc *ioat;
359e974f91cSConrad Meyer 	struct ioat_descriptor **ring;
360e974f91cSConrad Meyer 	struct ioat_descriptor *next;
361e974f91cSConrad Meyer 	struct ioat_dma_hw_descriptor *dma_hw_desc;
362e974f91cSConrad Meyer 	uint32_t capabilities;
363e974f91cSConrad Meyer 	int i, num_descriptors;
364e974f91cSConrad Meyer 	int error;
365e974f91cSConrad Meyer 	uint8_t xfercap;
366e974f91cSConrad Meyer 
367e974f91cSConrad Meyer 	error = 0;
368e974f91cSConrad Meyer 	ioat = DEVICE2SOFTC(device);
369e974f91cSConrad Meyer 	capabilities = ioat_read_dmacapability(ioat);
370e974f91cSConrad Meyer 
371e974f91cSConrad Meyer 	xfercap = ioat_read_xfercap(ioat);
372e974f91cSConrad Meyer 	ioat->max_xfer_size = 1 << xfercap;
373e974f91cSConrad Meyer 
374e974f91cSConrad Meyer 	/* TODO: need to check DCA here if we ever do XOR/PQ */
375e974f91cSConrad Meyer 
376e974f91cSConrad Meyer 	mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF);
377e974f91cSConrad Meyer 	mtx_init(&ioat->cleanup_lock, "ioat_process_events", NULL, MTX_DEF);
3787afbb263SConrad Meyer 	callout_init(&ioat->timer, 1);
379e974f91cSConrad Meyer 
380e974f91cSConrad Meyer 	ioat->is_resize_pending = FALSE;
381e974f91cSConrad Meyer 	ioat->is_completion_pending = FALSE;
382e974f91cSConrad Meyer 	ioat->is_reset_pending = FALSE;
383e974f91cSConrad Meyer 	ioat->is_channel_running = FALSE;
384e974f91cSConrad Meyer 	ioat->is_waiting_for_ack = FALSE;
385e974f91cSConrad Meyer 
386e974f91cSConrad Meyer 	bus_dma_tag_create(bus_get_dma_tag(ioat->device), sizeof(uint64_t), 0x0,
387e974f91cSConrad Meyer 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
388e974f91cSConrad Meyer 	    sizeof(uint64_t), 1, sizeof(uint64_t), 0, NULL, NULL,
389e974f91cSConrad Meyer 	    &ioat->comp_update_tag);
390e974f91cSConrad Meyer 
391e974f91cSConrad Meyer 	error = bus_dmamem_alloc(ioat->comp_update_tag,
392e974f91cSConrad Meyer 	    (void **)&ioat->comp_update, BUS_DMA_ZERO, &ioat->comp_update_map);
393e974f91cSConrad Meyer 	if (ioat->comp_update == NULL)
394e974f91cSConrad Meyer 		return (ENOMEM);
395e974f91cSConrad Meyer 
396e974f91cSConrad Meyer 	error = bus_dmamap_load(ioat->comp_update_tag, ioat->comp_update_map,
397e974f91cSConrad Meyer 	    ioat->comp_update, sizeof(uint64_t), ioat_comp_update_map, ioat,
398e974f91cSConrad Meyer 	    0);
399e974f91cSConrad Meyer 	if (error != 0)
400e974f91cSConrad Meyer 		return (error);
401e974f91cSConrad Meyer 
402e974f91cSConrad Meyer 	ioat->ring_size_order = IOAT_MIN_ORDER;
403e974f91cSConrad Meyer 
404e974f91cSConrad Meyer 	num_descriptors = 1 << ioat->ring_size_order;
405e974f91cSConrad Meyer 
406e974f91cSConrad Meyer 	bus_dma_tag_create(bus_get_dma_tag(ioat->device), 0x40, 0x0,
407e974f91cSConrad Meyer 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
408e974f91cSConrad Meyer 	    sizeof(struct ioat_dma_hw_descriptor), 1,
409e974f91cSConrad Meyer 	    sizeof(struct ioat_dma_hw_descriptor), 0, NULL, NULL,
410e974f91cSConrad Meyer 	    &ioat->hw_desc_tag);
411e974f91cSConrad Meyer 
412e974f91cSConrad Meyer 	ioat->ring = malloc(num_descriptors * sizeof(*ring), M_IOAT,
413e974f91cSConrad Meyer 	    M_ZERO | M_NOWAIT);
414e974f91cSConrad Meyer 	if (ioat->ring == NULL)
415e974f91cSConrad Meyer 		return (ENOMEM);
416e974f91cSConrad Meyer 
417e974f91cSConrad Meyer 	ring = ioat->ring;
418e974f91cSConrad Meyer 	for (i = 0; i < num_descriptors; i++) {
419e974f91cSConrad Meyer 		ring[i] = ioat_alloc_ring_entry(ioat);
420e974f91cSConrad Meyer 		if (ring[i] == NULL)
421e974f91cSConrad Meyer 			return (ENOMEM);
422e974f91cSConrad Meyer 
423e974f91cSConrad Meyer 		ring[i]->id = i;
424e974f91cSConrad Meyer 	}
425e974f91cSConrad Meyer 
426e974f91cSConrad Meyer 	for (i = 0; i < num_descriptors - 1; i++) {
427e974f91cSConrad Meyer 		next = ring[i + 1];
428e974f91cSConrad Meyer 		dma_hw_desc = ring[i]->u.dma;
429e974f91cSConrad Meyer 
430e974f91cSConrad Meyer 		dma_hw_desc->next = next->hw_desc_bus_addr;
431e974f91cSConrad Meyer 	}
432e974f91cSConrad Meyer 
433e974f91cSConrad Meyer 	ring[i]->u.dma->next = ring[0]->hw_desc_bus_addr;
434e974f91cSConrad Meyer 
435e974f91cSConrad Meyer 	ioat->head = 0;
436e974f91cSConrad Meyer 	ioat->tail = 0;
437e974f91cSConrad Meyer 	ioat->last_seen = 0;
438e974f91cSConrad Meyer 
439e974f91cSConrad Meyer 	error = ioat_reset_hw(ioat);
440e974f91cSConrad Meyer 	if (error != 0)
441e974f91cSConrad Meyer 		return (error);
442e974f91cSConrad Meyer 
443e974f91cSConrad Meyer 	ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
444e974f91cSConrad Meyer 	ioat_write_chancmp(ioat, ioat->comp_update_bus_addr);
445e974f91cSConrad Meyer 	ioat_write_chainaddr(ioat, ring[0]->hw_desc_bus_addr);
446e974f91cSConrad Meyer 	return (0);
447e974f91cSConrad Meyer }
448e974f91cSConrad Meyer 
449e974f91cSConrad Meyer static int
450e974f91cSConrad Meyer ioat_map_pci_bar(struct ioat_softc *ioat)
451e974f91cSConrad Meyer {
452e974f91cSConrad Meyer 
453e974f91cSConrad Meyer 	ioat->pci_resource_id = PCIR_BAR(0);
454*e88e14b9SConrad Meyer 	ioat->pci_resource = bus_alloc_resource_any(ioat->device,
455*e88e14b9SConrad Meyer 	    SYS_RES_MEMORY, &ioat->pci_resource_id, RF_ACTIVE);
456e974f91cSConrad Meyer 
457e974f91cSConrad Meyer 	if (ioat->pci_resource == NULL) {
458e974f91cSConrad Meyer 		ioat_log_message(0, "unable to allocate pci resource\n");
459e974f91cSConrad Meyer 		return (ENODEV);
460e974f91cSConrad Meyer 	}
461e974f91cSConrad Meyer 
462e974f91cSConrad Meyer 	ioat->pci_bus_tag = rman_get_bustag(ioat->pci_resource);
463e974f91cSConrad Meyer 	ioat->pci_bus_handle = rman_get_bushandle(ioat->pci_resource);
464e974f91cSConrad Meyer 	return (0);
465e974f91cSConrad Meyer }
466e974f91cSConrad Meyer 
467e974f91cSConrad Meyer static void
468e974f91cSConrad Meyer ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
469e974f91cSConrad Meyer {
470e974f91cSConrad Meyer 	struct ioat_softc *ioat = arg;
471e974f91cSConrad Meyer 
472e974f91cSConrad Meyer 	ioat->comp_update_bus_addr = seg[0].ds_addr;
473e974f91cSConrad Meyer }
474e974f91cSConrad Meyer 
475e974f91cSConrad Meyer static void
476e974f91cSConrad Meyer ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
477e974f91cSConrad Meyer {
478e974f91cSConrad Meyer 	bus_addr_t *baddr;
479e974f91cSConrad Meyer 
480e974f91cSConrad Meyer 	baddr = arg;
481e974f91cSConrad Meyer 	*baddr = segs->ds_addr;
482e974f91cSConrad Meyer }
483e974f91cSConrad Meyer 
484e974f91cSConrad Meyer /*
485e974f91cSConrad Meyer  * Interrupt setup and handlers
486e974f91cSConrad Meyer  */
487e974f91cSConrad Meyer static int
4884253ea50SConrad Meyer ioat_setup_intr(struct ioat_softc *ioat)
489e974f91cSConrad Meyer {
490e974f91cSConrad Meyer 	uint32_t num_vectors;
491e974f91cSConrad Meyer 	int error;
492e974f91cSConrad Meyer 	boolean_t use_msix;
493e974f91cSConrad Meyer 	boolean_t force_legacy_interrupts;
494e974f91cSConrad Meyer 
495e974f91cSConrad Meyer 	use_msix = FALSE;
496e974f91cSConrad Meyer 	force_legacy_interrupts = FALSE;
497e974f91cSConrad Meyer 
498e974f91cSConrad Meyer 	if (!g_force_legacy_interrupts && pci_msix_count(ioat->device) >= 1) {
499e974f91cSConrad Meyer 		num_vectors = 1;
500e974f91cSConrad Meyer 		pci_alloc_msix(ioat->device, &num_vectors);
501e974f91cSConrad Meyer 		if (num_vectors == 1)
502e974f91cSConrad Meyer 			use_msix = TRUE;
503e974f91cSConrad Meyer 	}
504e974f91cSConrad Meyer 
505e974f91cSConrad Meyer 	if (use_msix) {
506e974f91cSConrad Meyer 		ioat->rid = 1;
507e974f91cSConrad Meyer 		ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ,
508e974f91cSConrad Meyer 		    &ioat->rid, RF_ACTIVE);
509e974f91cSConrad Meyer 	} else {
510e974f91cSConrad Meyer 		ioat->rid = 0;
511e974f91cSConrad Meyer 		ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ,
512e974f91cSConrad Meyer 		    &ioat->rid, RF_SHAREABLE | RF_ACTIVE);
513e974f91cSConrad Meyer 	}
514e974f91cSConrad Meyer 	if (ioat->res == NULL) {
515e974f91cSConrad Meyer 		ioat_log_message(0, "bus_alloc_resource failed\n");
516e974f91cSConrad Meyer 		return (ENOMEM);
517e974f91cSConrad Meyer 	}
518e974f91cSConrad Meyer 
519e974f91cSConrad Meyer 	ioat->tag = NULL;
520e974f91cSConrad Meyer 	error = bus_setup_intr(ioat->device, ioat->res, INTR_MPSAFE |
521e974f91cSConrad Meyer 	    INTR_TYPE_MISC, NULL, ioat_interrupt_handler, ioat, &ioat->tag);
522e974f91cSConrad Meyer 	if (error != 0) {
523e974f91cSConrad Meyer 		ioat_log_message(0, "bus_setup_intr failed\n");
524e974f91cSConrad Meyer 		return (error);
525e974f91cSConrad Meyer 	}
526e974f91cSConrad Meyer 
527e974f91cSConrad Meyer 	ioat_write_intrctrl(ioat, IOAT_INTRCTRL_MASTER_INT_EN);
528e974f91cSConrad Meyer 	return (0);
529e974f91cSConrad Meyer }
530e974f91cSConrad Meyer 
5314253ea50SConrad Meyer static boolean_t
5320d1a05d9SConrad Meyer ioat_model_resets_msix(struct ioat_softc *ioat)
5334253ea50SConrad Meyer {
5344253ea50SConrad Meyer 	u_int32_t pciid;
5354253ea50SConrad Meyer 
5364253ea50SConrad Meyer 	pciid = pci_get_devid(ioat->device);
5374253ea50SConrad Meyer 	switch (pciid) {
5380d1a05d9SConrad Meyer 		/* BWD: */
5390d1a05d9SConrad Meyer 	case 0x0c508086:
5400d1a05d9SConrad Meyer 	case 0x0c518086:
5410d1a05d9SConrad Meyer 	case 0x0c528086:
5420d1a05d9SConrad Meyer 	case 0x0c538086:
5430d1a05d9SConrad Meyer 		/* BDXDE: */
5444253ea50SConrad Meyer 	case 0x6f508086:
5454253ea50SConrad Meyer 	case 0x6f518086:
5464253ea50SConrad Meyer 	case 0x6f528086:
5474253ea50SConrad Meyer 	case 0x6f538086:
5484253ea50SConrad Meyer 		return (TRUE);
5494253ea50SConrad Meyer 	}
5504253ea50SConrad Meyer 
5514253ea50SConrad Meyer 	return (FALSE);
5524253ea50SConrad Meyer }
5534253ea50SConrad Meyer 
554e974f91cSConrad Meyer static void
555e974f91cSConrad Meyer ioat_interrupt_handler(void *arg)
556e974f91cSConrad Meyer {
557e974f91cSConrad Meyer 	struct ioat_softc *ioat = arg;
558e974f91cSConrad Meyer 
559e974f91cSConrad Meyer 	ioat_process_events(ioat);
560e974f91cSConrad Meyer }
561e974f91cSConrad Meyer 
562e974f91cSConrad Meyer static void
563e974f91cSConrad Meyer ioat_process_events(struct ioat_softc *ioat)
564e974f91cSConrad Meyer {
565e974f91cSConrad Meyer 	struct ioat_descriptor *desc;
566e974f91cSConrad Meyer 	struct bus_dmadesc *dmadesc;
567e974f91cSConrad Meyer 	uint64_t comp_update, status;
568e974f91cSConrad Meyer 	uint32_t completed;
569e974f91cSConrad Meyer 
570e974f91cSConrad Meyer 	mtx_lock(&ioat->cleanup_lock);
571e974f91cSConrad Meyer 
572e974f91cSConrad Meyer 	completed = 0;
573e974f91cSConrad Meyer 	comp_update = *ioat->comp_update;
574e974f91cSConrad Meyer 	status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK;
575e974f91cSConrad Meyer 
576e974f91cSConrad Meyer 	ioat_log_message(3, "%s\n", __func__);
577e974f91cSConrad Meyer 
5784becebdfSConrad Meyer 	if (status == ioat->last_seen)
5794becebdfSConrad Meyer 		goto out;
580e974f91cSConrad Meyer 
581e974f91cSConrad Meyer 	while (1) {
582e974f91cSConrad Meyer 		desc = ioat_get_ring_entry(ioat, ioat->tail);
583e974f91cSConrad Meyer 		dmadesc = &desc->bus_dmadesc;
584e974f91cSConrad Meyer 		ioat_log_message(3, "completing desc %d\n", ioat->tail);
585e974f91cSConrad Meyer 
586e974f91cSConrad Meyer 		if (dmadesc->callback_fn)
587e974f91cSConrad Meyer 			(*dmadesc->callback_fn)(dmadesc->callback_arg);
588e974f91cSConrad Meyer 
589466b3540SConrad Meyer 		completed++;
590e974f91cSConrad Meyer 		ioat->tail++;
591e974f91cSConrad Meyer 		if (desc->hw_desc_bus_addr == status)
592e974f91cSConrad Meyer 			break;
593e974f91cSConrad Meyer 	}
594e974f91cSConrad Meyer 
595e974f91cSConrad Meyer 	ioat->last_seen = desc->hw_desc_bus_addr;
596e974f91cSConrad Meyer 
597e974f91cSConrad Meyer 	if (ioat->head == ioat->tail) {
598e974f91cSConrad Meyer 		ioat->is_completion_pending = FALSE;
599fe720f5aSConrad Meyer 		callout_reset(&ioat->timer, IOAT_INTR_TIMO,
600fe720f5aSConrad Meyer 		    ioat_timer_callback, ioat);
601e974f91cSConrad Meyer 	}
602e974f91cSConrad Meyer 
6034becebdfSConrad Meyer out:
604e974f91cSConrad Meyer 	ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
605e974f91cSConrad Meyer 	mtx_unlock(&ioat->cleanup_lock);
606466b3540SConrad Meyer 
607466b3540SConrad Meyer 	ioat_putn(ioat, completed, IOAT_ACTIVE_DESCR_REF);
608e974f91cSConrad Meyer }
609e974f91cSConrad Meyer 
610e974f91cSConrad Meyer /*
611e974f91cSConrad Meyer  * User API functions
612e974f91cSConrad Meyer  */
613e974f91cSConrad Meyer bus_dmaengine_t
614e974f91cSConrad Meyer ioat_get_dmaengine(uint32_t index)
615e974f91cSConrad Meyer {
616e974f91cSConrad Meyer 
617466b3540SConrad Meyer 	if (index >= ioat_channel_index)
618e974f91cSConrad Meyer 		return (NULL);
619466b3540SConrad Meyer 	return (&ioat_get(ioat_channel[index], IOAT_DMAENGINE_REF)->dmaengine);
620466b3540SConrad Meyer }
621466b3540SConrad Meyer 
622466b3540SConrad Meyer void
623466b3540SConrad Meyer ioat_put_dmaengine(bus_dmaengine_t dmaengine)
624466b3540SConrad Meyer {
625466b3540SConrad Meyer 	struct ioat_softc *ioat;
626466b3540SConrad Meyer 
627466b3540SConrad Meyer 	ioat = to_ioat_softc(dmaengine);
628466b3540SConrad Meyer 	ioat_put(ioat, IOAT_DMAENGINE_REF);
629e974f91cSConrad Meyer }
630e974f91cSConrad Meyer 
631e974f91cSConrad Meyer void
632e974f91cSConrad Meyer ioat_acquire(bus_dmaengine_t dmaengine)
633e974f91cSConrad Meyer {
634e974f91cSConrad Meyer 	struct ioat_softc *ioat;
635e974f91cSConrad Meyer 
636e974f91cSConrad Meyer 	ioat = to_ioat_softc(dmaengine);
637e974f91cSConrad Meyer 	mtx_lock(&ioat->submit_lock);
638e974f91cSConrad Meyer 	ioat_log_message(3, "%s\n", __func__);
639e974f91cSConrad Meyer }
640e974f91cSConrad Meyer 
641e974f91cSConrad Meyer void
642e974f91cSConrad Meyer ioat_release(bus_dmaengine_t dmaengine)
643e974f91cSConrad Meyer {
644e974f91cSConrad Meyer 	struct ioat_softc *ioat;
645e974f91cSConrad Meyer 
646e974f91cSConrad Meyer 	ioat = to_ioat_softc(dmaengine);
6471c25420eSConrad Meyer 	ioat_log_message(3, "%s\n", __func__);
648e974f91cSConrad Meyer 	ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, (uint16_t)ioat->head);
649e974f91cSConrad Meyer 	mtx_unlock(&ioat->submit_lock);
650e974f91cSConrad Meyer }
651e974f91cSConrad Meyer 
652e974f91cSConrad Meyer struct bus_dmadesc *
653e974f91cSConrad Meyer ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_callback_t callback_fn,
654e974f91cSConrad Meyer     void *callback_arg, uint32_t flags)
655e974f91cSConrad Meyer {
656e974f91cSConrad Meyer 	struct ioat_softc *ioat;
657e974f91cSConrad Meyer 	struct ioat_descriptor *desc;
658e974f91cSConrad Meyer 	struct ioat_dma_hw_descriptor *hw_desc;
659e974f91cSConrad Meyer 
660e974f91cSConrad Meyer 	KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x",
661e974f91cSConrad Meyer 		flags & ~DMA_ALL_FLAGS));
662e974f91cSConrad Meyer 
663e974f91cSConrad Meyer 	ioat = to_ioat_softc(dmaengine);
66409f49f24SConrad Meyer 	mtx_assert(&ioat->submit_lock, MA_OWNED);
665e974f91cSConrad Meyer 
666e974f91cSConrad Meyer 	if (ioat_reserve_space_and_lock(ioat, 1) != 0)
667e974f91cSConrad Meyer 		return (NULL);
668e974f91cSConrad Meyer 
669e974f91cSConrad Meyer 	ioat_log_message(3, "%s\n", __func__);
670e974f91cSConrad Meyer 
671e974f91cSConrad Meyer 	desc = ioat_get_ring_entry(ioat, ioat->head);
672e974f91cSConrad Meyer 	hw_desc = desc->u.dma;
673e974f91cSConrad Meyer 
674e974f91cSConrad Meyer 	hw_desc->u.control_raw = 0;
675e974f91cSConrad Meyer 	hw_desc->u.control.null = 1;
676e974f91cSConrad Meyer 	hw_desc->u.control.completion_update = 1;
677e974f91cSConrad Meyer 
678e974f91cSConrad Meyer 	if ((flags & DMA_INT_EN) != 0)
679e974f91cSConrad Meyer 		hw_desc->u.control.int_enable = 1;
680e974f91cSConrad Meyer 
681e974f91cSConrad Meyer 	hw_desc->size = 8;
682e974f91cSConrad Meyer 	hw_desc->src_addr = 0;
683e974f91cSConrad Meyer 	hw_desc->dest_addr = 0;
684e974f91cSConrad Meyer 
685e974f91cSConrad Meyer 	desc->bus_dmadesc.callback_fn = callback_fn;
686e974f91cSConrad Meyer 	desc->bus_dmadesc.callback_arg = callback_arg;
687e974f91cSConrad Meyer 
688e974f91cSConrad Meyer 	ioat_submit_single(ioat);
689e974f91cSConrad Meyer 	return (&desc->bus_dmadesc);
690e974f91cSConrad Meyer }
691e974f91cSConrad Meyer 
692e974f91cSConrad Meyer struct bus_dmadesc *
693e974f91cSConrad Meyer ioat_copy(bus_dmaengine_t dmaengine, bus_addr_t dst,
694e974f91cSConrad Meyer     bus_addr_t src, bus_size_t len, bus_dmaengine_callback_t callback_fn,
695e974f91cSConrad Meyer     void *callback_arg, uint32_t flags)
696e974f91cSConrad Meyer {
697e974f91cSConrad Meyer 	struct ioat_descriptor *desc;
698e974f91cSConrad Meyer 	struct ioat_dma_hw_descriptor *hw_desc;
699e974f91cSConrad Meyer 	struct ioat_softc *ioat;
700e974f91cSConrad Meyer 
701e974f91cSConrad Meyer 	KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x",
702e974f91cSConrad Meyer 		flags & ~DMA_ALL_FLAGS));
703e974f91cSConrad Meyer 
704e974f91cSConrad Meyer 	ioat = to_ioat_softc(dmaengine);
70509f49f24SConrad Meyer 	mtx_assert(&ioat->submit_lock, MA_OWNED);
706e974f91cSConrad Meyer 
707e974f91cSConrad Meyer 	if (len > ioat->max_xfer_size) {
708e974f91cSConrad Meyer 		ioat_log_message(0, "%s: max_xfer_size = %d, requested = %d\n",
709e974f91cSConrad Meyer 		    __func__, ioat->max_xfer_size, (int)len);
710e974f91cSConrad Meyer 		return (NULL);
711e974f91cSConrad Meyer 	}
712e974f91cSConrad Meyer 
713e974f91cSConrad Meyer 	if (ioat_reserve_space_and_lock(ioat, 1) != 0)
714e974f91cSConrad Meyer 		return (NULL);
715e974f91cSConrad Meyer 
716e974f91cSConrad Meyer 	ioat_log_message(3, "%s\n", __func__);
717e974f91cSConrad Meyer 
718e974f91cSConrad Meyer 	desc = ioat_get_ring_entry(ioat, ioat->head);
719e974f91cSConrad Meyer 	hw_desc = desc->u.dma;
720e974f91cSConrad Meyer 
721e974f91cSConrad Meyer 	hw_desc->u.control_raw = 0;
722e974f91cSConrad Meyer 	hw_desc->u.control.completion_update = 1;
723e974f91cSConrad Meyer 
724e974f91cSConrad Meyer 	if ((flags & DMA_INT_EN) != 0)
725e974f91cSConrad Meyer 		hw_desc->u.control.int_enable = 1;
726e974f91cSConrad Meyer 
727e974f91cSConrad Meyer 	hw_desc->size = len;
728e974f91cSConrad Meyer 	hw_desc->src_addr = src;
729e974f91cSConrad Meyer 	hw_desc->dest_addr = dst;
730e974f91cSConrad Meyer 
731e974f91cSConrad Meyer 	if (g_ioat_debug_level >= 3)
732e974f91cSConrad Meyer 		dump_descriptor(hw_desc);
733e974f91cSConrad Meyer 
734e974f91cSConrad Meyer 	desc->bus_dmadesc.callback_fn = callback_fn;
735e974f91cSConrad Meyer 	desc->bus_dmadesc.callback_arg = callback_arg;
736e974f91cSConrad Meyer 
737e974f91cSConrad Meyer 	ioat_submit_single(ioat);
738e974f91cSConrad Meyer 	return (&desc->bus_dmadesc);
739e974f91cSConrad Meyer }
740e974f91cSConrad Meyer 
741e974f91cSConrad Meyer /*
742e974f91cSConrad Meyer  * Ring Management
743e974f91cSConrad Meyer  */
744e974f91cSConrad Meyer static inline uint32_t
745e974f91cSConrad Meyer ioat_get_active(struct ioat_softc *ioat)
746e974f91cSConrad Meyer {
747e974f91cSConrad Meyer 
748e974f91cSConrad Meyer 	return ((ioat->head - ioat->tail) & ((1 << ioat->ring_size_order) - 1));
749e974f91cSConrad Meyer }
750e974f91cSConrad Meyer 
751e974f91cSConrad Meyer static inline uint32_t
752e974f91cSConrad Meyer ioat_get_ring_space(struct ioat_softc *ioat)
753e974f91cSConrad Meyer {
754e974f91cSConrad Meyer 
755e974f91cSConrad Meyer 	return ((1 << ioat->ring_size_order) - ioat_get_active(ioat) - 1);
756e974f91cSConrad Meyer }
757e974f91cSConrad Meyer 
758e974f91cSConrad Meyer static struct ioat_descriptor *
759e974f91cSConrad Meyer ioat_alloc_ring_entry(struct ioat_softc *ioat)
760e974f91cSConrad Meyer {
761e974f91cSConrad Meyer 	struct ioat_dma_hw_descriptor *hw_desc;
762e974f91cSConrad Meyer 	struct ioat_descriptor *desc;
763f46011aeSConrad Meyer 	int error;
764e974f91cSConrad Meyer 
765f46011aeSConrad Meyer 	error = ENOMEM;
766f46011aeSConrad Meyer 	hw_desc = NULL;
767f46011aeSConrad Meyer 
768f46011aeSConrad Meyer 	desc = malloc(sizeof(*desc), M_IOAT, M_NOWAIT);
769e974f91cSConrad Meyer 	if (desc == NULL)
770f46011aeSConrad Meyer 		goto out;
771e974f91cSConrad Meyer 
772f46011aeSConrad Meyer 	bus_dmamem_alloc(ioat->hw_desc_tag, (void **)&hw_desc,
773f46011aeSConrad Meyer 	    BUS_DMA_ZERO | BUS_DMA_NOWAIT, &ioat->hw_desc_map);
774f46011aeSConrad Meyer 	if (hw_desc == NULL)
775f46011aeSConrad Meyer 		goto out;
776e974f91cSConrad Meyer 
777e974f91cSConrad Meyer 	desc->u.dma = hw_desc;
778f46011aeSConrad Meyer 
779f46011aeSConrad Meyer 	error = bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc,
780f46011aeSConrad Meyer 	    sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr,
781f46011aeSConrad Meyer 	    BUS_DMA_NOWAIT);
782f46011aeSConrad Meyer 	if (error)
783f46011aeSConrad Meyer 		goto out;
784f46011aeSConrad Meyer 
785f46011aeSConrad Meyer out:
786f46011aeSConrad Meyer 	if (error) {
787f46011aeSConrad Meyer 		ioat_free_ring_entry(ioat, desc);
788f46011aeSConrad Meyer 		return (NULL);
789f46011aeSConrad Meyer 	}
790e974f91cSConrad Meyer 	return (desc);
791e974f91cSConrad Meyer }
792e974f91cSConrad Meyer 
793e974f91cSConrad Meyer static void
794e974f91cSConrad Meyer ioat_free_ring_entry(struct ioat_softc *ioat, struct ioat_descriptor *desc)
795e974f91cSConrad Meyer {
796e974f91cSConrad Meyer 
797e974f91cSConrad Meyer 	if (desc == NULL)
798e974f91cSConrad Meyer 		return;
799e974f91cSConrad Meyer 
800e974f91cSConrad Meyer 	if (desc->u.dma)
801e974f91cSConrad Meyer 		bus_dmamem_free(ioat->hw_desc_tag, desc->u.dma,
802e974f91cSConrad Meyer 		    ioat->hw_desc_map);
803e974f91cSConrad Meyer 	free(desc, M_IOAT);
804e974f91cSConrad Meyer }
805e974f91cSConrad Meyer 
806e974f91cSConrad Meyer static int
807e974f91cSConrad Meyer ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs)
808e974f91cSConrad Meyer {
809e974f91cSConrad Meyer 	boolean_t retry;
810e974f91cSConrad Meyer 
811e974f91cSConrad Meyer 	while (1) {
812e974f91cSConrad Meyer 		if (ioat_get_ring_space(ioat) >= num_descs)
813e974f91cSConrad Meyer 			return (0);
814e974f91cSConrad Meyer 
815e974f91cSConrad Meyer 		mtx_lock(&ioat->cleanup_lock);
816e974f91cSConrad Meyer 		retry = resize_ring(ioat, ioat->ring_size_order + 1);
817e974f91cSConrad Meyer 		mtx_unlock(&ioat->cleanup_lock);
818e974f91cSConrad Meyer 
819e974f91cSConrad Meyer 		if (!retry)
820e974f91cSConrad Meyer 			return (ENOMEM);
821e974f91cSConrad Meyer 	}
822e974f91cSConrad Meyer }
823e974f91cSConrad Meyer 
824e974f91cSConrad Meyer static struct ioat_descriptor *
825e974f91cSConrad Meyer ioat_get_ring_entry(struct ioat_softc *ioat, uint32_t index)
826e974f91cSConrad Meyer {
827e974f91cSConrad Meyer 
828e974f91cSConrad Meyer 	return (ioat->ring[index % (1 << ioat->ring_size_order)]);
829e974f91cSConrad Meyer }
830e974f91cSConrad Meyer 
831e974f91cSConrad Meyer static boolean_t
832e974f91cSConrad Meyer resize_ring(struct ioat_softc *ioat, int order)
833e974f91cSConrad Meyer {
834e974f91cSConrad Meyer 	struct ioat_descriptor **ring;
835e974f91cSConrad Meyer 	struct ioat_descriptor *next;
836e974f91cSConrad Meyer 	struct ioat_dma_hw_descriptor *hw;
837e974f91cSConrad Meyer 	struct ioat_descriptor *ent;
838e974f91cSConrad Meyer 	uint32_t current_size, active, new_size, i, new_idx, current_idx;
839e974f91cSConrad Meyer 	uint32_t new_idx2;
840e974f91cSConrad Meyer 
841e974f91cSConrad Meyer 	current_size = 1 << ioat->ring_size_order;
842e974f91cSConrad Meyer 	active = (ioat->head - ioat->tail) & (current_size - 1);
843e974f91cSConrad Meyer 	new_size = 1 << order;
844e974f91cSConrad Meyer 
845e974f91cSConrad Meyer 	if (order > IOAT_MAX_ORDER)
846e974f91cSConrad Meyer 		return (FALSE);
847e974f91cSConrad Meyer 
848e974f91cSConrad Meyer 	/*
849e974f91cSConrad Meyer 	 * when shrinking, verify that we can hold the current active
850e974f91cSConrad Meyer 	 * set in the new ring
851e974f91cSConrad Meyer 	 */
852e974f91cSConrad Meyer 	if (active >= new_size)
853e974f91cSConrad Meyer 		return (FALSE);
854e974f91cSConrad Meyer 
855e974f91cSConrad Meyer 	/* allocate the array to hold the software ring */
856e974f91cSConrad Meyer 	ring = malloc(new_size * sizeof(*ring), M_IOAT, M_ZERO | M_NOWAIT);
857e974f91cSConrad Meyer 	if (ring == NULL)
858e974f91cSConrad Meyer 		return (FALSE);
859e974f91cSConrad Meyer 
860e974f91cSConrad Meyer 	ioat_log_message(2, "ring resize: new: %d old: %d\n",
861e974f91cSConrad Meyer 	    new_size, current_size);
862e974f91cSConrad Meyer 
863e974f91cSConrad Meyer 	/* allocate/trim descriptors as needed */
864e974f91cSConrad Meyer 	if (new_size > current_size) {
865e974f91cSConrad Meyer 		/* copy current descriptors to the new ring */
866e974f91cSConrad Meyer 		for (i = 0; i < current_size; i++) {
867e974f91cSConrad Meyer 			current_idx = (ioat->tail + i) & (current_size - 1);
868e974f91cSConrad Meyer 			new_idx = (ioat->tail + i) & (new_size - 1);
869e974f91cSConrad Meyer 
870e974f91cSConrad Meyer 			ring[new_idx] = ioat->ring[current_idx];
871e974f91cSConrad Meyer 			ring[new_idx]->id = new_idx;
872e974f91cSConrad Meyer 		}
873e974f91cSConrad Meyer 
874e974f91cSConrad Meyer 		/* add new descriptors to the ring */
875e974f91cSConrad Meyer 		for (i = current_size; i < new_size; i++) {
876e974f91cSConrad Meyer 			new_idx = (ioat->tail + i) & (new_size - 1);
877e974f91cSConrad Meyer 
878e974f91cSConrad Meyer 			ring[new_idx] = ioat_alloc_ring_entry(ioat);
8798c8e8487SConrad Meyer 			if (ring[new_idx] == NULL) {
880e974f91cSConrad Meyer 				while (i--) {
881e974f91cSConrad Meyer 					new_idx2 = (ioat->tail + i) &
882e974f91cSConrad Meyer 					    (new_size - 1);
883e974f91cSConrad Meyer 
884e974f91cSConrad Meyer 					ioat_free_ring_entry(ioat,
885e974f91cSConrad Meyer 					    ring[new_idx2]);
886e974f91cSConrad Meyer 				}
887e974f91cSConrad Meyer 				free(ring, M_IOAT);
888e974f91cSConrad Meyer 				return (FALSE);
889e974f91cSConrad Meyer 			}
890e974f91cSConrad Meyer 			ring[new_idx]->id = new_idx;
891e974f91cSConrad Meyer 		}
892e974f91cSConrad Meyer 
893e974f91cSConrad Meyer 		for (i = current_size - 1; i < new_size; i++) {
894e974f91cSConrad Meyer 			new_idx = (ioat->tail + i) & (new_size - 1);
895e974f91cSConrad Meyer 			next = ring[(new_idx + 1) & (new_size - 1)];
896e974f91cSConrad Meyer 			hw = ring[new_idx]->u.dma;
897e974f91cSConrad Meyer 
898e974f91cSConrad Meyer 			hw->next = next->hw_desc_bus_addr;
899e974f91cSConrad Meyer 		}
900e974f91cSConrad Meyer 	} else {
901e974f91cSConrad Meyer 		/*
902e974f91cSConrad Meyer 		 * copy current descriptors to the new ring, dropping the
903e974f91cSConrad Meyer 		 * removed descriptors
904e974f91cSConrad Meyer 		 */
905e974f91cSConrad Meyer 		for (i = 0; i < new_size; i++) {
906e974f91cSConrad Meyer 			current_idx = (ioat->tail + i) & (current_size - 1);
907e974f91cSConrad Meyer 			new_idx = (ioat->tail + i) & (new_size - 1);
908e974f91cSConrad Meyer 
909e974f91cSConrad Meyer 			ring[new_idx] = ioat->ring[current_idx];
910e974f91cSConrad Meyer 			ring[new_idx]->id = new_idx;
911e974f91cSConrad Meyer 		}
912e974f91cSConrad Meyer 
913e974f91cSConrad Meyer 		/* free deleted descriptors */
914e974f91cSConrad Meyer 		for (i = new_size; i < current_size; i++) {
915e974f91cSConrad Meyer 			ent = ioat_get_ring_entry(ioat, ioat->tail + i);
916e974f91cSConrad Meyer 			ioat_free_ring_entry(ioat, ent);
917e974f91cSConrad Meyer 		}
918e974f91cSConrad Meyer 
919e974f91cSConrad Meyer 		/* fix up hardware ring */
920e974f91cSConrad Meyer 		hw = ring[(ioat->tail + new_size - 1) & (new_size - 1)]->u.dma;
921e974f91cSConrad Meyer 		next = ring[(ioat->tail + new_size) & (new_size - 1)];
922e974f91cSConrad Meyer 		hw->next = next->hw_desc_bus_addr;
923e974f91cSConrad Meyer 	}
924e974f91cSConrad Meyer 
925e974f91cSConrad Meyer 	free(ioat->ring, M_IOAT);
926e974f91cSConrad Meyer 	ioat->ring = ring;
927e974f91cSConrad Meyer 	ioat->ring_size_order = order;
928e974f91cSConrad Meyer 
929e974f91cSConrad Meyer 	return (TRUE);
930e974f91cSConrad Meyer }
931e974f91cSConrad Meyer 
932e974f91cSConrad Meyer static void
9338f274637SConrad Meyer ioat_halted_debug(struct ioat_softc *ioat, uint32_t chanerr)
934e974f91cSConrad Meyer {
935e974f91cSConrad Meyer 	struct ioat_descriptor *desc;
9368f274637SConrad Meyer 
9378f274637SConrad Meyer 	ioat_log_message(0, "Channel halted (%x)\n", chanerr);
9388f274637SConrad Meyer 	if (chanerr == 0)
9398f274637SConrad Meyer 		return;
9408f274637SConrad Meyer 
9418f274637SConrad Meyer 	desc = ioat_get_ring_entry(ioat, ioat->tail + 0);
9428f274637SConrad Meyer 	dump_descriptor(desc->u.raw);
9438f274637SConrad Meyer 
9448f274637SConrad Meyer 	desc = ioat_get_ring_entry(ioat, ioat->tail + 1);
9458f274637SConrad Meyer 	dump_descriptor(desc->u.raw);
9468f274637SConrad Meyer }
9478f274637SConrad Meyer 
9488f274637SConrad Meyer static void
9498f274637SConrad Meyer ioat_timer_callback(void *arg)
9508f274637SConrad Meyer {
951e974f91cSConrad Meyer 	struct ioat_softc *ioat;
952e974f91cSConrad Meyer 	uint64_t status;
953e974f91cSConrad Meyer 	uint32_t chanerr;
954e974f91cSConrad Meyer 
955e974f91cSConrad Meyer 	ioat = arg;
956fe720f5aSConrad Meyer 	ioat_log_message(1, "%s\n", __func__);
957e974f91cSConrad Meyer 
958e974f91cSConrad Meyer 	if (ioat->is_completion_pending) {
959e974f91cSConrad Meyer 		status = ioat_get_chansts(ioat);
960e974f91cSConrad Meyer 
961e974f91cSConrad Meyer 		/*
962e974f91cSConrad Meyer 		 * When halted due to errors, check for channel programming
963e974f91cSConrad Meyer 		 * errors before advancing the completion state.
964e974f91cSConrad Meyer 		 */
965e974f91cSConrad Meyer 		if (is_ioat_halted(status)) {
966e974f91cSConrad Meyer 			chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
9678f274637SConrad Meyer 			ioat_halted_debug(ioat, chanerr);
968e974f91cSConrad Meyer 		}
969e974f91cSConrad Meyer 		ioat_process_events(ioat);
970e974f91cSConrad Meyer 	} else {
971e974f91cSConrad Meyer 		mtx_lock(&ioat->submit_lock);
972e974f91cSConrad Meyer 		mtx_lock(&ioat->cleanup_lock);
973e974f91cSConrad Meyer 
974e974f91cSConrad Meyer 		if (ioat_get_active(ioat) == 0 &&
975e974f91cSConrad Meyer 		    ioat->ring_size_order > IOAT_MIN_ORDER)
976e974f91cSConrad Meyer 			resize_ring(ioat, ioat->ring_size_order - 1);
977e974f91cSConrad Meyer 
978e974f91cSConrad Meyer 		mtx_unlock(&ioat->cleanup_lock);
979e974f91cSConrad Meyer 		mtx_unlock(&ioat->submit_lock);
980e974f91cSConrad Meyer 
981e974f91cSConrad Meyer 		if (ioat->ring_size_order > IOAT_MIN_ORDER)
982fe720f5aSConrad Meyer 			callout_reset(&ioat->timer, IOAT_INTR_TIMO,
983e974f91cSConrad Meyer 			    ioat_timer_callback, ioat);
984e974f91cSConrad Meyer 	}
985e974f91cSConrad Meyer }
986e974f91cSConrad Meyer 
987e974f91cSConrad Meyer /*
988e974f91cSConrad Meyer  * Support Functions
989e974f91cSConrad Meyer  */
990e974f91cSConrad Meyer static void
991e974f91cSConrad Meyer ioat_submit_single(struct ioat_softc *ioat)
992e974f91cSConrad Meyer {
993e974f91cSConrad Meyer 
994466b3540SConrad Meyer 	ioat_get(ioat, IOAT_ACTIVE_DESCR_REF);
995e974f91cSConrad Meyer 	atomic_add_rel_int(&ioat->head, 1);
996e974f91cSConrad Meyer 
997e974f91cSConrad Meyer 	if (!ioat->is_completion_pending) {
998e974f91cSConrad Meyer 		ioat->is_completion_pending = TRUE;
999fe720f5aSConrad Meyer 		callout_reset(&ioat->timer, IOAT_INTR_TIMO,
1000fe720f5aSConrad Meyer 		    ioat_timer_callback, ioat);
1001e974f91cSConrad Meyer 	}
1002e974f91cSConrad Meyer }
1003e974f91cSConrad Meyer 
1004e974f91cSConrad Meyer static int
1005e974f91cSConrad Meyer ioat_reset_hw(struct ioat_softc *ioat)
1006e974f91cSConrad Meyer {
1007e974f91cSConrad Meyer 	uint64_t status;
1008e974f91cSConrad Meyer 	uint32_t chanerr;
10090d1a05d9SConrad Meyer 	int timeout;
1010e974f91cSConrad Meyer 
1011e974f91cSConrad Meyer 	status = ioat_get_chansts(ioat);
1012e974f91cSConrad Meyer 	if (is_ioat_active(status) || is_ioat_idle(status))
1013e974f91cSConrad Meyer 		ioat_suspend(ioat);
1014e974f91cSConrad Meyer 
1015e974f91cSConrad Meyer 	/* Wait at most 20 ms */
1016e974f91cSConrad Meyer 	for (timeout = 0; (is_ioat_active(status) || is_ioat_idle(status)) &&
1017e974f91cSConrad Meyer 	    timeout < 20; timeout++) {
1018e974f91cSConrad Meyer 		DELAY(1000);
1019e974f91cSConrad Meyer 		status = ioat_get_chansts(ioat);
1020e974f91cSConrad Meyer 	}
1021e974f91cSConrad Meyer 	if (timeout == 20)
1022e974f91cSConrad Meyer 		return (ETIMEDOUT);
1023e974f91cSConrad Meyer 
1024e974f91cSConrad Meyer 	chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
1025e974f91cSConrad Meyer 	ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr);
1026e974f91cSConrad Meyer 
1027e974f91cSConrad Meyer 	/*
1028e974f91cSConrad Meyer 	 * IOAT v3 workaround - CHANERRMSK_INT with 3E07h to masks out errors
1029e974f91cSConrad Meyer 	 *  that can cause stability issues for IOAT v3.
1030e974f91cSConrad Meyer 	 */
1031e974f91cSConrad Meyer 	pci_write_config(ioat->device, IOAT_CFG_CHANERRMASK_INT_OFFSET, 0x3e07,
1032e974f91cSConrad Meyer 	    4);
1033e974f91cSConrad Meyer 	chanerr = pci_read_config(ioat->device, IOAT_CFG_CHANERR_INT_OFFSET, 4);
1034e974f91cSConrad Meyer 	pci_write_config(ioat->device, IOAT_CFG_CHANERR_INT_OFFSET, chanerr, 4);
1035e974f91cSConrad Meyer 
10360d1a05d9SConrad Meyer 	/*
10370d1a05d9SConrad Meyer 	 * BDXDE and BWD models reset MSI-X registers on device reset.
10380d1a05d9SConrad Meyer 	 * Save/restore their contents manually.
10390d1a05d9SConrad Meyer 	 */
1040f7157235SConrad Meyer 	if (ioat_model_resets_msix(ioat)) {
1041f7157235SConrad Meyer 		ioat_log_message(1, "device resets MSI-X registers; saving\n");
10420d1a05d9SConrad Meyer 		pci_save_state(ioat->device);
1043f7157235SConrad Meyer 	}
10440d1a05d9SConrad Meyer 
1045e974f91cSConrad Meyer 	ioat_reset(ioat);
1046e974f91cSConrad Meyer 
1047e974f91cSConrad Meyer 	/* Wait at most 20 ms */
1048e974f91cSConrad Meyer 	for (timeout = 0; ioat_reset_pending(ioat) && timeout < 20; timeout++)
1049e974f91cSConrad Meyer 		DELAY(1000);
1050e974f91cSConrad Meyer 	if (timeout == 20)
1051e974f91cSConrad Meyer 		return (ETIMEDOUT);
1052e974f91cSConrad Meyer 
1053f7157235SConrad Meyer 	if (ioat_model_resets_msix(ioat)) {
1054f7157235SConrad Meyer 		ioat_log_message(1, "device resets registers; restored\n");
10550d1a05d9SConrad Meyer 		pci_restore_state(ioat->device);
1056f7157235SConrad Meyer 	}
10574253ea50SConrad Meyer 
1058e974f91cSConrad Meyer 	return (0);
1059e974f91cSConrad Meyer }
1060e974f91cSConrad Meyer 
1061f7157235SConrad Meyer static int
1062f7157235SConrad Meyer sysctl_handle_reset(SYSCTL_HANDLER_ARGS)
1063f7157235SConrad Meyer {
1064f7157235SConrad Meyer 	struct ioat_softc *ioat;
1065f7157235SConrad Meyer 	int error, arg;
1066f7157235SConrad Meyer 
1067f7157235SConrad Meyer 	ioat = arg1;
1068f7157235SConrad Meyer 
1069f7157235SConrad Meyer 	arg = 0;
1070f7157235SConrad Meyer 	error = SYSCTL_OUT(req, &arg, sizeof(arg));
1071f7157235SConrad Meyer 	if (error != 0 || req->newptr == NULL)
1072f7157235SConrad Meyer 		return (error);
1073f7157235SConrad Meyer 
1074f7157235SConrad Meyer 	error = SYSCTL_IN(req, &arg, sizeof(arg));
1075f7157235SConrad Meyer 	if (error != 0)
1076f7157235SConrad Meyer 		return (error);
1077f7157235SConrad Meyer 
1078f7157235SConrad Meyer 	if (arg != 0)
1079f7157235SConrad Meyer 		error = ioat_reset_hw(ioat);
1080f7157235SConrad Meyer 
1081f7157235SConrad Meyer 	return (error);
1082f7157235SConrad Meyer }
1083f7157235SConrad Meyer 
1084e974f91cSConrad Meyer static void
1085e974f91cSConrad Meyer dump_descriptor(void *hw_desc)
1086e974f91cSConrad Meyer {
1087e974f91cSConrad Meyer 	int i, j;
1088e974f91cSConrad Meyer 
1089e974f91cSConrad Meyer 	for (i = 0; i < 2; i++) {
1090e974f91cSConrad Meyer 		for (j = 0; j < 8; j++)
1091e974f91cSConrad Meyer 			printf("%08x ", ((uint32_t *)hw_desc)[i * 8 + j]);
1092e974f91cSConrad Meyer 		printf("\n");
1093e974f91cSConrad Meyer 	}
1094e974f91cSConrad Meyer }
1095e974f91cSConrad Meyer 
1096e974f91cSConrad Meyer static void
1097e974f91cSConrad Meyer ioat_setup_sysctl(device_t device)
1098e974f91cSConrad Meyer {
1099f7157235SConrad Meyer 	struct sysctl_oid_list *par;
1100f7157235SConrad Meyer 	struct sysctl_ctx_list *ctx;
1101f7157235SConrad Meyer 	struct sysctl_oid *tree;
1102e974f91cSConrad Meyer 	struct ioat_softc *ioat;
1103e974f91cSConrad Meyer 
1104e974f91cSConrad Meyer 	ioat = DEVICE2SOFTC(device);
1105f7157235SConrad Meyer 	ctx = device_get_sysctl_ctx(device);
1106f7157235SConrad Meyer 	tree = device_get_sysctl_tree(device);
1107f7157235SConrad Meyer 	par = SYSCTL_CHILDREN(tree);
1108e974f91cSConrad Meyer 
1109f7157235SConrad Meyer 	SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "ring_size_order", CTLFLAG_RD,
1110f7157235SConrad Meyer 	    &ioat->ring_size_order, 0, "HW descriptor ring size order");
1111f7157235SConrad Meyer 	SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "head", CTLFLAG_RD, &ioat->head, 0,
1112f7157235SConrad Meyer 	    "HW descriptor head pointer index");
1113f7157235SConrad Meyer 	SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "tail", CTLFLAG_RD, &ioat->tail, 0,
1114f7157235SConrad Meyer 	    "HW descriptor tail pointer index");
1115f7157235SConrad Meyer 
1116f7157235SConrad Meyer 	SYSCTL_ADD_PROC(ctx, par, OID_AUTO, "force_hw_reset",
1117f7157235SConrad Meyer 	    CTLTYPE_INT | CTLFLAG_RW, ioat, 0, sysctl_handle_reset, "I",
1118f7157235SConrad Meyer 	    "Set to non-zero to reset the hardware");
1119e974f91cSConrad Meyer }
1120466b3540SConrad Meyer 
1121466b3540SConrad Meyer static inline struct ioat_softc *
1122466b3540SConrad Meyer ioat_get(struct ioat_softc *ioat, enum ioat_ref_kind kind)
1123466b3540SConrad Meyer {
1124466b3540SConrad Meyer 	uint32_t old;
1125466b3540SConrad Meyer 
1126466b3540SConrad Meyer 	KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus"));
1127466b3540SConrad Meyer 
1128466b3540SConrad Meyer 	old = atomic_fetchadd_32(&ioat->refcnt, 1);
1129466b3540SConrad Meyer 	KASSERT(old < UINT32_MAX, ("refcnt overflow"));
1130466b3540SConrad Meyer 
1131466b3540SConrad Meyer #ifdef INVARIANTS
1132466b3540SConrad Meyer 	old = atomic_fetchadd_32(&ioat->refkinds[kind], 1);
1133466b3540SConrad Meyer 	KASSERT(old < UINT32_MAX, ("refcnt kind overflow"));
1134466b3540SConrad Meyer #endif
1135466b3540SConrad Meyer 
1136466b3540SConrad Meyer 	return (ioat);
1137466b3540SConrad Meyer }
1138466b3540SConrad Meyer 
1139466b3540SConrad Meyer static inline void
1140466b3540SConrad Meyer ioat_putn(struct ioat_softc *ioat, uint32_t n, enum ioat_ref_kind kind)
1141466b3540SConrad Meyer {
1142466b3540SConrad Meyer 	uint32_t old;
1143466b3540SConrad Meyer 
1144466b3540SConrad Meyer 	KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus"));
1145466b3540SConrad Meyer 
1146466b3540SConrad Meyer 	if (n == 0)
1147466b3540SConrad Meyer 		return;
1148466b3540SConrad Meyer 
1149466b3540SConrad Meyer #ifdef INVARIANTS
1150466b3540SConrad Meyer 	old = atomic_fetchadd_32(&ioat->refkinds[kind], -n);
1151466b3540SConrad Meyer 	KASSERT(old >= n, ("refcnt kind underflow"));
1152466b3540SConrad Meyer #endif
1153466b3540SConrad Meyer 
1154466b3540SConrad Meyer 	/* Skip acquiring the lock if resulting refcnt > 0. */
1155466b3540SConrad Meyer 	for (;;) {
1156466b3540SConrad Meyer 		old = ioat->refcnt;
1157466b3540SConrad Meyer 		if (old <= n)
1158466b3540SConrad Meyer 			break;
1159466b3540SConrad Meyer 		if (atomic_cmpset_32(&ioat->refcnt, old, old - n))
1160466b3540SConrad Meyer 			return;
1161466b3540SConrad Meyer 	}
1162466b3540SConrad Meyer 
1163466b3540SConrad Meyer 	mtx_lock(IOAT_REFLK);
1164466b3540SConrad Meyer 	old = atomic_fetchadd_32(&ioat->refcnt, -n);
1165466b3540SConrad Meyer 	KASSERT(old >= n, ("refcnt error"));
1166466b3540SConrad Meyer 
1167466b3540SConrad Meyer 	if (old == n)
1168466b3540SConrad Meyer 		wakeup(IOAT_REFLK);
1169466b3540SConrad Meyer 	mtx_unlock(IOAT_REFLK);
1170466b3540SConrad Meyer }
1171466b3540SConrad Meyer 
1172466b3540SConrad Meyer static inline void
1173466b3540SConrad Meyer ioat_put(struct ioat_softc *ioat, enum ioat_ref_kind kind)
1174466b3540SConrad Meyer {
1175466b3540SConrad Meyer 
1176466b3540SConrad Meyer 	ioat_putn(ioat, 1, kind);
1177466b3540SConrad Meyer }
1178466b3540SConrad Meyer 
1179466b3540SConrad Meyer static void
1180466b3540SConrad Meyer ioat_drain(struct ioat_softc *ioat)
1181466b3540SConrad Meyer {
1182466b3540SConrad Meyer 
1183466b3540SConrad Meyer 	mtx_lock(IOAT_REFLK);
1184466b3540SConrad Meyer 	while (ioat->refcnt > 0)
1185466b3540SConrad Meyer 		msleep(IOAT_REFLK, IOAT_REFLK, 0, "ioat_drain", 0);
1186466b3540SConrad Meyer 	mtx_unlock(IOAT_REFLK);
1187466b3540SConrad Meyer }
1188