xref: /freebsd/sys/dev/ioat/ioat.c (revision e974f91c38cfb7a97b684082089d6dae948a68fd)
1*e974f91cSConrad Meyer /*-
2*e974f91cSConrad Meyer  * Copyright (C) 2012 Intel Corporation
3*e974f91cSConrad Meyer  * All rights reserved.
4*e974f91cSConrad Meyer  *
5*e974f91cSConrad Meyer  * Redistribution and use in source and binary forms, with or without
6*e974f91cSConrad Meyer  * modification, are permitted provided that the following conditions
7*e974f91cSConrad Meyer  * are met:
8*e974f91cSConrad Meyer  * 1. Redistributions of source code must retain the above copyright
9*e974f91cSConrad Meyer  *    notice, this list of conditions and the following disclaimer.
10*e974f91cSConrad Meyer  * 2. Redistributions in binary form must reproduce the above copyright
11*e974f91cSConrad Meyer  *    notice, this list of conditions and the following disclaimer in the
12*e974f91cSConrad Meyer  *    documentation and/or other materials provided with the distribution.
13*e974f91cSConrad Meyer  *
14*e974f91cSConrad Meyer  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15*e974f91cSConrad Meyer  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16*e974f91cSConrad Meyer  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17*e974f91cSConrad Meyer  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18*e974f91cSConrad Meyer  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19*e974f91cSConrad Meyer  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20*e974f91cSConrad Meyer  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21*e974f91cSConrad Meyer  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22*e974f91cSConrad Meyer  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23*e974f91cSConrad Meyer  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24*e974f91cSConrad Meyer  * SUCH DAMAGE.
25*e974f91cSConrad Meyer  */
26*e974f91cSConrad Meyer 
27*e974f91cSConrad Meyer #include <sys/cdefs.h>
28*e974f91cSConrad Meyer __FBSDID("$FreeBSD$");
29*e974f91cSConrad Meyer 
30*e974f91cSConrad Meyer #include <sys/param.h>
31*e974f91cSConrad Meyer #include <sys/systm.h>
32*e974f91cSConrad Meyer #include <sys/bus.h>
33*e974f91cSConrad Meyer #include <sys/conf.h>
34*e974f91cSConrad Meyer #include <sys/ioccom.h>
35*e974f91cSConrad Meyer #include <sys/kernel.h>
36*e974f91cSConrad Meyer #include <sys/lock.h>
37*e974f91cSConrad Meyer #include <sys/malloc.h>
38*e974f91cSConrad Meyer #include <sys/module.h>
39*e974f91cSConrad Meyer #include <sys/mutex.h>
40*e974f91cSConrad Meyer #include <sys/rman.h>
41*e974f91cSConrad Meyer #include <sys/sysctl.h>
42*e974f91cSConrad Meyer #include <sys/time.h>
43*e974f91cSConrad Meyer #include <dev/pci/pcireg.h>
44*e974f91cSConrad Meyer #include <dev/pci/pcivar.h>
45*e974f91cSConrad Meyer #include <machine/bus.h>
46*e974f91cSConrad Meyer #include <machine/resource.h>
47*e974f91cSConrad Meyer #include <machine/stdarg.h>
48*e974f91cSConrad Meyer 
49*e974f91cSConrad Meyer #include "ioat.h"
50*e974f91cSConrad Meyer #include "ioat_hw.h"
51*e974f91cSConrad Meyer #include "ioat_internal.h"
52*e974f91cSConrad Meyer 
53*e974f91cSConrad Meyer static int ioat_probe(device_t device);
54*e974f91cSConrad Meyer static int ioat_attach(device_t device);
55*e974f91cSConrad Meyer static int ioat_detach(device_t device);
56*e974f91cSConrad Meyer static int ioat3_attach(device_t device);
57*e974f91cSConrad Meyer static int ioat_map_pci_bar(struct ioat_softc *ioat);
58*e974f91cSConrad Meyer static void ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg,
59*e974f91cSConrad Meyer     int error);
60*e974f91cSConrad Meyer static int ioat_interrupt_setup(struct ioat_softc *ioat);
61*e974f91cSConrad Meyer static void ioat_interrupt_handler(void *arg);
62*e974f91cSConrad Meyer static void ioat_process_events(struct ioat_softc *ioat);
63*e974f91cSConrad Meyer static inline uint32_t ioat_get_active(struct ioat_softc *ioat);
64*e974f91cSConrad Meyer static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat);
65*e974f91cSConrad Meyer static void ioat_free_ring_entry(struct ioat_softc *ioat,
66*e974f91cSConrad Meyer     struct ioat_descriptor *desc);
67*e974f91cSConrad Meyer static struct ioat_descriptor * ioat_alloc_ring_entry(struct ioat_softc *ioat);
68*e974f91cSConrad Meyer static int ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs);
69*e974f91cSConrad Meyer static struct ioat_descriptor * ioat_get_ring_entry(struct ioat_softc *ioat,
70*e974f91cSConrad Meyer     uint32_t index);
71*e974f91cSConrad Meyer static boolean_t resize_ring(struct ioat_softc *ioat, int order);
72*e974f91cSConrad Meyer static void ioat_timer_callback(void *arg);
73*e974f91cSConrad Meyer static void dump_descriptor(void *hw_desc);
74*e974f91cSConrad Meyer static void ioat_submit_single(struct ioat_softc *ioat);
75*e974f91cSConrad Meyer static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg,
76*e974f91cSConrad Meyer     int error);
77*e974f91cSConrad Meyer static int ioat_reset_hw(struct ioat_softc *ioat);
78*e974f91cSConrad Meyer static void ioat_setup_sysctl(device_t device);
79*e974f91cSConrad Meyer 
80*e974f91cSConrad Meyer MALLOC_DEFINE(M_IOAT, "ioat", "ioat driver memory allocations");
81*e974f91cSConrad Meyer SYSCTL_NODE(_hw, OID_AUTO, ioat, CTLFLAG_RD, 0, "ioat node");
82*e974f91cSConrad Meyer 
83*e974f91cSConrad Meyer static int g_force_legacy_interrupts;
84*e974f91cSConrad Meyer SYSCTL_INT(_hw_ioat, OID_AUTO, force_legacy_interrupts, CTLFLAG_RDTUN,
85*e974f91cSConrad Meyer     &g_force_legacy_interrupts, 0, "Set to non-zero to force MSI-X disabled");
86*e974f91cSConrad Meyer 
87*e974f91cSConrad Meyer static int g_ioat_debug_level = 0;
88*e974f91cSConrad Meyer SYSCTL_INT(_hw_ioat, OID_AUTO, debug_level, CTLFLAG_RWTUN, &g_ioat_debug_level,
89*e974f91cSConrad Meyer     0, "Set log level (0-3) for ioat(4). Higher is more verbose.");
90*e974f91cSConrad Meyer 
91*e974f91cSConrad Meyer /*
92*e974f91cSConrad Meyer  * OS <-> Driver interface structures
93*e974f91cSConrad Meyer  */
94*e974f91cSConrad Meyer static device_method_t ioat_pci_methods[] = {
95*e974f91cSConrad Meyer 	/* Device interface */
96*e974f91cSConrad Meyer 	DEVMETHOD(device_probe,     ioat_probe),
97*e974f91cSConrad Meyer 	DEVMETHOD(device_attach,    ioat_attach),
98*e974f91cSConrad Meyer 	DEVMETHOD(device_detach,    ioat_detach),
99*e974f91cSConrad Meyer 	{ 0, 0 }
100*e974f91cSConrad Meyer };
101*e974f91cSConrad Meyer 
102*e974f91cSConrad Meyer static driver_t ioat_pci_driver = {
103*e974f91cSConrad Meyer 	"ioat",
104*e974f91cSConrad Meyer 	ioat_pci_methods,
105*e974f91cSConrad Meyer 	sizeof(struct ioat_softc),
106*e974f91cSConrad Meyer };
107*e974f91cSConrad Meyer 
108*e974f91cSConrad Meyer static devclass_t ioat_devclass;
109*e974f91cSConrad Meyer DRIVER_MODULE(ioat, pci, ioat_pci_driver, ioat_devclass, 0, 0);
110*e974f91cSConrad Meyer 
111*e974f91cSConrad Meyer /*
112*e974f91cSConrad Meyer  * Private data structures
113*e974f91cSConrad Meyer  */
114*e974f91cSConrad Meyer static struct ioat_softc *ioat_channel[IOAT_MAX_CHANNELS];
115*e974f91cSConrad Meyer static int ioat_channel_index = 0;
116*e974f91cSConrad Meyer SYSCTL_INT(_hw_ioat, OID_AUTO, channels, CTLFLAG_RD, &ioat_channel_index, 0,
117*e974f91cSConrad Meyer     "Number of IOAT channels attached");
118*e974f91cSConrad Meyer 
119*e974f91cSConrad Meyer static struct _pcsid
120*e974f91cSConrad Meyer {
121*e974f91cSConrad Meyer 	u_int32_t   type;
122*e974f91cSConrad Meyer 	const char  *desc;
123*e974f91cSConrad Meyer } pci_ids[] = {
124*e974f91cSConrad Meyer 	{ 0x34308086, "TBG IOAT Ch0" },
125*e974f91cSConrad Meyer 	{ 0x34318086, "TBG IOAT Ch1" },
126*e974f91cSConrad Meyer 	{ 0x34328086, "TBG IOAT Ch2" },
127*e974f91cSConrad Meyer 	{ 0x34338086, "TBG IOAT Ch3" },
128*e974f91cSConrad Meyer 	{ 0x34298086, "TBG IOAT Ch4" },
129*e974f91cSConrad Meyer 	{ 0x342a8086, "TBG IOAT Ch5" },
130*e974f91cSConrad Meyer 	{ 0x342b8086, "TBG IOAT Ch6" },
131*e974f91cSConrad Meyer 	{ 0x342c8086, "TBG IOAT Ch7" },
132*e974f91cSConrad Meyer 
133*e974f91cSConrad Meyer 	{ 0x37108086, "JSF IOAT Ch0" },
134*e974f91cSConrad Meyer 	{ 0x37118086, "JSF IOAT Ch1" },
135*e974f91cSConrad Meyer 	{ 0x37128086, "JSF IOAT Ch2" },
136*e974f91cSConrad Meyer 	{ 0x37138086, "JSF IOAT Ch3" },
137*e974f91cSConrad Meyer 	{ 0x37148086, "JSF IOAT Ch4" },
138*e974f91cSConrad Meyer 	{ 0x37158086, "JSF IOAT Ch5" },
139*e974f91cSConrad Meyer 	{ 0x37168086, "JSF IOAT Ch6" },
140*e974f91cSConrad Meyer 	{ 0x37178086, "JSF IOAT Ch7" },
141*e974f91cSConrad Meyer 	{ 0x37188086, "JSF IOAT Ch0 (RAID)" },
142*e974f91cSConrad Meyer 	{ 0x37198086, "JSF IOAT Ch1 (RAID)" },
143*e974f91cSConrad Meyer 
144*e974f91cSConrad Meyer 	{ 0x3c208086, "SNB IOAT Ch0" },
145*e974f91cSConrad Meyer 	{ 0x3c218086, "SNB IOAT Ch1" },
146*e974f91cSConrad Meyer 	{ 0x3c228086, "SNB IOAT Ch2" },
147*e974f91cSConrad Meyer 	{ 0x3c238086, "SNB IOAT Ch3" },
148*e974f91cSConrad Meyer 	{ 0x3c248086, "SNB IOAT Ch4" },
149*e974f91cSConrad Meyer 	{ 0x3c258086, "SNB IOAT Ch5" },
150*e974f91cSConrad Meyer 	{ 0x3c268086, "SNB IOAT Ch6" },
151*e974f91cSConrad Meyer 	{ 0x3c278086, "SNB IOAT Ch7" },
152*e974f91cSConrad Meyer 	{ 0x3c2e8086, "SNB IOAT Ch0 (RAID)" },
153*e974f91cSConrad Meyer 	{ 0x3c2f8086, "SNB IOAT Ch1 (RAID)" },
154*e974f91cSConrad Meyer 
155*e974f91cSConrad Meyer 	{ 0x0e208086, "IVB IOAT Ch0" },
156*e974f91cSConrad Meyer 	{ 0x0e218086, "IVB IOAT Ch1" },
157*e974f91cSConrad Meyer 	{ 0x0e228086, "IVB IOAT Ch2" },
158*e974f91cSConrad Meyer 	{ 0x0e238086, "IVB IOAT Ch3" },
159*e974f91cSConrad Meyer 	{ 0x0e248086, "IVB IOAT Ch4" },
160*e974f91cSConrad Meyer 	{ 0x0e258086, "IVB IOAT Ch5" },
161*e974f91cSConrad Meyer 	{ 0x0e268086, "IVB IOAT Ch6" },
162*e974f91cSConrad Meyer 	{ 0x0e278086, "IVB IOAT Ch7" },
163*e974f91cSConrad Meyer 	{ 0x0e2e8086, "IVB IOAT Ch0 (RAID)" },
164*e974f91cSConrad Meyer 	{ 0x0e2f8086, "IVB IOAT Ch1 (RAID)" },
165*e974f91cSConrad Meyer 
166*e974f91cSConrad Meyer 	{ 0x2f208086, "HSW IOAT Ch0" },
167*e974f91cSConrad Meyer 	{ 0x2f218086, "HSW IOAT Ch1" },
168*e974f91cSConrad Meyer 	{ 0x2f228086, "HSW IOAT Ch2" },
169*e974f91cSConrad Meyer 	{ 0x2f238086, "HSW IOAT Ch3" },
170*e974f91cSConrad Meyer 	{ 0x2f248086, "HSW IOAT Ch4" },
171*e974f91cSConrad Meyer 	{ 0x2f258086, "HSW IOAT Ch5" },
172*e974f91cSConrad Meyer 	{ 0x2f268086, "HSW IOAT Ch6" },
173*e974f91cSConrad Meyer 	{ 0x2f278086, "HSW IOAT Ch7" },
174*e974f91cSConrad Meyer 	{ 0x2f2e8086, "HSW IOAT Ch0 (RAID)" },
175*e974f91cSConrad Meyer 	{ 0x2f2f8086, "HSW IOAT Ch1 (RAID)" },
176*e974f91cSConrad Meyer 
177*e974f91cSConrad Meyer 	{ 0x0c508086, "BWD IOAT Ch0" },
178*e974f91cSConrad Meyer 	{ 0x0c518086, "BWD IOAT Ch1" },
179*e974f91cSConrad Meyer 	{ 0x0c528086, "BWD IOAT Ch2" },
180*e974f91cSConrad Meyer 	{ 0x0c538086, "BWD IOAT Ch3" },
181*e974f91cSConrad Meyer 
182*e974f91cSConrad Meyer 	{ 0x6f508086, "BDXDE IOAT Ch0" },
183*e974f91cSConrad Meyer 	{ 0x6f518086, "BDXDE IOAT Ch1" },
184*e974f91cSConrad Meyer 	{ 0x6f528086, "BDXDE IOAT Ch2" },
185*e974f91cSConrad Meyer 	{ 0x6f538086, "BDXDE IOAT Ch3" },
186*e974f91cSConrad Meyer 
187*e974f91cSConrad Meyer 	{ 0x00000000, NULL           }
188*e974f91cSConrad Meyer };
189*e974f91cSConrad Meyer 
190*e974f91cSConrad Meyer /*
191*e974f91cSConrad Meyer  * OS <-> Driver linkage functions
192*e974f91cSConrad Meyer  */
193*e974f91cSConrad Meyer static int
194*e974f91cSConrad Meyer ioat_probe(device_t device)
195*e974f91cSConrad Meyer {
196*e974f91cSConrad Meyer 	struct _pcsid *ep;
197*e974f91cSConrad Meyer 	u_int32_t type;
198*e974f91cSConrad Meyer 
199*e974f91cSConrad Meyer 	type = pci_get_devid(device);
200*e974f91cSConrad Meyer 	for (ep = pci_ids; ep->type; ep++) {
201*e974f91cSConrad Meyer 		if (ep->type == type) {
202*e974f91cSConrad Meyer 			device_set_desc(device, ep->desc);
203*e974f91cSConrad Meyer 			return (0);
204*e974f91cSConrad Meyer 		}
205*e974f91cSConrad Meyer 	}
206*e974f91cSConrad Meyer 	return (ENXIO);
207*e974f91cSConrad Meyer }
208*e974f91cSConrad Meyer 
209*e974f91cSConrad Meyer static int
210*e974f91cSConrad Meyer ioat_attach(device_t device)
211*e974f91cSConrad Meyer {
212*e974f91cSConrad Meyer 	struct ioat_softc *ioat;
213*e974f91cSConrad Meyer 	int error;
214*e974f91cSConrad Meyer 
215*e974f91cSConrad Meyer 	ioat = DEVICE2SOFTC(device);
216*e974f91cSConrad Meyer 	ioat->device = device;
217*e974f91cSConrad Meyer 
218*e974f91cSConrad Meyer 	error = ioat_map_pci_bar(ioat);
219*e974f91cSConrad Meyer 	if (error != 0)
220*e974f91cSConrad Meyer 		goto err;
221*e974f91cSConrad Meyer 
222*e974f91cSConrad Meyer 	ioat->version = ioat_read_cbver(ioat);
223*e974f91cSConrad Meyer 	ioat_interrupt_setup(ioat);
224*e974f91cSConrad Meyer 
225*e974f91cSConrad Meyer 	if (ioat->version < IOAT_VER_3_0) {
226*e974f91cSConrad Meyer 		error = ENODEV;
227*e974f91cSConrad Meyer 		goto err;
228*e974f91cSConrad Meyer 	}
229*e974f91cSConrad Meyer 
230*e974f91cSConrad Meyer 	error = ioat3_attach(device);
231*e974f91cSConrad Meyer 	if (error != 0)
232*e974f91cSConrad Meyer 		goto err;
233*e974f91cSConrad Meyer 
234*e974f91cSConrad Meyer 	error = pci_enable_busmaster(device);
235*e974f91cSConrad Meyer 	if (error != 0)
236*e974f91cSConrad Meyer 		goto err;
237*e974f91cSConrad Meyer 
238*e974f91cSConrad Meyer 	ioat_channel[ioat_channel_index++] = ioat;
239*e974f91cSConrad Meyer 
240*e974f91cSConrad Meyer err:
241*e974f91cSConrad Meyer 	if (error != 0)
242*e974f91cSConrad Meyer 		ioat_detach(device);
243*e974f91cSConrad Meyer 	return (error);
244*e974f91cSConrad Meyer }
245*e974f91cSConrad Meyer 
246*e974f91cSConrad Meyer static int
247*e974f91cSConrad Meyer ioat_detach(device_t device)
248*e974f91cSConrad Meyer {
249*e974f91cSConrad Meyer 	struct ioat_softc *ioat;
250*e974f91cSConrad Meyer 	uint32_t i;
251*e974f91cSConrad Meyer 
252*e974f91cSConrad Meyer 	ioat = DEVICE2SOFTC(device);
253*e974f91cSConrad Meyer 	callout_drain(&ioat->timer);
254*e974f91cSConrad Meyer 
255*e974f91cSConrad Meyer 	pci_disable_busmaster(device);
256*e974f91cSConrad Meyer 
257*e974f91cSConrad Meyer 	if (ioat->pci_resource != NULL)
258*e974f91cSConrad Meyer 		bus_release_resource(device, SYS_RES_MEMORY,
259*e974f91cSConrad Meyer 		    ioat->pci_resource_id, ioat->pci_resource);
260*e974f91cSConrad Meyer 
261*e974f91cSConrad Meyer 	if (ioat->ring != NULL) {
262*e974f91cSConrad Meyer 		for (i = 0; i < (1 << ioat->ring_size_order); i++)
263*e974f91cSConrad Meyer 			ioat_free_ring_entry(ioat, ioat->ring[i]);
264*e974f91cSConrad Meyer 		free(ioat->ring, M_IOAT);
265*e974f91cSConrad Meyer 	}
266*e974f91cSConrad Meyer 
267*e974f91cSConrad Meyer 	if (ioat->comp_update != NULL) {
268*e974f91cSConrad Meyer 		bus_dmamap_unload(ioat->comp_update_tag, ioat->comp_update_map);
269*e974f91cSConrad Meyer 		bus_dmamem_free(ioat->comp_update_tag, ioat->comp_update,
270*e974f91cSConrad Meyer 		    ioat->comp_update_map);
271*e974f91cSConrad Meyer 		bus_dma_tag_destroy(ioat->comp_update_tag);
272*e974f91cSConrad Meyer 	}
273*e974f91cSConrad Meyer 
274*e974f91cSConrad Meyer 	bus_dma_tag_destroy(ioat->hw_desc_tag);
275*e974f91cSConrad Meyer 
276*e974f91cSConrad Meyer 	if (ioat->tag != NULL)
277*e974f91cSConrad Meyer 		bus_teardown_intr(device, ioat->res, ioat->tag);
278*e974f91cSConrad Meyer 
279*e974f91cSConrad Meyer 	if (ioat->res != NULL)
280*e974f91cSConrad Meyer 		bus_release_resource(device, SYS_RES_IRQ,
281*e974f91cSConrad Meyer 		    rman_get_rid(ioat->res), ioat->res);
282*e974f91cSConrad Meyer 
283*e974f91cSConrad Meyer 	pci_release_msi(device);
284*e974f91cSConrad Meyer 
285*e974f91cSConrad Meyer 	return (0);
286*e974f91cSConrad Meyer }
287*e974f91cSConrad Meyer 
288*e974f91cSConrad Meyer static int
289*e974f91cSConrad Meyer ioat3_selftest(struct ioat_softc *ioat)
290*e974f91cSConrad Meyer {
291*e974f91cSConrad Meyer 	uint64_t status;
292*e974f91cSConrad Meyer 	uint32_t chanerr;
293*e974f91cSConrad Meyer 	int i;
294*e974f91cSConrad Meyer 
295*e974f91cSConrad Meyer 	ioat_acquire(&ioat->dmaengine);
296*e974f91cSConrad Meyer 	ioat_null(&ioat->dmaengine, NULL, NULL, 0);
297*e974f91cSConrad Meyer 	ioat_release(&ioat->dmaengine);
298*e974f91cSConrad Meyer 
299*e974f91cSConrad Meyer 	for (i = 0; i < 100; i++) {
300*e974f91cSConrad Meyer 		DELAY(1);
301*e974f91cSConrad Meyer 		status = ioat_get_chansts(ioat);
302*e974f91cSConrad Meyer 		if (is_ioat_idle(status))
303*e974f91cSConrad Meyer 			return (0);
304*e974f91cSConrad Meyer 	}
305*e974f91cSConrad Meyer 
306*e974f91cSConrad Meyer 	chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
307*e974f91cSConrad Meyer 	ioat_log_message(0, "could not start channel: "
308*e974f91cSConrad Meyer 	    "status = %#jx error = %x\n", (uintmax_t)status, chanerr);
309*e974f91cSConrad Meyer 	return (ENXIO);
310*e974f91cSConrad Meyer }
311*e974f91cSConrad Meyer 
312*e974f91cSConrad Meyer /*
313*e974f91cSConrad Meyer  * Initialize Hardware
314*e974f91cSConrad Meyer  */
315*e974f91cSConrad Meyer static int
316*e974f91cSConrad Meyer ioat3_attach(device_t device)
317*e974f91cSConrad Meyer {
318*e974f91cSConrad Meyer 	struct ioat_softc *ioat;
319*e974f91cSConrad Meyer 	struct ioat_descriptor **ring;
320*e974f91cSConrad Meyer 	struct ioat_descriptor *next;
321*e974f91cSConrad Meyer 	struct ioat_dma_hw_descriptor *dma_hw_desc;
322*e974f91cSConrad Meyer 	uint32_t capabilities;
323*e974f91cSConrad Meyer 	int i, num_descriptors;
324*e974f91cSConrad Meyer 	int error;
325*e974f91cSConrad Meyer 	uint8_t xfercap;
326*e974f91cSConrad Meyer 
327*e974f91cSConrad Meyer 	error = 0;
328*e974f91cSConrad Meyer 	ioat = DEVICE2SOFTC(device);
329*e974f91cSConrad Meyer 	capabilities = ioat_read_dmacapability(ioat);
330*e974f91cSConrad Meyer 
331*e974f91cSConrad Meyer 	xfercap = ioat_read_xfercap(ioat);
332*e974f91cSConrad Meyer 
333*e974f91cSConrad Meyer 	/* Only bits [4:0] are valid. */
334*e974f91cSConrad Meyer 	xfercap &= 0x1f;
335*e974f91cSConrad Meyer 	ioat->max_xfer_size = 1 << xfercap;
336*e974f91cSConrad Meyer 
337*e974f91cSConrad Meyer 	/* TODO: need to check DCA here if we ever do XOR/PQ */
338*e974f91cSConrad Meyer 
339*e974f91cSConrad Meyer 	mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF);
340*e974f91cSConrad Meyer 	mtx_init(&ioat->cleanup_lock, "ioat_process_events", NULL, MTX_DEF);
341*e974f91cSConrad Meyer 	callout_init(&ioat->timer, CALLOUT_MPSAFE);
342*e974f91cSConrad Meyer 
343*e974f91cSConrad Meyer 	ioat->is_resize_pending = FALSE;
344*e974f91cSConrad Meyer 	ioat->is_completion_pending = FALSE;
345*e974f91cSConrad Meyer 	ioat->is_reset_pending = FALSE;
346*e974f91cSConrad Meyer 	ioat->is_channel_running = FALSE;
347*e974f91cSConrad Meyer 	ioat->is_waiting_for_ack = FALSE;
348*e974f91cSConrad Meyer 
349*e974f91cSConrad Meyer 	bus_dma_tag_create(bus_get_dma_tag(ioat->device), sizeof(uint64_t), 0x0,
350*e974f91cSConrad Meyer 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
351*e974f91cSConrad Meyer 	    sizeof(uint64_t), 1, sizeof(uint64_t), 0, NULL, NULL,
352*e974f91cSConrad Meyer 	    &ioat->comp_update_tag);
353*e974f91cSConrad Meyer 
354*e974f91cSConrad Meyer 	error = bus_dmamem_alloc(ioat->comp_update_tag,
355*e974f91cSConrad Meyer 	    (void **)&ioat->comp_update, BUS_DMA_ZERO, &ioat->comp_update_map);
356*e974f91cSConrad Meyer 	if (ioat->comp_update == NULL)
357*e974f91cSConrad Meyer 		return (ENOMEM);
358*e974f91cSConrad Meyer 
359*e974f91cSConrad Meyer 	error = bus_dmamap_load(ioat->comp_update_tag, ioat->comp_update_map,
360*e974f91cSConrad Meyer 	    ioat->comp_update, sizeof(uint64_t), ioat_comp_update_map, ioat,
361*e974f91cSConrad Meyer 	    0);
362*e974f91cSConrad Meyer 	if (error != 0)
363*e974f91cSConrad Meyer 		return (error);
364*e974f91cSConrad Meyer 
365*e974f91cSConrad Meyer 	ioat->ring_size_order = IOAT_MIN_ORDER;
366*e974f91cSConrad Meyer 
367*e974f91cSConrad Meyer 	num_descriptors = 1 << ioat->ring_size_order;
368*e974f91cSConrad Meyer 
369*e974f91cSConrad Meyer 	bus_dma_tag_create(bus_get_dma_tag(ioat->device), 0x40, 0x0,
370*e974f91cSConrad Meyer 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
371*e974f91cSConrad Meyer 	    sizeof(struct ioat_dma_hw_descriptor), 1,
372*e974f91cSConrad Meyer 	    sizeof(struct ioat_dma_hw_descriptor), 0, NULL, NULL,
373*e974f91cSConrad Meyer 	    &ioat->hw_desc_tag);
374*e974f91cSConrad Meyer 
375*e974f91cSConrad Meyer 	ioat->ring = malloc(num_descriptors * sizeof(*ring), M_IOAT,
376*e974f91cSConrad Meyer 	    M_ZERO | M_NOWAIT);
377*e974f91cSConrad Meyer 	if (ioat->ring == NULL)
378*e974f91cSConrad Meyer 		return (ENOMEM);
379*e974f91cSConrad Meyer 
380*e974f91cSConrad Meyer 	ring = ioat->ring;
381*e974f91cSConrad Meyer 	for (i = 0; i < num_descriptors; i++) {
382*e974f91cSConrad Meyer 		ring[i] = ioat_alloc_ring_entry(ioat);
383*e974f91cSConrad Meyer 		if (ring[i] == NULL)
384*e974f91cSConrad Meyer 			return (ENOMEM);
385*e974f91cSConrad Meyer 
386*e974f91cSConrad Meyer 		ring[i]->id = i;
387*e974f91cSConrad Meyer 	}
388*e974f91cSConrad Meyer 
389*e974f91cSConrad Meyer 	for (i = 0; i < num_descriptors - 1; i++) {
390*e974f91cSConrad Meyer 		next = ring[i + 1];
391*e974f91cSConrad Meyer 		dma_hw_desc = ring[i]->u.dma;
392*e974f91cSConrad Meyer 
393*e974f91cSConrad Meyer 		dma_hw_desc->next = next->hw_desc_bus_addr;
394*e974f91cSConrad Meyer 	}
395*e974f91cSConrad Meyer 
396*e974f91cSConrad Meyer 	ring[i]->u.dma->next = ring[0]->hw_desc_bus_addr;
397*e974f91cSConrad Meyer 
398*e974f91cSConrad Meyer 	ioat->head = 0;
399*e974f91cSConrad Meyer 	ioat->tail = 0;
400*e974f91cSConrad Meyer 	ioat->last_seen = 0;
401*e974f91cSConrad Meyer 
402*e974f91cSConrad Meyer 	error = ioat_reset_hw(ioat);
403*e974f91cSConrad Meyer 	if (error != 0)
404*e974f91cSConrad Meyer 		return (error);
405*e974f91cSConrad Meyer 
406*e974f91cSConrad Meyer 	ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
407*e974f91cSConrad Meyer 	ioat_write_chancmp(ioat, ioat->comp_update_bus_addr);
408*e974f91cSConrad Meyer 	ioat_write_chainaddr(ioat, ring[0]->hw_desc_bus_addr);
409*e974f91cSConrad Meyer 
410*e974f91cSConrad Meyer 	error = ioat3_selftest(ioat);
411*e974f91cSConrad Meyer 	if (error != 0)
412*e974f91cSConrad Meyer 		return (error);
413*e974f91cSConrad Meyer 
414*e974f91cSConrad Meyer 	ioat_process_events(ioat);
415*e974f91cSConrad Meyer 	ioat_setup_sysctl(device);
416*e974f91cSConrad Meyer 	return (0);
417*e974f91cSConrad Meyer }
418*e974f91cSConrad Meyer 
419*e974f91cSConrad Meyer static int
420*e974f91cSConrad Meyer ioat_map_pci_bar(struct ioat_softc *ioat)
421*e974f91cSConrad Meyer {
422*e974f91cSConrad Meyer 
423*e974f91cSConrad Meyer 	ioat->pci_resource_id = PCIR_BAR(0);
424*e974f91cSConrad Meyer 	ioat->pci_resource = bus_alloc_resource(ioat->device, SYS_RES_MEMORY,
425*e974f91cSConrad Meyer 	    &ioat->pci_resource_id, 0, ~0, 1, RF_ACTIVE);
426*e974f91cSConrad Meyer 
427*e974f91cSConrad Meyer 	if (ioat->pci_resource == NULL) {
428*e974f91cSConrad Meyer 		ioat_log_message(0, "unable to allocate pci resource\n");
429*e974f91cSConrad Meyer 		return (ENODEV);
430*e974f91cSConrad Meyer 	}
431*e974f91cSConrad Meyer 
432*e974f91cSConrad Meyer 	ioat->pci_bus_tag = rman_get_bustag(ioat->pci_resource);
433*e974f91cSConrad Meyer 	ioat->pci_bus_handle = rman_get_bushandle(ioat->pci_resource);
434*e974f91cSConrad Meyer 	return (0);
435*e974f91cSConrad Meyer }
436*e974f91cSConrad Meyer 
437*e974f91cSConrad Meyer static void
438*e974f91cSConrad Meyer ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
439*e974f91cSConrad Meyer {
440*e974f91cSConrad Meyer 	struct ioat_softc *ioat = arg;
441*e974f91cSConrad Meyer 
442*e974f91cSConrad Meyer 	ioat->comp_update_bus_addr = seg[0].ds_addr;
443*e974f91cSConrad Meyer }
444*e974f91cSConrad Meyer 
445*e974f91cSConrad Meyer static void
446*e974f91cSConrad Meyer ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
447*e974f91cSConrad Meyer {
448*e974f91cSConrad Meyer 	bus_addr_t *baddr;
449*e974f91cSConrad Meyer 
450*e974f91cSConrad Meyer 	baddr = arg;
451*e974f91cSConrad Meyer 	*baddr = segs->ds_addr;
452*e974f91cSConrad Meyer }
453*e974f91cSConrad Meyer 
454*e974f91cSConrad Meyer /*
455*e974f91cSConrad Meyer  * Interrupt setup and handlers
456*e974f91cSConrad Meyer  */
457*e974f91cSConrad Meyer static int
458*e974f91cSConrad Meyer ioat_interrupt_setup(struct ioat_softc *ioat)
459*e974f91cSConrad Meyer {
460*e974f91cSConrad Meyer 	uint32_t num_vectors;
461*e974f91cSConrad Meyer 	int error;
462*e974f91cSConrad Meyer 	boolean_t use_msix;
463*e974f91cSConrad Meyer 	boolean_t force_legacy_interrupts;
464*e974f91cSConrad Meyer 
465*e974f91cSConrad Meyer 	use_msix = FALSE;
466*e974f91cSConrad Meyer 	force_legacy_interrupts = FALSE;
467*e974f91cSConrad Meyer 
468*e974f91cSConrad Meyer 	if (!g_force_legacy_interrupts && pci_msix_count(ioat->device) >= 1) {
469*e974f91cSConrad Meyer 		num_vectors = 1;
470*e974f91cSConrad Meyer 		pci_alloc_msix(ioat->device, &num_vectors);
471*e974f91cSConrad Meyer 		if (num_vectors == 1)
472*e974f91cSConrad Meyer 			use_msix = TRUE;
473*e974f91cSConrad Meyer 	}
474*e974f91cSConrad Meyer 
475*e974f91cSConrad Meyer 	if (use_msix) {
476*e974f91cSConrad Meyer 		ioat->rid = 1;
477*e974f91cSConrad Meyer 		ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ,
478*e974f91cSConrad Meyer 		    &ioat->rid, RF_ACTIVE);
479*e974f91cSConrad Meyer 	} else {
480*e974f91cSConrad Meyer 		ioat->rid = 0;
481*e974f91cSConrad Meyer 		ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ,
482*e974f91cSConrad Meyer 		    &ioat->rid, RF_SHAREABLE | RF_ACTIVE);
483*e974f91cSConrad Meyer 	}
484*e974f91cSConrad Meyer 	if (ioat->res == NULL) {
485*e974f91cSConrad Meyer 		ioat_log_message(0, "bus_alloc_resource failed\n");
486*e974f91cSConrad Meyer 		return (ENOMEM);
487*e974f91cSConrad Meyer 	}
488*e974f91cSConrad Meyer 
489*e974f91cSConrad Meyer 	ioat->tag = NULL;
490*e974f91cSConrad Meyer 	error = bus_setup_intr(ioat->device, ioat->res, INTR_MPSAFE |
491*e974f91cSConrad Meyer 	    INTR_TYPE_MISC, NULL, ioat_interrupt_handler, ioat, &ioat->tag);
492*e974f91cSConrad Meyer 	if (error != 0) {
493*e974f91cSConrad Meyer 		ioat_log_message(0, "bus_setup_intr failed\n");
494*e974f91cSConrad Meyer 		return (error);
495*e974f91cSConrad Meyer 	}
496*e974f91cSConrad Meyer 
497*e974f91cSConrad Meyer 	ioat_write_intrctrl(ioat, IOAT_INTRCTRL_MASTER_INT_EN);
498*e974f91cSConrad Meyer 	return (0);
499*e974f91cSConrad Meyer }
500*e974f91cSConrad Meyer 
501*e974f91cSConrad Meyer static void
502*e974f91cSConrad Meyer ioat_interrupt_handler(void *arg)
503*e974f91cSConrad Meyer {
504*e974f91cSConrad Meyer 	struct ioat_softc *ioat = arg;
505*e974f91cSConrad Meyer 
506*e974f91cSConrad Meyer 	ioat_process_events(ioat);
507*e974f91cSConrad Meyer }
508*e974f91cSConrad Meyer 
509*e974f91cSConrad Meyer static void
510*e974f91cSConrad Meyer ioat_process_events(struct ioat_softc *ioat)
511*e974f91cSConrad Meyer {
512*e974f91cSConrad Meyer 	struct ioat_descriptor *desc;
513*e974f91cSConrad Meyer 	struct bus_dmadesc *dmadesc;
514*e974f91cSConrad Meyer 	uint64_t comp_update, status;
515*e974f91cSConrad Meyer 	uint32_t completed;
516*e974f91cSConrad Meyer 
517*e974f91cSConrad Meyer 	mtx_lock(&ioat->cleanup_lock);
518*e974f91cSConrad Meyer 
519*e974f91cSConrad Meyer 	completed = 0;
520*e974f91cSConrad Meyer 	comp_update = *ioat->comp_update;
521*e974f91cSConrad Meyer 	status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK;
522*e974f91cSConrad Meyer 
523*e974f91cSConrad Meyer 	ioat_log_message(3, "%s\n", __func__);
524*e974f91cSConrad Meyer 
525*e974f91cSConrad Meyer 	if (status == ioat->last_seen) {
526*e974f91cSConrad Meyer 	 	mtx_unlock(&ioat->cleanup_lock);
527*e974f91cSConrad Meyer 		return;
528*e974f91cSConrad Meyer 	}
529*e974f91cSConrad Meyer 
530*e974f91cSConrad Meyer 	while (1) {
531*e974f91cSConrad Meyer 		desc = ioat_get_ring_entry(ioat, ioat->tail);
532*e974f91cSConrad Meyer 		dmadesc = &desc->bus_dmadesc;
533*e974f91cSConrad Meyer 		ioat_log_message(3, "completing desc %d\n", ioat->tail);
534*e974f91cSConrad Meyer 
535*e974f91cSConrad Meyer 		if (dmadesc->callback_fn)
536*e974f91cSConrad Meyer 			(*dmadesc->callback_fn)(dmadesc->callback_arg);
537*e974f91cSConrad Meyer 
538*e974f91cSConrad Meyer 		ioat->tail++;
539*e974f91cSConrad Meyer 		if (desc->hw_desc_bus_addr == status)
540*e974f91cSConrad Meyer 			break;
541*e974f91cSConrad Meyer 	}
542*e974f91cSConrad Meyer 
543*e974f91cSConrad Meyer 	ioat->last_seen = desc->hw_desc_bus_addr;
544*e974f91cSConrad Meyer 
545*e974f91cSConrad Meyer 	if (ioat->head == ioat->tail) {
546*e974f91cSConrad Meyer 		ioat->is_completion_pending = FALSE;
547*e974f91cSConrad Meyer 		callout_reset(&ioat->timer, 5 * hz, ioat_timer_callback, ioat);
548*e974f91cSConrad Meyer 	}
549*e974f91cSConrad Meyer 
550*e974f91cSConrad Meyer 	ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
551*e974f91cSConrad Meyer 	mtx_unlock(&ioat->cleanup_lock);
552*e974f91cSConrad Meyer }
553*e974f91cSConrad Meyer 
554*e974f91cSConrad Meyer /*
555*e974f91cSConrad Meyer  * User API functions
556*e974f91cSConrad Meyer  */
557*e974f91cSConrad Meyer bus_dmaengine_t
558*e974f91cSConrad Meyer ioat_get_dmaengine(uint32_t index)
559*e974f91cSConrad Meyer {
560*e974f91cSConrad Meyer 
561*e974f91cSConrad Meyer 	if (index < ioat_channel_index)
562*e974f91cSConrad Meyer 		return (&ioat_channel[index]->dmaengine);
563*e974f91cSConrad Meyer 	return (NULL);
564*e974f91cSConrad Meyer }
565*e974f91cSConrad Meyer 
566*e974f91cSConrad Meyer void
567*e974f91cSConrad Meyer ioat_acquire(bus_dmaengine_t dmaengine)
568*e974f91cSConrad Meyer {
569*e974f91cSConrad Meyer 	struct ioat_softc *ioat;
570*e974f91cSConrad Meyer 
571*e974f91cSConrad Meyer 	ioat = to_ioat_softc(dmaengine);
572*e974f91cSConrad Meyer 	mtx_lock(&ioat->submit_lock);
573*e974f91cSConrad Meyer 	ioat_log_message(3, "%s\n", __func__);
574*e974f91cSConrad Meyer }
575*e974f91cSConrad Meyer 
576*e974f91cSConrad Meyer void
577*e974f91cSConrad Meyer ioat_release(bus_dmaengine_t dmaengine)
578*e974f91cSConrad Meyer {
579*e974f91cSConrad Meyer 	struct ioat_softc *ioat;
580*e974f91cSConrad Meyer 
581*e974f91cSConrad Meyer 	ioat_log_message(3, "%s\n", __func__);
582*e974f91cSConrad Meyer 	ioat = to_ioat_softc(dmaengine);
583*e974f91cSConrad Meyer 	ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, (uint16_t)ioat->head);
584*e974f91cSConrad Meyer 	mtx_unlock(&ioat->submit_lock);
585*e974f91cSConrad Meyer }
586*e974f91cSConrad Meyer 
587*e974f91cSConrad Meyer struct bus_dmadesc *
588*e974f91cSConrad Meyer ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_callback_t callback_fn,
589*e974f91cSConrad Meyer     void *callback_arg, uint32_t flags)
590*e974f91cSConrad Meyer {
591*e974f91cSConrad Meyer 	struct ioat_softc *ioat;
592*e974f91cSConrad Meyer 	struct ioat_descriptor *desc;
593*e974f91cSConrad Meyer 	struct ioat_dma_hw_descriptor *hw_desc;
594*e974f91cSConrad Meyer 
595*e974f91cSConrad Meyer 	KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x",
596*e974f91cSConrad Meyer 		flags & ~DMA_ALL_FLAGS));
597*e974f91cSConrad Meyer 
598*e974f91cSConrad Meyer 	ioat = to_ioat_softc(dmaengine);
599*e974f91cSConrad Meyer 
600*e974f91cSConrad Meyer 	if (ioat_reserve_space_and_lock(ioat, 1) != 0)
601*e974f91cSConrad Meyer 		return (NULL);
602*e974f91cSConrad Meyer 
603*e974f91cSConrad Meyer 	ioat_log_message(3, "%s\n", __func__);
604*e974f91cSConrad Meyer 
605*e974f91cSConrad Meyer 	desc = ioat_get_ring_entry(ioat, ioat->head);
606*e974f91cSConrad Meyer 	hw_desc = desc->u.dma;
607*e974f91cSConrad Meyer 
608*e974f91cSConrad Meyer 	hw_desc->u.control_raw = 0;
609*e974f91cSConrad Meyer 	hw_desc->u.control.null = 1;
610*e974f91cSConrad Meyer 	hw_desc->u.control.completion_update = 1;
611*e974f91cSConrad Meyer 
612*e974f91cSConrad Meyer 	if ((flags & DMA_INT_EN) != 0)
613*e974f91cSConrad Meyer 		hw_desc->u.control.int_enable = 1;
614*e974f91cSConrad Meyer 
615*e974f91cSConrad Meyer 	hw_desc->size = 8;
616*e974f91cSConrad Meyer 	hw_desc->src_addr = 0;
617*e974f91cSConrad Meyer 	hw_desc->dest_addr = 0;
618*e974f91cSConrad Meyer 
619*e974f91cSConrad Meyer 	desc->bus_dmadesc.callback_fn = callback_fn;
620*e974f91cSConrad Meyer 	desc->bus_dmadesc.callback_arg = callback_arg;
621*e974f91cSConrad Meyer 
622*e974f91cSConrad Meyer 	ioat_submit_single(ioat);
623*e974f91cSConrad Meyer 	return (&desc->bus_dmadesc);
624*e974f91cSConrad Meyer }
625*e974f91cSConrad Meyer 
626*e974f91cSConrad Meyer struct bus_dmadesc *
627*e974f91cSConrad Meyer ioat_copy(bus_dmaengine_t dmaengine, bus_addr_t dst,
628*e974f91cSConrad Meyer     bus_addr_t src, bus_size_t len, bus_dmaengine_callback_t callback_fn,
629*e974f91cSConrad Meyer     void *callback_arg, uint32_t flags)
630*e974f91cSConrad Meyer {
631*e974f91cSConrad Meyer 	struct ioat_descriptor *desc;
632*e974f91cSConrad Meyer 	struct ioat_dma_hw_descriptor *hw_desc;
633*e974f91cSConrad Meyer 	struct ioat_softc *ioat;
634*e974f91cSConrad Meyer 
635*e974f91cSConrad Meyer 	KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x",
636*e974f91cSConrad Meyer 		flags & ~DMA_ALL_FLAGS));
637*e974f91cSConrad Meyer 
638*e974f91cSConrad Meyer 	ioat = to_ioat_softc(dmaengine);
639*e974f91cSConrad Meyer 
640*e974f91cSConrad Meyer 	if (len > ioat->max_xfer_size) {
641*e974f91cSConrad Meyer 		ioat_log_message(0, "%s: max_xfer_size = %d, requested = %d\n",
642*e974f91cSConrad Meyer 		    __func__, ioat->max_xfer_size, (int)len);
643*e974f91cSConrad Meyer 		return (NULL);
644*e974f91cSConrad Meyer 	}
645*e974f91cSConrad Meyer 
646*e974f91cSConrad Meyer 	if (ioat_reserve_space_and_lock(ioat, 1) != 0)
647*e974f91cSConrad Meyer 		return (NULL);
648*e974f91cSConrad Meyer 
649*e974f91cSConrad Meyer 	ioat_log_message(3, "%s\n", __func__);
650*e974f91cSConrad Meyer 
651*e974f91cSConrad Meyer 	desc = ioat_get_ring_entry(ioat, ioat->head);
652*e974f91cSConrad Meyer 	hw_desc = desc->u.dma;
653*e974f91cSConrad Meyer 
654*e974f91cSConrad Meyer 	hw_desc->u.control_raw = 0;
655*e974f91cSConrad Meyer 	hw_desc->u.control.completion_update = 1;
656*e974f91cSConrad Meyer 
657*e974f91cSConrad Meyer 	if ((flags & DMA_INT_EN) != 0)
658*e974f91cSConrad Meyer 		hw_desc->u.control.int_enable = 1;
659*e974f91cSConrad Meyer 
660*e974f91cSConrad Meyer 	hw_desc->size = len;
661*e974f91cSConrad Meyer 	hw_desc->src_addr = src;
662*e974f91cSConrad Meyer 	hw_desc->dest_addr = dst;
663*e974f91cSConrad Meyer 
664*e974f91cSConrad Meyer 	if (g_ioat_debug_level >= 3)
665*e974f91cSConrad Meyer 		dump_descriptor(hw_desc);
666*e974f91cSConrad Meyer 
667*e974f91cSConrad Meyer 	desc->bus_dmadesc.callback_fn = callback_fn;
668*e974f91cSConrad Meyer 	desc->bus_dmadesc.callback_arg = callback_arg;
669*e974f91cSConrad Meyer 
670*e974f91cSConrad Meyer 	ioat_submit_single(ioat);
671*e974f91cSConrad Meyer 	return (&desc->bus_dmadesc);
672*e974f91cSConrad Meyer }
673*e974f91cSConrad Meyer 
674*e974f91cSConrad Meyer /*
675*e974f91cSConrad Meyer  * Ring Management
676*e974f91cSConrad Meyer  */
677*e974f91cSConrad Meyer static inline uint32_t
678*e974f91cSConrad Meyer ioat_get_active(struct ioat_softc *ioat)
679*e974f91cSConrad Meyer {
680*e974f91cSConrad Meyer 
681*e974f91cSConrad Meyer 	return ((ioat->head - ioat->tail) & ((1 << ioat->ring_size_order) - 1));
682*e974f91cSConrad Meyer }
683*e974f91cSConrad Meyer 
684*e974f91cSConrad Meyer static inline uint32_t
685*e974f91cSConrad Meyer ioat_get_ring_space(struct ioat_softc *ioat)
686*e974f91cSConrad Meyer {
687*e974f91cSConrad Meyer 
688*e974f91cSConrad Meyer 	return ((1 << ioat->ring_size_order) - ioat_get_active(ioat) - 1);
689*e974f91cSConrad Meyer }
690*e974f91cSConrad Meyer 
691*e974f91cSConrad Meyer static struct ioat_descriptor *
692*e974f91cSConrad Meyer ioat_alloc_ring_entry(struct ioat_softc *ioat)
693*e974f91cSConrad Meyer {
694*e974f91cSConrad Meyer 	struct ioat_dma_hw_descriptor *hw_desc;
695*e974f91cSConrad Meyer 	struct ioat_descriptor *desc;
696*e974f91cSConrad Meyer 
697*e974f91cSConrad Meyer 	desc = malloc(sizeof(struct ioat_descriptor), M_IOAT, M_NOWAIT);
698*e974f91cSConrad Meyer 	if (desc == NULL)
699*e974f91cSConrad Meyer 		return (NULL);
700*e974f91cSConrad Meyer 
701*e974f91cSConrad Meyer 	bus_dmamem_alloc(ioat->hw_desc_tag, (void **)&hw_desc, BUS_DMA_ZERO,
702*e974f91cSConrad Meyer 	    &ioat->hw_desc_map);
703*e974f91cSConrad Meyer 	if (hw_desc == NULL) {
704*e974f91cSConrad Meyer 		free(desc, M_IOAT);
705*e974f91cSConrad Meyer 		return (NULL);
706*e974f91cSConrad Meyer 	}
707*e974f91cSConrad Meyer 
708*e974f91cSConrad Meyer 	bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc,
709*e974f91cSConrad Meyer 	    sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr, 0);
710*e974f91cSConrad Meyer 
711*e974f91cSConrad Meyer 	desc->u.dma = hw_desc;
712*e974f91cSConrad Meyer 	return (desc);
713*e974f91cSConrad Meyer }
714*e974f91cSConrad Meyer 
715*e974f91cSConrad Meyer static void
716*e974f91cSConrad Meyer ioat_free_ring_entry(struct ioat_softc *ioat, struct ioat_descriptor *desc)
717*e974f91cSConrad Meyer {
718*e974f91cSConrad Meyer 
719*e974f91cSConrad Meyer 	if (desc == NULL)
720*e974f91cSConrad Meyer 		return;
721*e974f91cSConrad Meyer 
722*e974f91cSConrad Meyer 	if (desc->u.dma)
723*e974f91cSConrad Meyer 		bus_dmamem_free(ioat->hw_desc_tag, desc->u.dma,
724*e974f91cSConrad Meyer 		    ioat->hw_desc_map);
725*e974f91cSConrad Meyer 	free(desc, M_IOAT);
726*e974f91cSConrad Meyer }
727*e974f91cSConrad Meyer 
728*e974f91cSConrad Meyer static int
729*e974f91cSConrad Meyer ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs)
730*e974f91cSConrad Meyer {
731*e974f91cSConrad Meyer 	boolean_t retry;
732*e974f91cSConrad Meyer 
733*e974f91cSConrad Meyer 	while (1) {
734*e974f91cSConrad Meyer 		if (ioat_get_ring_space(ioat) >= num_descs)
735*e974f91cSConrad Meyer 			return (0);
736*e974f91cSConrad Meyer 
737*e974f91cSConrad Meyer 		mtx_lock(&ioat->cleanup_lock);
738*e974f91cSConrad Meyer 		retry = resize_ring(ioat, ioat->ring_size_order + 1);
739*e974f91cSConrad Meyer 		mtx_unlock(&ioat->cleanup_lock);
740*e974f91cSConrad Meyer 
741*e974f91cSConrad Meyer 		if (!retry)
742*e974f91cSConrad Meyer 			return (ENOMEM);
743*e974f91cSConrad Meyer 	}
744*e974f91cSConrad Meyer }
745*e974f91cSConrad Meyer 
746*e974f91cSConrad Meyer static struct ioat_descriptor *
747*e974f91cSConrad Meyer ioat_get_ring_entry(struct ioat_softc *ioat, uint32_t index)
748*e974f91cSConrad Meyer {
749*e974f91cSConrad Meyer 
750*e974f91cSConrad Meyer 	return (ioat->ring[index % (1 << ioat->ring_size_order)]);
751*e974f91cSConrad Meyer }
752*e974f91cSConrad Meyer 
753*e974f91cSConrad Meyer static boolean_t
754*e974f91cSConrad Meyer resize_ring(struct ioat_softc *ioat, int order)
755*e974f91cSConrad Meyer {
756*e974f91cSConrad Meyer 	struct ioat_descriptor **ring;
757*e974f91cSConrad Meyer 	struct ioat_descriptor *next;
758*e974f91cSConrad Meyer 	struct ioat_dma_hw_descriptor *hw;
759*e974f91cSConrad Meyer 	struct ioat_descriptor *ent;
760*e974f91cSConrad Meyer 	uint32_t current_size, active, new_size, i, new_idx, current_idx;
761*e974f91cSConrad Meyer 	uint32_t new_idx2;
762*e974f91cSConrad Meyer 
763*e974f91cSConrad Meyer 	current_size = 1 << ioat->ring_size_order;
764*e974f91cSConrad Meyer 	active = (ioat->head - ioat->tail) & (current_size - 1);
765*e974f91cSConrad Meyer 	new_size = 1 << order;
766*e974f91cSConrad Meyer 
767*e974f91cSConrad Meyer 	if (order > IOAT_MAX_ORDER)
768*e974f91cSConrad Meyer 		return (FALSE);
769*e974f91cSConrad Meyer 
770*e974f91cSConrad Meyer 	/*
771*e974f91cSConrad Meyer 	 * when shrinking, verify that we can hold the current active
772*e974f91cSConrad Meyer 	 * set in the new ring
773*e974f91cSConrad Meyer 	 */
774*e974f91cSConrad Meyer 	if (active >= new_size)
775*e974f91cSConrad Meyer 		return (FALSE);
776*e974f91cSConrad Meyer 
777*e974f91cSConrad Meyer 	/* allocate the array to hold the software ring */
778*e974f91cSConrad Meyer 	ring = malloc(new_size * sizeof(*ring), M_IOAT, M_ZERO | M_NOWAIT);
779*e974f91cSConrad Meyer 	if (ring == NULL)
780*e974f91cSConrad Meyer 		return (FALSE);
781*e974f91cSConrad Meyer 
782*e974f91cSConrad Meyer 	ioat_log_message(2, "ring resize: new: %d old: %d\n",
783*e974f91cSConrad Meyer 	    new_size, current_size);
784*e974f91cSConrad Meyer 
785*e974f91cSConrad Meyer 	/* allocate/trim descriptors as needed */
786*e974f91cSConrad Meyer 	if (new_size > current_size) {
787*e974f91cSConrad Meyer 		/* copy current descriptors to the new ring */
788*e974f91cSConrad Meyer 		for (i = 0; i < current_size; i++) {
789*e974f91cSConrad Meyer 			current_idx = (ioat->tail + i) & (current_size - 1);
790*e974f91cSConrad Meyer 			new_idx = (ioat->tail + i) & (new_size - 1);
791*e974f91cSConrad Meyer 
792*e974f91cSConrad Meyer 			ring[new_idx] = ioat->ring[current_idx];
793*e974f91cSConrad Meyer 			ring[new_idx]->id = new_idx;
794*e974f91cSConrad Meyer 		}
795*e974f91cSConrad Meyer 
796*e974f91cSConrad Meyer 		/* add new descriptors to the ring */
797*e974f91cSConrad Meyer 		for (i = current_size; i < new_size; i++) {
798*e974f91cSConrad Meyer 			new_idx = (ioat->tail + i) & (new_size - 1);
799*e974f91cSConrad Meyer 
800*e974f91cSConrad Meyer 			ring[new_idx] = ioat_alloc_ring_entry(ioat);
801*e974f91cSConrad Meyer 			if (!ring[new_idx]) {
802*e974f91cSConrad Meyer 				while (i--) {
803*e974f91cSConrad Meyer 					new_idx2 = (ioat->tail + i) &
804*e974f91cSConrad Meyer 					    (new_size - 1);
805*e974f91cSConrad Meyer 
806*e974f91cSConrad Meyer 					ioat_free_ring_entry(ioat,
807*e974f91cSConrad Meyer 					    ring[new_idx2]);
808*e974f91cSConrad Meyer 				}
809*e974f91cSConrad Meyer 				free(ring, M_IOAT);
810*e974f91cSConrad Meyer 				return (FALSE);
811*e974f91cSConrad Meyer 			}
812*e974f91cSConrad Meyer 			ring[new_idx]->id = new_idx;
813*e974f91cSConrad Meyer 		}
814*e974f91cSConrad Meyer 
815*e974f91cSConrad Meyer 		for (i = current_size - 1; i < new_size; i++) {
816*e974f91cSConrad Meyer 			new_idx = (ioat->tail + i) & (new_size - 1);
817*e974f91cSConrad Meyer 			next = ring[(new_idx + 1) & (new_size - 1)];
818*e974f91cSConrad Meyer 			hw = ring[new_idx]->u.dma;
819*e974f91cSConrad Meyer 
820*e974f91cSConrad Meyer 			hw->next = next->hw_desc_bus_addr;
821*e974f91cSConrad Meyer 		}
822*e974f91cSConrad Meyer 	} else {
823*e974f91cSConrad Meyer 		/*
824*e974f91cSConrad Meyer 		 * copy current descriptors to the new ring, dropping the
825*e974f91cSConrad Meyer 		 * removed descriptors
826*e974f91cSConrad Meyer 		 */
827*e974f91cSConrad Meyer 		for (i = 0; i < new_size; i++) {
828*e974f91cSConrad Meyer 			current_idx = (ioat->tail + i) & (current_size - 1);
829*e974f91cSConrad Meyer 			new_idx = (ioat->tail + i) & (new_size - 1);
830*e974f91cSConrad Meyer 
831*e974f91cSConrad Meyer 			ring[new_idx] = ioat->ring[current_idx];
832*e974f91cSConrad Meyer 			ring[new_idx]->id = new_idx;
833*e974f91cSConrad Meyer 		}
834*e974f91cSConrad Meyer 
835*e974f91cSConrad Meyer 		/* free deleted descriptors */
836*e974f91cSConrad Meyer 		for (i = new_size; i < current_size; i++) {
837*e974f91cSConrad Meyer 			ent = ioat_get_ring_entry(ioat, ioat->tail + i);
838*e974f91cSConrad Meyer 			ioat_free_ring_entry(ioat, ent);
839*e974f91cSConrad Meyer 		}
840*e974f91cSConrad Meyer 
841*e974f91cSConrad Meyer 		/* fix up hardware ring */
842*e974f91cSConrad Meyer 		hw = ring[(ioat->tail + new_size - 1) & (new_size - 1)]->u.dma;
843*e974f91cSConrad Meyer 		next = ring[(ioat->tail + new_size) & (new_size - 1)];
844*e974f91cSConrad Meyer 		hw->next = next->hw_desc_bus_addr;
845*e974f91cSConrad Meyer 	}
846*e974f91cSConrad Meyer 
847*e974f91cSConrad Meyer 	free(ioat->ring, M_IOAT);
848*e974f91cSConrad Meyer 	ioat->ring = ring;
849*e974f91cSConrad Meyer 	ioat->ring_size_order = order;
850*e974f91cSConrad Meyer 
851*e974f91cSConrad Meyer 	return (TRUE);
852*e974f91cSConrad Meyer }
853*e974f91cSConrad Meyer 
854*e974f91cSConrad Meyer static void
855*e974f91cSConrad Meyer ioat_timer_callback(void *arg)
856*e974f91cSConrad Meyer {
857*e974f91cSConrad Meyer 	struct ioat_descriptor *desc;
858*e974f91cSConrad Meyer 	struct ioat_softc *ioat;
859*e974f91cSConrad Meyer 	uint64_t status;
860*e974f91cSConrad Meyer 	uint32_t chanerr;
861*e974f91cSConrad Meyer 
862*e974f91cSConrad Meyer 	ioat = arg;
863*e974f91cSConrad Meyer 	ioat_log_message(2, "%s\n", __func__);
864*e974f91cSConrad Meyer 
865*e974f91cSConrad Meyer 	if (ioat->is_completion_pending) {
866*e974f91cSConrad Meyer 		status = ioat_get_chansts(ioat);
867*e974f91cSConrad Meyer 
868*e974f91cSConrad Meyer 		/*
869*e974f91cSConrad Meyer 		 * When halted due to errors, check for channel programming
870*e974f91cSConrad Meyer 		 * errors before advancing the completion state.
871*e974f91cSConrad Meyer 		 */
872*e974f91cSConrad Meyer 		if (is_ioat_halted(status)) {
873*e974f91cSConrad Meyer 			chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
874*e974f91cSConrad Meyer 			ioat_log_message(0, "Channel halted (%x)\n", chanerr);
875*e974f91cSConrad Meyer 
876*e974f91cSConrad Meyer 			desc = ioat_get_ring_entry(ioat, ioat->tail + 0);
877*e974f91cSConrad Meyer 			dump_descriptor(desc->u.raw);
878*e974f91cSConrad Meyer 
879*e974f91cSConrad Meyer 			desc = ioat_get_ring_entry(ioat, ioat->tail + 1);
880*e974f91cSConrad Meyer 			dump_descriptor(desc->u.raw);
881*e974f91cSConrad Meyer 		}
882*e974f91cSConrad Meyer 		ioat_process_events(ioat);
883*e974f91cSConrad Meyer 	} else {
884*e974f91cSConrad Meyer 		mtx_lock(&ioat->submit_lock);
885*e974f91cSConrad Meyer 		mtx_lock(&ioat->cleanup_lock);
886*e974f91cSConrad Meyer 
887*e974f91cSConrad Meyer 		if (ioat_get_active(ioat) == 0 &&
888*e974f91cSConrad Meyer 		    ioat->ring_size_order > IOAT_MIN_ORDER)
889*e974f91cSConrad Meyer 			resize_ring(ioat, ioat->ring_size_order - 1);
890*e974f91cSConrad Meyer 
891*e974f91cSConrad Meyer 		mtx_unlock(&ioat->cleanup_lock);
892*e974f91cSConrad Meyer 		mtx_unlock(&ioat->submit_lock);
893*e974f91cSConrad Meyer 
894*e974f91cSConrad Meyer 		if (ioat->ring_size_order > IOAT_MIN_ORDER)
895*e974f91cSConrad Meyer 			callout_reset(&ioat->timer, 5 * hz,
896*e974f91cSConrad Meyer 			    ioat_timer_callback, ioat);
897*e974f91cSConrad Meyer 	}
898*e974f91cSConrad Meyer }
899*e974f91cSConrad Meyer 
900*e974f91cSConrad Meyer /*
901*e974f91cSConrad Meyer  * Support Functions
902*e974f91cSConrad Meyer  */
903*e974f91cSConrad Meyer static void
904*e974f91cSConrad Meyer ioat_submit_single(struct ioat_softc *ioat)
905*e974f91cSConrad Meyer {
906*e974f91cSConrad Meyer 
907*e974f91cSConrad Meyer 	atomic_add_rel_int(&ioat->head, 1);
908*e974f91cSConrad Meyer 
909*e974f91cSConrad Meyer 	if (!ioat->is_completion_pending) {
910*e974f91cSConrad Meyer 		ioat->is_completion_pending = TRUE;
911*e974f91cSConrad Meyer 		callout_reset(&ioat->timer, 10 * hz, ioat_timer_callback,
912*e974f91cSConrad Meyer 		    ioat);
913*e974f91cSConrad Meyer 	}
914*e974f91cSConrad Meyer }
915*e974f91cSConrad Meyer 
916*e974f91cSConrad Meyer static int
917*e974f91cSConrad Meyer ioat_reset_hw(struct ioat_softc *ioat)
918*e974f91cSConrad Meyer {
919*e974f91cSConrad Meyer 	uint64_t status;
920*e974f91cSConrad Meyer 	uint32_t chanerr;
921*e974f91cSConrad Meyer 	int timeout;
922*e974f91cSConrad Meyer 
923*e974f91cSConrad Meyer 	status = ioat_get_chansts(ioat);
924*e974f91cSConrad Meyer 	if (is_ioat_active(status) || is_ioat_idle(status))
925*e974f91cSConrad Meyer 		ioat_suspend(ioat);
926*e974f91cSConrad Meyer 
927*e974f91cSConrad Meyer 	/* Wait at most 20 ms */
928*e974f91cSConrad Meyer 	for (timeout = 0; (is_ioat_active(status) || is_ioat_idle(status)) &&
929*e974f91cSConrad Meyer 	    timeout < 20; timeout++) {
930*e974f91cSConrad Meyer 		DELAY(1000);
931*e974f91cSConrad Meyer 		status = ioat_get_chansts(ioat);
932*e974f91cSConrad Meyer 	}
933*e974f91cSConrad Meyer 	if (timeout == 20)
934*e974f91cSConrad Meyer 		return (ETIMEDOUT);
935*e974f91cSConrad Meyer 
936*e974f91cSConrad Meyer 	chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
937*e974f91cSConrad Meyer 	ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr);
938*e974f91cSConrad Meyer 
939*e974f91cSConrad Meyer 	/*
940*e974f91cSConrad Meyer 	 * IOAT v3 workaround - CHANERRMSK_INT with 3E07h to masks out errors
941*e974f91cSConrad Meyer 	 *  that can cause stability issues for IOAT v3.
942*e974f91cSConrad Meyer 	 */
943*e974f91cSConrad Meyer 	pci_write_config(ioat->device, IOAT_CFG_CHANERRMASK_INT_OFFSET, 0x3e07,
944*e974f91cSConrad Meyer 	    4);
945*e974f91cSConrad Meyer 	chanerr = pci_read_config(ioat->device, IOAT_CFG_CHANERR_INT_OFFSET, 4);
946*e974f91cSConrad Meyer 	pci_write_config(ioat->device, IOAT_CFG_CHANERR_INT_OFFSET, chanerr, 4);
947*e974f91cSConrad Meyer 
948*e974f91cSConrad Meyer 	ioat_reset(ioat);
949*e974f91cSConrad Meyer 
950*e974f91cSConrad Meyer 	/* Wait at most 20 ms */
951*e974f91cSConrad Meyer 	for (timeout = 0; ioat_reset_pending(ioat) && timeout < 20; timeout++)
952*e974f91cSConrad Meyer 		DELAY(1000);
953*e974f91cSConrad Meyer 	if (timeout == 20)
954*e974f91cSConrad Meyer 		return (ETIMEDOUT);
955*e974f91cSConrad Meyer 
956*e974f91cSConrad Meyer 	return (0);
957*e974f91cSConrad Meyer }
958*e974f91cSConrad Meyer 
959*e974f91cSConrad Meyer static void
960*e974f91cSConrad Meyer dump_descriptor(void *hw_desc)
961*e974f91cSConrad Meyer {
962*e974f91cSConrad Meyer 	int i, j;
963*e974f91cSConrad Meyer 
964*e974f91cSConrad Meyer 	for (i = 0; i < 2; i++) {
965*e974f91cSConrad Meyer 		for (j = 0; j < 8; j++)
966*e974f91cSConrad Meyer 			printf("%08x ", ((uint32_t *)hw_desc)[i * 8 + j]);
967*e974f91cSConrad Meyer 		printf("\n");
968*e974f91cSConrad Meyer 	}
969*e974f91cSConrad Meyer }
970*e974f91cSConrad Meyer 
971*e974f91cSConrad Meyer static void
972*e974f91cSConrad Meyer ioat_setup_sysctl(device_t device)
973*e974f91cSConrad Meyer {
974*e974f91cSConrad Meyer 	struct sysctl_ctx_list *sysctl_ctx;
975*e974f91cSConrad Meyer 	struct sysctl_oid *sysctl_tree;
976*e974f91cSConrad Meyer 	struct ioat_softc *ioat;
977*e974f91cSConrad Meyer 
978*e974f91cSConrad Meyer 	ioat = DEVICE2SOFTC(device);
979*e974f91cSConrad Meyer 	sysctl_ctx = device_get_sysctl_ctx(device);
980*e974f91cSConrad Meyer 	sysctl_tree = device_get_sysctl_tree(device);
981*e974f91cSConrad Meyer 
982*e974f91cSConrad Meyer 	SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO,
983*e974f91cSConrad Meyer 	    "ring_size_order", CTLFLAG_RD, &ioat->ring_size_order,
984*e974f91cSConrad Meyer 	    0, "HW descriptor ring size order");
985*e974f91cSConrad Meyer 	SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO,
986*e974f91cSConrad Meyer 	    "head", CTLFLAG_RD, &ioat->head,
987*e974f91cSConrad Meyer 	    0, "HW descriptor head pointer index");
988*e974f91cSConrad Meyer 	SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO,
989*e974f91cSConrad Meyer 	    "tail", CTLFLAG_RD, &ioat->tail,
990*e974f91cSConrad Meyer 	    0, "HW descriptor tail pointer index");
991*e974f91cSConrad Meyer }
992*e974f91cSConrad Meyer 
993*e974f91cSConrad Meyer void
994*e974f91cSConrad Meyer ioat_log_message(int verbosity, char *fmt, ...)
995*e974f91cSConrad Meyer {
996*e974f91cSConrad Meyer 	va_list argp;
997*e974f91cSConrad Meyer 	char buffer[512];
998*e974f91cSConrad Meyer 	struct timeval tv;
999*e974f91cSConrad Meyer 
1000*e974f91cSConrad Meyer 	if (verbosity > g_ioat_debug_level)
1001*e974f91cSConrad Meyer 		return;
1002*e974f91cSConrad Meyer 
1003*e974f91cSConrad Meyer 	va_start(argp, fmt);
1004*e974f91cSConrad Meyer 	vsnprintf(buffer, sizeof(buffer) - 1, fmt, argp);
1005*e974f91cSConrad Meyer 	va_end(argp);
1006*e974f91cSConrad Meyer 	microuptime(&tv);
1007*e974f91cSConrad Meyer 
1008*e974f91cSConrad Meyer 	printf("[%d:%06d] ioat: %s", (int)tv.tv_sec, (int)tv.tv_usec, buffer);
1009*e974f91cSConrad Meyer }
1010