xref: /freebsd/sys/dev/ioat/ioat.c (revision c1eb4109f3b1d90a9c1538179ea6a52a191899b2)
1 /*-
2  * Copyright (C) 2012 Intel Corporation
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/bus.h>
33 #include <sys/conf.h>
34 #include <sys/ioccom.h>
35 #include <sys/kernel.h>
36 #include <sys/lock.h>
37 #include <sys/malloc.h>
38 #include <sys/module.h>
39 #include <sys/mutex.h>
40 #include <sys/rman.h>
41 #include <sys/sysctl.h>
42 #include <sys/time.h>
43 #include <dev/pci/pcireg.h>
44 #include <dev/pci/pcivar.h>
45 #include <machine/bus.h>
46 #include <machine/resource.h>
47 #include <machine/stdarg.h>
48 
49 #include "ioat.h"
50 #include "ioat_hw.h"
51 #include "ioat_internal.h"
52 
53 static int ioat_probe(device_t device);
54 static int ioat_attach(device_t device);
55 static int ioat_detach(device_t device);
56 static int ioat_setup_intr(struct ioat_softc *ioat);
57 static int ioat_teardown_intr(struct ioat_softc *ioat);
58 static int ioat3_attach(device_t device);
59 static int ioat_map_pci_bar(struct ioat_softc *ioat);
60 static void ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg,
61     int error);
62 static void ioat_interrupt_handler(void *arg);
63 static boolean_t ioat_model_resets_msix(struct ioat_softc *ioat);
64 static void ioat_process_events(struct ioat_softc *ioat);
65 static inline uint32_t ioat_get_active(struct ioat_softc *ioat);
66 static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat);
67 static void ioat_free_ring_entry(struct ioat_softc *ioat,
68     struct ioat_descriptor *desc);
69 static struct ioat_descriptor * ioat_alloc_ring_entry(struct ioat_softc *ioat);
70 static int ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs);
71 static struct ioat_descriptor * ioat_get_ring_entry(struct ioat_softc *ioat,
72     uint32_t index);
73 static boolean_t resize_ring(struct ioat_softc *ioat, int order);
74 static void ioat_timer_callback(void *arg);
75 static void dump_descriptor(void *hw_desc);
76 static void ioat_submit_single(struct ioat_softc *ioat);
77 static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg,
78     int error);
79 static int ioat_reset_hw(struct ioat_softc *ioat);
80 static void ioat_setup_sysctl(device_t device);
81 
82 MALLOC_DEFINE(M_IOAT, "ioat", "ioat driver memory allocations");
83 SYSCTL_NODE(_hw, OID_AUTO, ioat, CTLFLAG_RD, 0, "ioat node");
84 
85 static int g_force_legacy_interrupts;
86 SYSCTL_INT(_hw_ioat, OID_AUTO, force_legacy_interrupts, CTLFLAG_RDTUN,
87     &g_force_legacy_interrupts, 0, "Set to non-zero to force MSI-X disabled");
88 
89 static int g_ioat_debug_level = 0;
90 SYSCTL_INT(_hw_ioat, OID_AUTO, debug_level, CTLFLAG_RWTUN, &g_ioat_debug_level,
91     0, "Set log level (0-3) for ioat(4). Higher is more verbose.");
92 
93 /*
94  * OS <-> Driver interface structures
95  */
96 static device_method_t ioat_pci_methods[] = {
97 	/* Device interface */
98 	DEVMETHOD(device_probe,     ioat_probe),
99 	DEVMETHOD(device_attach,    ioat_attach),
100 	DEVMETHOD(device_detach,    ioat_detach),
101 	{ 0, 0 }
102 };
103 
104 static driver_t ioat_pci_driver = {
105 	"ioat",
106 	ioat_pci_methods,
107 	sizeof(struct ioat_softc),
108 };
109 
110 static devclass_t ioat_devclass;
111 DRIVER_MODULE(ioat, pci, ioat_pci_driver, ioat_devclass, 0, 0);
112 
113 /*
114  * Private data structures
115  */
116 static struct ioat_softc *ioat_channel[IOAT_MAX_CHANNELS];
117 static int ioat_channel_index = 0;
118 SYSCTL_INT(_hw_ioat, OID_AUTO, channels, CTLFLAG_RD, &ioat_channel_index, 0,
119     "Number of IOAT channels attached");
120 
121 static struct _pcsid
122 {
123 	u_int32_t   type;
124 	const char  *desc;
125 } pci_ids[] = {
126 	{ 0x34308086, "TBG IOAT Ch0" },
127 	{ 0x34318086, "TBG IOAT Ch1" },
128 	{ 0x34328086, "TBG IOAT Ch2" },
129 	{ 0x34338086, "TBG IOAT Ch3" },
130 	{ 0x34298086, "TBG IOAT Ch4" },
131 	{ 0x342a8086, "TBG IOAT Ch5" },
132 	{ 0x342b8086, "TBG IOAT Ch6" },
133 	{ 0x342c8086, "TBG IOAT Ch7" },
134 
135 	{ 0x37108086, "JSF IOAT Ch0" },
136 	{ 0x37118086, "JSF IOAT Ch1" },
137 	{ 0x37128086, "JSF IOAT Ch2" },
138 	{ 0x37138086, "JSF IOAT Ch3" },
139 	{ 0x37148086, "JSF IOAT Ch4" },
140 	{ 0x37158086, "JSF IOAT Ch5" },
141 	{ 0x37168086, "JSF IOAT Ch6" },
142 	{ 0x37178086, "JSF IOAT Ch7" },
143 	{ 0x37188086, "JSF IOAT Ch0 (RAID)" },
144 	{ 0x37198086, "JSF IOAT Ch1 (RAID)" },
145 
146 	{ 0x3c208086, "SNB IOAT Ch0" },
147 	{ 0x3c218086, "SNB IOAT Ch1" },
148 	{ 0x3c228086, "SNB IOAT Ch2" },
149 	{ 0x3c238086, "SNB IOAT Ch3" },
150 	{ 0x3c248086, "SNB IOAT Ch4" },
151 	{ 0x3c258086, "SNB IOAT Ch5" },
152 	{ 0x3c268086, "SNB IOAT Ch6" },
153 	{ 0x3c278086, "SNB IOAT Ch7" },
154 	{ 0x3c2e8086, "SNB IOAT Ch0 (RAID)" },
155 	{ 0x3c2f8086, "SNB IOAT Ch1 (RAID)" },
156 
157 	{ 0x0e208086, "IVB IOAT Ch0" },
158 	{ 0x0e218086, "IVB IOAT Ch1" },
159 	{ 0x0e228086, "IVB IOAT Ch2" },
160 	{ 0x0e238086, "IVB IOAT Ch3" },
161 	{ 0x0e248086, "IVB IOAT Ch4" },
162 	{ 0x0e258086, "IVB IOAT Ch5" },
163 	{ 0x0e268086, "IVB IOAT Ch6" },
164 	{ 0x0e278086, "IVB IOAT Ch7" },
165 	{ 0x0e2e8086, "IVB IOAT Ch0 (RAID)" },
166 	{ 0x0e2f8086, "IVB IOAT Ch1 (RAID)" },
167 
168 	{ 0x2f208086, "HSW IOAT Ch0" },
169 	{ 0x2f218086, "HSW IOAT Ch1" },
170 	{ 0x2f228086, "HSW IOAT Ch2" },
171 	{ 0x2f238086, "HSW IOAT Ch3" },
172 	{ 0x2f248086, "HSW IOAT Ch4" },
173 	{ 0x2f258086, "HSW IOAT Ch5" },
174 	{ 0x2f268086, "HSW IOAT Ch6" },
175 	{ 0x2f278086, "HSW IOAT Ch7" },
176 	{ 0x2f2e8086, "HSW IOAT Ch0 (RAID)" },
177 	{ 0x2f2f8086, "HSW IOAT Ch1 (RAID)" },
178 
179 	{ 0x0c508086, "BWD IOAT Ch0" },
180 	{ 0x0c518086, "BWD IOAT Ch1" },
181 	{ 0x0c528086, "BWD IOAT Ch2" },
182 	{ 0x0c538086, "BWD IOAT Ch3" },
183 
184 	{ 0x6f508086, "BDXDE IOAT Ch0" },
185 	{ 0x6f518086, "BDXDE IOAT Ch1" },
186 	{ 0x6f528086, "BDXDE IOAT Ch2" },
187 	{ 0x6f538086, "BDXDE IOAT Ch3" },
188 
189 	{ 0x00000000, NULL           }
190 };
191 
192 /*
193  * OS <-> Driver linkage functions
194  */
195 static int
196 ioat_probe(device_t device)
197 {
198 	struct _pcsid *ep;
199 	u_int32_t type;
200 
201 	type = pci_get_devid(device);
202 	for (ep = pci_ids; ep->type; ep++) {
203 		if (ep->type == type) {
204 			device_set_desc(device, ep->desc);
205 			return (0);
206 		}
207 	}
208 	return (ENXIO);
209 }
210 
211 static int
212 ioat_attach(device_t device)
213 {
214 	struct ioat_softc *ioat;
215 	int error;
216 
217 	ioat = DEVICE2SOFTC(device);
218 	ioat->device = device;
219 
220 	error = ioat_map_pci_bar(ioat);
221 	if (error != 0)
222 		goto err;
223 
224 	ioat->version = ioat_read_cbver(ioat);
225 	if (ioat->version < IOAT_VER_3_0) {
226 		error = ENODEV;
227 		goto err;
228 	}
229 
230 	error = ioat_setup_intr(ioat);
231 	if (error != 0)
232 		return (error);
233 
234 	error = ioat3_attach(device);
235 	if (error != 0)
236 		goto err;
237 
238 	error = pci_enable_busmaster(device);
239 	if (error != 0)
240 		goto err;
241 
242 	ioat_channel[ioat_channel_index++] = ioat;
243 
244 err:
245 	if (error != 0)
246 		ioat_detach(device);
247 	return (error);
248 }
249 
250 static int
251 ioat_detach(device_t device)
252 {
253 	struct ioat_softc *ioat;
254 	uint32_t i;
255 
256 	ioat = DEVICE2SOFTC(device);
257 	callout_drain(&ioat->timer);
258 
259 	pci_disable_busmaster(device);
260 
261 	if (ioat->pci_resource != NULL)
262 		bus_release_resource(device, SYS_RES_MEMORY,
263 		    ioat->pci_resource_id, ioat->pci_resource);
264 
265 	if (ioat->ring != NULL) {
266 		for (i = 0; i < (1 << ioat->ring_size_order); i++)
267 			ioat_free_ring_entry(ioat, ioat->ring[i]);
268 		free(ioat->ring, M_IOAT);
269 	}
270 
271 	if (ioat->comp_update != NULL) {
272 		bus_dmamap_unload(ioat->comp_update_tag, ioat->comp_update_map);
273 		bus_dmamem_free(ioat->comp_update_tag, ioat->comp_update,
274 		    ioat->comp_update_map);
275 		bus_dma_tag_destroy(ioat->comp_update_tag);
276 	}
277 
278 	bus_dma_tag_destroy(ioat->hw_desc_tag);
279 
280 	ioat_teardown_intr(ioat);
281 
282 	return (0);
283 }
284 
285 static int
286 ioat_teardown_intr(struct ioat_softc *ioat)
287 {
288 
289 	if (ioat->tag != NULL)
290 		bus_teardown_intr(ioat->device, ioat->res, ioat->tag);
291 
292 	if (ioat->res != NULL)
293 		bus_release_resource(ioat->device, SYS_RES_IRQ,
294 		    rman_get_rid(ioat->res), ioat->res);
295 
296 	pci_release_msi(ioat->device);
297 	return (0);
298 }
299 
300 static int
301 ioat3_selftest(struct ioat_softc *ioat)
302 {
303 	uint64_t status;
304 	uint32_t chanerr;
305 	int i;
306 
307 	ioat_acquire(&ioat->dmaengine);
308 	ioat_null(&ioat->dmaengine, NULL, NULL, 0);
309 	ioat_release(&ioat->dmaengine);
310 
311 	for (i = 0; i < 100; i++) {
312 		DELAY(1);
313 		status = ioat_get_chansts(ioat);
314 		if (is_ioat_idle(status))
315 			return (0);
316 	}
317 
318 	chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
319 	ioat_log_message(0, "could not start channel: "
320 	    "status = %#jx error = %x\n", (uintmax_t)status, chanerr);
321 	return (ENXIO);
322 }
323 
324 /*
325  * Initialize Hardware
326  */
327 static int
328 ioat3_attach(device_t device)
329 {
330 	struct ioat_softc *ioat;
331 	struct ioat_descriptor **ring;
332 	struct ioat_descriptor *next;
333 	struct ioat_dma_hw_descriptor *dma_hw_desc;
334 	uint32_t capabilities;
335 	int i, num_descriptors;
336 	int error;
337 	uint8_t xfercap;
338 
339 	error = 0;
340 	ioat = DEVICE2SOFTC(device);
341 	capabilities = ioat_read_dmacapability(ioat);
342 
343 	xfercap = ioat_read_xfercap(ioat);
344 
345 	/* Only bits [4:0] are valid. */
346 	xfercap &= 0x1f;
347 	ioat->max_xfer_size = 1 << xfercap;
348 
349 	/* TODO: need to check DCA here if we ever do XOR/PQ */
350 
351 	mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF);
352 	mtx_init(&ioat->cleanup_lock, "ioat_process_events", NULL, MTX_DEF);
353 	callout_init(&ioat->timer, CALLOUT_MPSAFE);
354 
355 	ioat->is_resize_pending = FALSE;
356 	ioat->is_completion_pending = FALSE;
357 	ioat->is_reset_pending = FALSE;
358 	ioat->is_channel_running = FALSE;
359 	ioat->is_waiting_for_ack = FALSE;
360 
361 	bus_dma_tag_create(bus_get_dma_tag(ioat->device), sizeof(uint64_t), 0x0,
362 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
363 	    sizeof(uint64_t), 1, sizeof(uint64_t), 0, NULL, NULL,
364 	    &ioat->comp_update_tag);
365 
366 	error = bus_dmamem_alloc(ioat->comp_update_tag,
367 	    (void **)&ioat->comp_update, BUS_DMA_ZERO, &ioat->comp_update_map);
368 	if (ioat->comp_update == NULL)
369 		return (ENOMEM);
370 
371 	error = bus_dmamap_load(ioat->comp_update_tag, ioat->comp_update_map,
372 	    ioat->comp_update, sizeof(uint64_t), ioat_comp_update_map, ioat,
373 	    0);
374 	if (error != 0)
375 		return (error);
376 
377 	ioat->ring_size_order = IOAT_MIN_ORDER;
378 
379 	num_descriptors = 1 << ioat->ring_size_order;
380 
381 	bus_dma_tag_create(bus_get_dma_tag(ioat->device), 0x40, 0x0,
382 	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
383 	    sizeof(struct ioat_dma_hw_descriptor), 1,
384 	    sizeof(struct ioat_dma_hw_descriptor), 0, NULL, NULL,
385 	    &ioat->hw_desc_tag);
386 
387 	ioat->ring = malloc(num_descriptors * sizeof(*ring), M_IOAT,
388 	    M_ZERO | M_NOWAIT);
389 	if (ioat->ring == NULL)
390 		return (ENOMEM);
391 
392 	ring = ioat->ring;
393 	for (i = 0; i < num_descriptors; i++) {
394 		ring[i] = ioat_alloc_ring_entry(ioat);
395 		if (ring[i] == NULL)
396 			return (ENOMEM);
397 
398 		ring[i]->id = i;
399 	}
400 
401 	for (i = 0; i < num_descriptors - 1; i++) {
402 		next = ring[i + 1];
403 		dma_hw_desc = ring[i]->u.dma;
404 
405 		dma_hw_desc->next = next->hw_desc_bus_addr;
406 	}
407 
408 	ring[i]->u.dma->next = ring[0]->hw_desc_bus_addr;
409 
410 	ioat->head = 0;
411 	ioat->tail = 0;
412 	ioat->last_seen = 0;
413 
414 	error = ioat_reset_hw(ioat);
415 	if (error != 0)
416 		return (error);
417 
418 	ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
419 	ioat_write_chancmp(ioat, ioat->comp_update_bus_addr);
420 	ioat_write_chainaddr(ioat, ring[0]->hw_desc_bus_addr);
421 
422 	error = ioat3_selftest(ioat);
423 	if (error != 0)
424 		return (error);
425 
426 	ioat_process_events(ioat);
427 	ioat_setup_sysctl(device);
428 	return (0);
429 }
430 
431 static int
432 ioat_map_pci_bar(struct ioat_softc *ioat)
433 {
434 
435 	ioat->pci_resource_id = PCIR_BAR(0);
436 	ioat->pci_resource = bus_alloc_resource(ioat->device, SYS_RES_MEMORY,
437 	    &ioat->pci_resource_id, 0, ~0, 1, RF_ACTIVE);
438 
439 	if (ioat->pci_resource == NULL) {
440 		ioat_log_message(0, "unable to allocate pci resource\n");
441 		return (ENODEV);
442 	}
443 
444 	ioat->pci_bus_tag = rman_get_bustag(ioat->pci_resource);
445 	ioat->pci_bus_handle = rman_get_bushandle(ioat->pci_resource);
446 	return (0);
447 }
448 
449 static void
450 ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
451 {
452 	struct ioat_softc *ioat = arg;
453 
454 	ioat->comp_update_bus_addr = seg[0].ds_addr;
455 }
456 
457 static void
458 ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
459 {
460 	bus_addr_t *baddr;
461 
462 	baddr = arg;
463 	*baddr = segs->ds_addr;
464 }
465 
466 /*
467  * Interrupt setup and handlers
468  */
469 static int
470 ioat_setup_intr(struct ioat_softc *ioat)
471 {
472 	uint32_t num_vectors;
473 	int error;
474 	boolean_t use_msix;
475 	boolean_t force_legacy_interrupts;
476 
477 	use_msix = FALSE;
478 	force_legacy_interrupts = FALSE;
479 
480 	if (!g_force_legacy_interrupts && pci_msix_count(ioat->device) >= 1) {
481 		num_vectors = 1;
482 		pci_alloc_msix(ioat->device, &num_vectors);
483 		if (num_vectors == 1)
484 			use_msix = TRUE;
485 	}
486 
487 	if (use_msix) {
488 		ioat->rid = 1;
489 		ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ,
490 		    &ioat->rid, RF_ACTIVE);
491 	} else {
492 		ioat->rid = 0;
493 		ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ,
494 		    &ioat->rid, RF_SHAREABLE | RF_ACTIVE);
495 	}
496 	if (ioat->res == NULL) {
497 		ioat_log_message(0, "bus_alloc_resource failed\n");
498 		return (ENOMEM);
499 	}
500 
501 	ioat->tag = NULL;
502 	error = bus_setup_intr(ioat->device, ioat->res, INTR_MPSAFE |
503 	    INTR_TYPE_MISC, NULL, ioat_interrupt_handler, ioat, &ioat->tag);
504 	if (error != 0) {
505 		ioat_log_message(0, "bus_setup_intr failed\n");
506 		return (error);
507 	}
508 
509 	ioat_write_intrctrl(ioat, IOAT_INTRCTRL_MASTER_INT_EN);
510 	return (0);
511 }
512 
513 static boolean_t
514 ioat_model_resets_msix(struct ioat_softc *ioat)
515 {
516 	u_int32_t pciid;
517 
518 	pciid = pci_get_devid(ioat->device);
519 	switch (pciid) {
520 		/* BWD: */
521 	case 0x0c508086:
522 	case 0x0c518086:
523 	case 0x0c528086:
524 	case 0x0c538086:
525 		/* BDXDE: */
526 	case 0x6f508086:
527 	case 0x6f518086:
528 	case 0x6f528086:
529 	case 0x6f538086:
530 		return (TRUE);
531 	}
532 
533 	return (FALSE);
534 }
535 
536 static void
537 ioat_interrupt_handler(void *arg)
538 {
539 	struct ioat_softc *ioat = arg;
540 
541 	ioat_process_events(ioat);
542 }
543 
544 static void
545 ioat_process_events(struct ioat_softc *ioat)
546 {
547 	struct ioat_descriptor *desc;
548 	struct bus_dmadesc *dmadesc;
549 	uint64_t comp_update, status;
550 	uint32_t completed;
551 
552 	mtx_lock(&ioat->cleanup_lock);
553 
554 	completed = 0;
555 	comp_update = *ioat->comp_update;
556 	status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK;
557 
558 	ioat_log_message(3, "%s\n", __func__);
559 
560 	if (status == ioat->last_seen) {
561 	 	mtx_unlock(&ioat->cleanup_lock);
562 		return;
563 	}
564 
565 	while (1) {
566 		desc = ioat_get_ring_entry(ioat, ioat->tail);
567 		dmadesc = &desc->bus_dmadesc;
568 		ioat_log_message(3, "completing desc %d\n", ioat->tail);
569 
570 		if (dmadesc->callback_fn)
571 			(*dmadesc->callback_fn)(dmadesc->callback_arg);
572 
573 		ioat->tail++;
574 		if (desc->hw_desc_bus_addr == status)
575 			break;
576 	}
577 
578 	ioat->last_seen = desc->hw_desc_bus_addr;
579 
580 	if (ioat->head == ioat->tail) {
581 		ioat->is_completion_pending = FALSE;
582 		callout_reset(&ioat->timer, 5 * hz, ioat_timer_callback, ioat);
583 	}
584 
585 	ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
586 	mtx_unlock(&ioat->cleanup_lock);
587 }
588 
589 /*
590  * User API functions
591  */
592 bus_dmaengine_t
593 ioat_get_dmaengine(uint32_t index)
594 {
595 
596 	if (index < ioat_channel_index)
597 		return (&ioat_channel[index]->dmaengine);
598 	return (NULL);
599 }
600 
601 void
602 ioat_acquire(bus_dmaengine_t dmaengine)
603 {
604 	struct ioat_softc *ioat;
605 
606 	ioat = to_ioat_softc(dmaengine);
607 	mtx_lock(&ioat->submit_lock);
608 	ioat_log_message(3, "%s\n", __func__);
609 }
610 
611 void
612 ioat_release(bus_dmaengine_t dmaengine)
613 {
614 	struct ioat_softc *ioat;
615 
616 	ioat_log_message(3, "%s\n", __func__);
617 	ioat = to_ioat_softc(dmaengine);
618 	ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, (uint16_t)ioat->head);
619 	mtx_unlock(&ioat->submit_lock);
620 }
621 
622 struct bus_dmadesc *
623 ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_callback_t callback_fn,
624     void *callback_arg, uint32_t flags)
625 {
626 	struct ioat_softc *ioat;
627 	struct ioat_descriptor *desc;
628 	struct ioat_dma_hw_descriptor *hw_desc;
629 
630 	KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x",
631 		flags & ~DMA_ALL_FLAGS));
632 
633 	ioat = to_ioat_softc(dmaengine);
634 
635 	if (ioat_reserve_space_and_lock(ioat, 1) != 0)
636 		return (NULL);
637 
638 	ioat_log_message(3, "%s\n", __func__);
639 
640 	desc = ioat_get_ring_entry(ioat, ioat->head);
641 	hw_desc = desc->u.dma;
642 
643 	hw_desc->u.control_raw = 0;
644 	hw_desc->u.control.null = 1;
645 	hw_desc->u.control.completion_update = 1;
646 
647 	if ((flags & DMA_INT_EN) != 0)
648 		hw_desc->u.control.int_enable = 1;
649 
650 	hw_desc->size = 8;
651 	hw_desc->src_addr = 0;
652 	hw_desc->dest_addr = 0;
653 
654 	desc->bus_dmadesc.callback_fn = callback_fn;
655 	desc->bus_dmadesc.callback_arg = callback_arg;
656 
657 	ioat_submit_single(ioat);
658 	return (&desc->bus_dmadesc);
659 }
660 
661 struct bus_dmadesc *
662 ioat_copy(bus_dmaengine_t dmaengine, bus_addr_t dst,
663     bus_addr_t src, bus_size_t len, bus_dmaengine_callback_t callback_fn,
664     void *callback_arg, uint32_t flags)
665 {
666 	struct ioat_descriptor *desc;
667 	struct ioat_dma_hw_descriptor *hw_desc;
668 	struct ioat_softc *ioat;
669 
670 	KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x",
671 		flags & ~DMA_ALL_FLAGS));
672 
673 	ioat = to_ioat_softc(dmaengine);
674 
675 	if (len > ioat->max_xfer_size) {
676 		ioat_log_message(0, "%s: max_xfer_size = %d, requested = %d\n",
677 		    __func__, ioat->max_xfer_size, (int)len);
678 		return (NULL);
679 	}
680 
681 	if (ioat_reserve_space_and_lock(ioat, 1) != 0)
682 		return (NULL);
683 
684 	ioat_log_message(3, "%s\n", __func__);
685 
686 	desc = ioat_get_ring_entry(ioat, ioat->head);
687 	hw_desc = desc->u.dma;
688 
689 	hw_desc->u.control_raw = 0;
690 	hw_desc->u.control.completion_update = 1;
691 
692 	if ((flags & DMA_INT_EN) != 0)
693 		hw_desc->u.control.int_enable = 1;
694 
695 	hw_desc->size = len;
696 	hw_desc->src_addr = src;
697 	hw_desc->dest_addr = dst;
698 
699 	if (g_ioat_debug_level >= 3)
700 		dump_descriptor(hw_desc);
701 
702 	desc->bus_dmadesc.callback_fn = callback_fn;
703 	desc->bus_dmadesc.callback_arg = callback_arg;
704 
705 	ioat_submit_single(ioat);
706 	return (&desc->bus_dmadesc);
707 }
708 
709 /*
710  * Ring Management
711  */
712 static inline uint32_t
713 ioat_get_active(struct ioat_softc *ioat)
714 {
715 
716 	return ((ioat->head - ioat->tail) & ((1 << ioat->ring_size_order) - 1));
717 }
718 
719 static inline uint32_t
720 ioat_get_ring_space(struct ioat_softc *ioat)
721 {
722 
723 	return ((1 << ioat->ring_size_order) - ioat_get_active(ioat) - 1);
724 }
725 
726 static struct ioat_descriptor *
727 ioat_alloc_ring_entry(struct ioat_softc *ioat)
728 {
729 	struct ioat_dma_hw_descriptor *hw_desc;
730 	struct ioat_descriptor *desc;
731 
732 	desc = malloc(sizeof(struct ioat_descriptor), M_IOAT, M_NOWAIT);
733 	if (desc == NULL)
734 		return (NULL);
735 
736 	bus_dmamem_alloc(ioat->hw_desc_tag, (void **)&hw_desc, BUS_DMA_ZERO,
737 	    &ioat->hw_desc_map);
738 	if (hw_desc == NULL) {
739 		free(desc, M_IOAT);
740 		return (NULL);
741 	}
742 
743 	bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc,
744 	    sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr, 0);
745 
746 	desc->u.dma = hw_desc;
747 	return (desc);
748 }
749 
750 static void
751 ioat_free_ring_entry(struct ioat_softc *ioat, struct ioat_descriptor *desc)
752 {
753 
754 	if (desc == NULL)
755 		return;
756 
757 	if (desc->u.dma)
758 		bus_dmamem_free(ioat->hw_desc_tag, desc->u.dma,
759 		    ioat->hw_desc_map);
760 	free(desc, M_IOAT);
761 }
762 
763 static int
764 ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs)
765 {
766 	boolean_t retry;
767 
768 	while (1) {
769 		if (ioat_get_ring_space(ioat) >= num_descs)
770 			return (0);
771 
772 		mtx_lock(&ioat->cleanup_lock);
773 		retry = resize_ring(ioat, ioat->ring_size_order + 1);
774 		mtx_unlock(&ioat->cleanup_lock);
775 
776 		if (!retry)
777 			return (ENOMEM);
778 	}
779 }
780 
781 static struct ioat_descriptor *
782 ioat_get_ring_entry(struct ioat_softc *ioat, uint32_t index)
783 {
784 
785 	return (ioat->ring[index % (1 << ioat->ring_size_order)]);
786 }
787 
788 static boolean_t
789 resize_ring(struct ioat_softc *ioat, int order)
790 {
791 	struct ioat_descriptor **ring;
792 	struct ioat_descriptor *next;
793 	struct ioat_dma_hw_descriptor *hw;
794 	struct ioat_descriptor *ent;
795 	uint32_t current_size, active, new_size, i, new_idx, current_idx;
796 	uint32_t new_idx2;
797 
798 	current_size = 1 << ioat->ring_size_order;
799 	active = (ioat->head - ioat->tail) & (current_size - 1);
800 	new_size = 1 << order;
801 
802 	if (order > IOAT_MAX_ORDER)
803 		return (FALSE);
804 
805 	/*
806 	 * when shrinking, verify that we can hold the current active
807 	 * set in the new ring
808 	 */
809 	if (active >= new_size)
810 		return (FALSE);
811 
812 	/* allocate the array to hold the software ring */
813 	ring = malloc(new_size * sizeof(*ring), M_IOAT, M_ZERO | M_NOWAIT);
814 	if (ring == NULL)
815 		return (FALSE);
816 
817 	ioat_log_message(2, "ring resize: new: %d old: %d\n",
818 	    new_size, current_size);
819 
820 	/* allocate/trim descriptors as needed */
821 	if (new_size > current_size) {
822 		/* copy current descriptors to the new ring */
823 		for (i = 0; i < current_size; i++) {
824 			current_idx = (ioat->tail + i) & (current_size - 1);
825 			new_idx = (ioat->tail + i) & (new_size - 1);
826 
827 			ring[new_idx] = ioat->ring[current_idx];
828 			ring[new_idx]->id = new_idx;
829 		}
830 
831 		/* add new descriptors to the ring */
832 		for (i = current_size; i < new_size; i++) {
833 			new_idx = (ioat->tail + i) & (new_size - 1);
834 
835 			ring[new_idx] = ioat_alloc_ring_entry(ioat);
836 			if (ring[new_idx] == NULL) {
837 				while (i--) {
838 					new_idx2 = (ioat->tail + i) &
839 					    (new_size - 1);
840 
841 					ioat_free_ring_entry(ioat,
842 					    ring[new_idx2]);
843 				}
844 				free(ring, M_IOAT);
845 				return (FALSE);
846 			}
847 			ring[new_idx]->id = new_idx;
848 		}
849 
850 		for (i = current_size - 1; i < new_size; i++) {
851 			new_idx = (ioat->tail + i) & (new_size - 1);
852 			next = ring[(new_idx + 1) & (new_size - 1)];
853 			hw = ring[new_idx]->u.dma;
854 
855 			hw->next = next->hw_desc_bus_addr;
856 		}
857 	} else {
858 		/*
859 		 * copy current descriptors to the new ring, dropping the
860 		 * removed descriptors
861 		 */
862 		for (i = 0; i < new_size; i++) {
863 			current_idx = (ioat->tail + i) & (current_size - 1);
864 			new_idx = (ioat->tail + i) & (new_size - 1);
865 
866 			ring[new_idx] = ioat->ring[current_idx];
867 			ring[new_idx]->id = new_idx;
868 		}
869 
870 		/* free deleted descriptors */
871 		for (i = new_size; i < current_size; i++) {
872 			ent = ioat_get_ring_entry(ioat, ioat->tail + i);
873 			ioat_free_ring_entry(ioat, ent);
874 		}
875 
876 		/* fix up hardware ring */
877 		hw = ring[(ioat->tail + new_size - 1) & (new_size - 1)]->u.dma;
878 		next = ring[(ioat->tail + new_size) & (new_size - 1)];
879 		hw->next = next->hw_desc_bus_addr;
880 	}
881 
882 	free(ioat->ring, M_IOAT);
883 	ioat->ring = ring;
884 	ioat->ring_size_order = order;
885 
886 	return (TRUE);
887 }
888 
889 static void
890 ioat_timer_callback(void *arg)
891 {
892 	struct ioat_descriptor *desc;
893 	struct ioat_softc *ioat;
894 	uint64_t status;
895 	uint32_t chanerr;
896 
897 	ioat = arg;
898 	ioat_log_message(2, "%s\n", __func__);
899 
900 	if (ioat->is_completion_pending) {
901 		status = ioat_get_chansts(ioat);
902 
903 		/*
904 		 * When halted due to errors, check for channel programming
905 		 * errors before advancing the completion state.
906 		 */
907 		if (is_ioat_halted(status)) {
908 			chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
909 			ioat_log_message(0, "Channel halted (%x)\n", chanerr);
910 
911 			desc = ioat_get_ring_entry(ioat, ioat->tail + 0);
912 			dump_descriptor(desc->u.raw);
913 
914 			desc = ioat_get_ring_entry(ioat, ioat->tail + 1);
915 			dump_descriptor(desc->u.raw);
916 		}
917 		ioat_process_events(ioat);
918 	} else {
919 		mtx_lock(&ioat->submit_lock);
920 		mtx_lock(&ioat->cleanup_lock);
921 
922 		if (ioat_get_active(ioat) == 0 &&
923 		    ioat->ring_size_order > IOAT_MIN_ORDER)
924 			resize_ring(ioat, ioat->ring_size_order - 1);
925 
926 		mtx_unlock(&ioat->cleanup_lock);
927 		mtx_unlock(&ioat->submit_lock);
928 
929 		if (ioat->ring_size_order > IOAT_MIN_ORDER)
930 			callout_reset(&ioat->timer, 5 * hz,
931 			    ioat_timer_callback, ioat);
932 	}
933 }
934 
935 /*
936  * Support Functions
937  */
938 static void
939 ioat_submit_single(struct ioat_softc *ioat)
940 {
941 
942 	atomic_add_rel_int(&ioat->head, 1);
943 
944 	if (!ioat->is_completion_pending) {
945 		ioat->is_completion_pending = TRUE;
946 		callout_reset(&ioat->timer, 10 * hz, ioat_timer_callback,
947 		    ioat);
948 	}
949 }
950 
951 static int
952 ioat_reset_hw(struct ioat_softc *ioat)
953 {
954 	uint64_t status;
955 	uint32_t chanerr;
956 	int timeout;
957 
958 	status = ioat_get_chansts(ioat);
959 	if (is_ioat_active(status) || is_ioat_idle(status))
960 		ioat_suspend(ioat);
961 
962 	/* Wait at most 20 ms */
963 	for (timeout = 0; (is_ioat_active(status) || is_ioat_idle(status)) &&
964 	    timeout < 20; timeout++) {
965 		DELAY(1000);
966 		status = ioat_get_chansts(ioat);
967 	}
968 	if (timeout == 20)
969 		return (ETIMEDOUT);
970 
971 	chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET);
972 	ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr);
973 
974 	/*
975 	 * IOAT v3 workaround - CHANERRMSK_INT with 3E07h to masks out errors
976 	 *  that can cause stability issues for IOAT v3.
977 	 */
978 	pci_write_config(ioat->device, IOAT_CFG_CHANERRMASK_INT_OFFSET, 0x3e07,
979 	    4);
980 	chanerr = pci_read_config(ioat->device, IOAT_CFG_CHANERR_INT_OFFSET, 4);
981 	pci_write_config(ioat->device, IOAT_CFG_CHANERR_INT_OFFSET, chanerr, 4);
982 
983 	/*
984 	 * BDXDE and BWD models reset MSI-X registers on device reset.
985 	 * Save/restore their contents manually.
986 	 */
987 	if (ioat_model_resets_msix(ioat))
988 		pci_save_state(ioat->device);
989 
990 	ioat_reset(ioat);
991 
992 	/* Wait at most 20 ms */
993 	for (timeout = 0; ioat_reset_pending(ioat) && timeout < 20; timeout++)
994 		DELAY(1000);
995 	if (timeout == 20)
996 		return (ETIMEDOUT);
997 
998 	if (ioat_model_resets_msix(ioat))
999 		pci_restore_state(ioat->device);
1000 
1001 	return (0);
1002 }
1003 
1004 static void
1005 dump_descriptor(void *hw_desc)
1006 {
1007 	int i, j;
1008 
1009 	for (i = 0; i < 2; i++) {
1010 		for (j = 0; j < 8; j++)
1011 			printf("%08x ", ((uint32_t *)hw_desc)[i * 8 + j]);
1012 		printf("\n");
1013 	}
1014 }
1015 
1016 static void
1017 ioat_setup_sysctl(device_t device)
1018 {
1019 	struct sysctl_ctx_list *sysctl_ctx;
1020 	struct sysctl_oid *sysctl_tree;
1021 	struct ioat_softc *ioat;
1022 
1023 	ioat = DEVICE2SOFTC(device);
1024 	sysctl_ctx = device_get_sysctl_ctx(device);
1025 	sysctl_tree = device_get_sysctl_tree(device);
1026 
1027 	SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO,
1028 	    "ring_size_order", CTLFLAG_RD, &ioat->ring_size_order,
1029 	    0, "HW descriptor ring size order");
1030 	SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO,
1031 	    "head", CTLFLAG_RD, &ioat->head,
1032 	    0, "HW descriptor head pointer index");
1033 	SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO,
1034 	    "tail", CTLFLAG_RD, &ioat->tail,
1035 	    0, "HW descriptor tail pointer index");
1036 }
1037 
1038 void
1039 ioat_log_message(int verbosity, char *fmt, ...)
1040 {
1041 	va_list argp;
1042 	char buffer[512];
1043 	struct timeval tv;
1044 
1045 	if (verbosity > g_ioat_debug_level)
1046 		return;
1047 
1048 	va_start(argp, fmt);
1049 	vsnprintf(buffer, sizeof(buffer) - 1, fmt, argp);
1050 	va_end(argp);
1051 	microuptime(&tv);
1052 
1053 	printf("[%d:%06d] ioat: %s", (int)tv.tv_sec, (int)tv.tv_usec, buffer);
1054 }
1055