xref: /illumos-gate/usr/src/uts/i86pc/io/ioat/ioat_chan.c (revision 584b574a3b16c6772c8204ec1d1c957c56f22a87)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * Copyright (c) 2009, Intel Corporation.
29  * All rights reserved.
30  */
31 
32 #include <sys/errno.h>
33 #include <sys/types.h>
34 #include <sys/conf.h>
35 #include <sys/kmem.h>
36 #include <sys/ddi.h>
37 #include <sys/stat.h>
38 #include <sys/sunddi.h>
39 #include <sys/file.h>
40 #include <sys/open.h>
41 #include <sys/modctl.h>
42 #include <sys/ddi_impldefs.h>
43 #include <sys/sysmacros.h>
44 #include <vm/hat.h>
45 #include <vm/as.h>
46 #include <sys/mach_mmu.h>
47 #ifdef __xpv
48 #include <sys/hypervisor.h>
49 #endif
50 
51 #include <sys/ioat.h>
52 
53 
54 extern ddi_device_acc_attr_t ioat_acc_attr;
55 
56 /* dma attr for the descriptor rings */
57 ddi_dma_attr_t ioat_desc_dma_attr = {
58 	DMA_ATTR_V0,		/* dma_attr_version */
59 	0x0,			/* dma_attr_addr_lo */
60 	0xffffffffffffffff,	/* dma_attr_addr_hi */
61 	0xffffffff,		/* dma_attr_count_max */
62 	0x1000,			/* dma_attr_align */
63 	0x1,			/* dma_attr_burstsizes */
64 	0x1,			/* dma_attr_minxfer */
65 	0xffffffff,		/* dma_attr_maxxfer */
66 	0xffffffff,		/* dma_attr_seg */
67 	0x1,			/* dma_attr_sgllen */
68 	0x1,			/* dma_attr_granular */
69 	0x0,			/* dma_attr_flags */
70 };
71 
72 /* dma attr for the completion buffers */
73 ddi_dma_attr_t ioat_cmpl_dma_attr = {
74 	DMA_ATTR_V0,		/* dma_attr_version */
75 	0x0,			/* dma_attr_addr_lo */
76 	0xffffffffffffffff,	/* dma_attr_addr_hi */
77 	0xffffffff,		/* dma_attr_count_max */
78 	0x40,			/* dma_attr_align */
79 	0x1,			/* dma_attr_burstsizes */
80 	0x1,			/* dma_attr_minxfer */
81 	0xffffffff,		/* dma_attr_maxxfer */
82 	0xffffffff,		/* dma_attr_seg */
83 	0x1,			/* dma_attr_sgllen */
84 	0x1,			/* dma_attr_granular */
85 	0x0,			/* dma_attr_flags */
86 };
87 
88 static int ioat_completion_alloc(ioat_channel_t channel);
89 static void ioat_completion_free(ioat_channel_t channel);
90 static void ioat_channel_start(ioat_channel_t channel);
91 static void ioat_channel_reset(ioat_channel_t channel);
92 
93 int ioat_ring_alloc(ioat_channel_t channel, uint_t desc_cnt);
94 void ioat_ring_free(ioat_channel_t channel);
95 void ioat_ring_seed(ioat_channel_t channel, ioat_chan_dma_desc_t *desc);
96 int ioat_ring_reserve(ioat_channel_t channel, ioat_channel_ring_t *ring,
97     dcopy_cmd_t cmd);
98 
99 static void ioat_cmd_post_copy(ioat_channel_ring_t *ring, uint64_t src_addr,
100     uint64_t dest_addr, uint32_t size, uint32_t ctrl);
101 static void ioat_cmd_post_dca(ioat_channel_ring_t *ring, uint32_t dca_id);
102 
103 
104 /*
105  * ioat_channel_init()
106  */
107 int
ioat_channel_init(ioat_state_t * state)108 ioat_channel_init(ioat_state_t *state)
109 {
110 	int i;
111 
112 	/*
113 	 * initialize each dma channel's state which doesn't change across
114 	 * channel alloc/free.
115 	 */
116 	state->is_chansize = sizeof (struct ioat_channel_s) *
117 	    state->is_num_channels;
118 	state->is_channel = kmem_zalloc(state->is_chansize, KM_SLEEP);
119 	for (i = 0; i < state->is_num_channels; i++) {
120 		state->is_channel[i].ic_state = state;
121 		state->is_channel[i].ic_regs = (uint8_t *)
122 		    ((uintptr_t)state->is_genregs +
123 		    (uintptr_t)(IOAT_CHANNELREG_OFFSET * (i + 1)));
124 	}
125 
126 	/* initial the allocator (from 0 to state->is_num_channels) */
127 	ioat_rs_init(state, 0, state->is_num_channels, &state->is_channel_rs);
128 
129 	return (DDI_SUCCESS);
130 }
131 
132 
133 /*
134  * ioat_channel_fini()
135  */
136 void
ioat_channel_fini(ioat_state_t * state)137 ioat_channel_fini(ioat_state_t *state)
138 {
139 	ioat_rs_fini(&state->is_channel_rs);
140 	kmem_free(state->is_channel, state->is_chansize);
141 }
142 
143 
144 /*
145  * ioat_channel_alloc()
146  *   NOTE: We intentionaly don't handle DCOPY_SLEEP (if no channels are
147  *	available)
148  */
149 /*ARGSUSED*/
150 int
ioat_channel_alloc(void * device_private,dcopy_handle_t handle,int flags,uint_t size,dcopy_query_channel_t * info,void * channel_private)151 ioat_channel_alloc(void *device_private, dcopy_handle_t handle, int flags,
152     uint_t size, dcopy_query_channel_t *info, void *channel_private)
153 {
154 #define	CHANSTRSIZE	20
155 	struct ioat_channel_s *channel;
156 	char chanstr[CHANSTRSIZE];
157 	ioat_channel_t *chan;
158 	ioat_state_t *state;
159 	size_t cmd_size;
160 	uint_t chan_num;
161 	uint32_t estat;
162 	int e;
163 
164 
165 	state = (ioat_state_t *)device_private;
166 	chan = (ioat_channel_t *)channel_private;
167 
168 	/* allocate a H/W channel */
169 	e = ioat_rs_alloc(state->is_channel_rs, &chan_num);
170 	if (e != DDI_SUCCESS) {
171 		return (DCOPY_NORESOURCES);
172 	}
173 
174 	channel = &state->is_channel[chan_num];
175 	channel->ic_inuse = B_TRUE;
176 	channel->ic_chan_num = chan_num;
177 	channel->ic_ver = state->is_ver;
178 	channel->ic_dca_active = B_FALSE;
179 	channel->ic_channel_state = IOAT_CHANNEL_OK;
180 	channel->ic_dcopy_handle = handle;
181 
182 #ifdef	DEBUG
183 	{
184 		/* if we're cbv2, verify that the V2 compatibility bit is set */
185 		uint16_t reg;
186 		if (channel->ic_ver == IOAT_CBv2) {
187 			reg = ddi_get16(state->is_reg_handle,
188 			    (uint16_t *)&channel->ic_regs[IOAT_CHAN_COMP]);
189 			ASSERT(reg & 0x2);
190 		}
191 	}
192 #endif
193 
194 	/*
195 	 * Configure DMA channel
196 	 *   Channel In Use
197 	 *   Error Interrupt Enable
198 	 *   Any Error Abort Enable
199 	 *   Error Completion Enable
200 	 */
201 	ddi_put16(state->is_reg_handle,
202 	    (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], 0x011C);
203 
204 	/* check channel error register, clear any errors */
205 	estat = ddi_get32(state->is_reg_handle,
206 	    (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR]);
207 	if (estat != 0) {
208 #ifdef	DEBUG
209 		cmn_err(CE_CONT, "cleared errors (0x%x) before channel (%d) "
210 		    "enable\n", estat, channel->ic_chan_num);
211 #endif
212 		ddi_put32(state->is_reg_handle,
213 		    (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR], estat);
214 	}
215 
216 	/* allocate and initialize the descriptor buf */
217 	e = ioat_ring_alloc(channel, size);
218 	if (e != DDI_SUCCESS) {
219 		goto chinitfail_desc_alloc;
220 	}
221 
222 	/* allocate and initialize the completion space */
223 	e = ioat_completion_alloc(channel);
224 	if (e != DDI_SUCCESS) {
225 		goto chinitfail_completion_alloc;
226 	}
227 
228 	/* setup kmem_cache for commands */
229 	cmd_size = sizeof (struct dcopy_cmd_s) +
230 	    sizeof (struct dcopy_cmd_priv_s) +
231 	    sizeof (struct ioat_cmd_private_s);
232 	(void) snprintf(chanstr, CHANSTRSIZE, "ioat%dchan%dcmd",
233 	    state->is_instance, channel->ic_chan_num);
234 	channel->ic_cmd_cache = kmem_cache_create(chanstr, cmd_size, 64,
235 	    NULL, NULL, NULL, NULL, NULL, 0);
236 	if (channel->ic_cmd_cache == NULL) {
237 		goto chinitfail_kmem_cache;
238 	}
239 
240 	/* start-up the channel */
241 	ioat_channel_start(channel);
242 
243 	/* fill in the channel info returned to dcopy */
244 	info->qc_version = DCOPY_QUERY_CHANNEL_V0;
245 	info->qc_id = state->is_deviceinfo.di_id;
246 	info->qc_capabilities = (uint64_t)state->is_capabilities;
247 	info->qc_channel_size = (uint64_t)size;
248 	info->qc_chan_num = (uint64_t)channel->ic_chan_num;
249 	if (channel->ic_ver == IOAT_CBv1) {
250 		info->qc_dca_supported = B_FALSE;
251 	} else {
252 		if (info->qc_capabilities & IOAT_DMACAP_DCA) {
253 			info->qc_dca_supported = B_TRUE;
254 		} else {
255 			info->qc_dca_supported = B_FALSE;
256 		}
257 	}
258 
259 	*chan = channel;
260 
261 	return (DCOPY_SUCCESS);
262 
263 chinitfail_kmem_cache:
264 	ioat_completion_free(channel);
265 chinitfail_completion_alloc:
266 	ioat_ring_free(channel);
267 chinitfail_desc_alloc:
268 	return (DCOPY_FAILURE);
269 }
270 
271 
272 /*
273  * ioat_channel_suspend()
274  */
275 /*ARGSUSED*/
276 void
ioat_channel_suspend(ioat_state_t * state)277 ioat_channel_suspend(ioat_state_t *state)
278 {
279 	/*
280 	 * normally you would disable interrupts and reset the H/W here. But
281 	 * since the suspend framework doesn't know who is using us, it may
282 	 * not suspend their I/O before us.  Since we won't actively be doing
283 	 * any DMA or interrupts unless someone asks us to, it's safe to not
284 	 * do anything here.
285 	 */
286 }
287 
288 
289 /*
290  * ioat_channel_resume()
291  */
292 int
ioat_channel_resume(ioat_state_t * state)293 ioat_channel_resume(ioat_state_t *state)
294 {
295 	ioat_channel_ring_t *ring;
296 	ioat_channel_t channel;
297 	uint32_t estat;
298 	int i;
299 
300 
301 	for (i = 0; i < state->is_num_channels; i++) {
302 		channel = &state->is_channel[i];
303 		ring = channel->ic_ring;
304 
305 		if (!channel->ic_inuse) {
306 			continue;
307 		}
308 
309 		/*
310 		 * Configure DMA channel
311 		 *   Channel In Use
312 		 *   Error Interrupt Enable
313 		 *   Any Error Abort Enable
314 		 *   Error Completion Enable
315 		 */
316 		ddi_put16(state->is_reg_handle,
317 		    (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], 0x011C);
318 
319 		/* check channel error register, clear any errors */
320 		estat = ddi_get32(state->is_reg_handle,
321 		    (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR]);
322 		if (estat != 0) {
323 #ifdef	DEBUG
324 			cmn_err(CE_CONT, "cleared errors (0x%x) before channel"
325 			    " (%d) enable\n", estat, channel->ic_chan_num);
326 #endif
327 			ddi_put32(state->is_reg_handle,
328 			    (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR],
329 			    estat);
330 		}
331 
332 		/* Re-initialize the ring */
333 		bzero(ring->cr_desc, channel->ic_desc_alloc_size);
334 		/* write the physical address into the chain address register */
335 		if (channel->ic_ver == IOAT_CBv1) {
336 			ddi_put32(state->is_reg_handle,
337 			    (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_LO],
338 			    (uint32_t)(ring->cr_phys_desc & 0xffffffff));
339 			ddi_put32(state->is_reg_handle,
340 			    (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_HI],
341 			    (uint32_t)(ring->cr_phys_desc >> 32));
342 		} else {
343 			ASSERT(channel->ic_ver == IOAT_CBv2);
344 			ddi_put32(state->is_reg_handle,
345 			    (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_LO],
346 			    (uint32_t)(ring->cr_phys_desc & 0xffffffff));
347 			ddi_put32(state->is_reg_handle,
348 			    (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_HI],
349 			    (uint32_t)(ring->cr_phys_desc >> 32));
350 		}
351 
352 		/* re-initialize the completion buffer */
353 		bzero((void *)channel->ic_cmpl, channel->ic_cmpl_alloc_size);
354 		/* write the phys addr into the completion address register */
355 		ddi_put32(state->is_reg_handle,
356 		    (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_LO],
357 		    (uint32_t)(channel->ic_phys_cmpl & 0xffffffff));
358 		ddi_put32(state->is_reg_handle,
359 		    (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_HI],
360 		    (uint32_t)(channel->ic_phys_cmpl >> 32));
361 
362 		/* start-up the channel */
363 		ioat_channel_start(channel);
364 
365 	}
366 
367 	return (DDI_SUCCESS);
368 }
369 
370 /*
371  * quiesce(9E) entry point.
372  *
373  * This function is called when the system is single-threaded at high
374  * PIL with preemption disabled. Therefore, this function must not be
375  * blocked.
376  *
377  * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
378  * DDI_FAILURE indicates an error condition and should almost never happen.
379  */
380 void
ioat_channel_quiesce(ioat_state_t * state)381 ioat_channel_quiesce(ioat_state_t *state)
382 {
383 	int i;
384 
385 	/*
386 	 * Walk through all channels and quiesce
387 	 */
388 	for (i = 0; i < state->is_num_channels; i++) {
389 
390 		ioat_channel_t	channel = state->is_channel + i;
391 
392 		if (!channel->ic_inuse)
393 			continue;
394 
395 		/* disable the interrupts */
396 		ddi_put16(state->is_reg_handle,
397 		    (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL],
398 		    0x0);
399 
400 		ioat_channel_reset(channel);
401 	}
402 }
403 
404 
405 /*
406  * ioat_channel_free()
407  */
408 void
ioat_channel_free(void * channel_private)409 ioat_channel_free(void *channel_private)
410 {
411 	struct ioat_channel_s *channel;
412 	ioat_channel_t *chan;
413 	ioat_state_t *state;
414 	uint_t chan_num;
415 
416 
417 	chan = (ioat_channel_t *)channel_private;
418 	channel = *chan;
419 
420 	state = channel->ic_state;
421 	chan_num = channel->ic_chan_num;
422 
423 	/* disable the interrupts */
424 	ddi_put16(state->is_reg_handle,
425 	    (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], 0x0);
426 
427 	ioat_channel_reset(channel);
428 
429 	/* cleanup command cache */
430 	kmem_cache_destroy(channel->ic_cmd_cache);
431 
432 	/* clean-up/free-up the completion space and descriptors */
433 	ioat_completion_free(channel);
434 	ioat_ring_free(channel);
435 
436 	channel->ic_inuse = B_FALSE;
437 
438 	/* free the H/W DMA engine */
439 	ioat_rs_free(state->is_channel_rs, chan_num);
440 
441 	*chan = NULL;
442 }
443 
444 
445 /*
446  * ioat_channel_intr()
447  */
448 void
ioat_channel_intr(ioat_channel_t channel)449 ioat_channel_intr(ioat_channel_t channel)
450 {
451 	ioat_state_t *state;
452 	uint16_t chanctrl;
453 	uint32_t chanerr;
454 	uint32_t status;
455 
456 
457 	state = channel->ic_state;
458 
459 	if (channel->ic_ver == IOAT_CBv1) {
460 		status = ddi_get32(state->is_reg_handle,
461 		    (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_STS_LO]);
462 	} else {
463 		ASSERT(channel->ic_ver == IOAT_CBv2);
464 		status = ddi_get32(state->is_reg_handle,
465 		    (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_STS_LO]);
466 	}
467 
468 	/* if that status isn't ACTIVE or IDLE, the channel has failed */
469 	if (status & IOAT_CHAN_STS_FAIL_MASK) {
470 		chanerr = ddi_get32(state->is_reg_handle,
471 		    (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR]);
472 		cmn_err(CE_WARN, "channel(%d) fatal failure! "
473 		    "chanstat_lo=0x%X; chanerr=0x%X\n",
474 		    channel->ic_chan_num, status, chanerr);
475 		channel->ic_channel_state = IOAT_CHANNEL_IN_FAILURE;
476 		ioat_channel_reset(channel);
477 
478 		return;
479 	}
480 
481 	/*
482 	 * clear interrupt disable bit if set (it's a RW1C). Read it back to
483 	 * ensure the write completes.
484 	 */
485 	chanctrl = ddi_get16(state->is_reg_handle,
486 	    (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL]);
487 	ddi_put16(state->is_reg_handle,
488 	    (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], chanctrl);
489 	(void) ddi_get16(state->is_reg_handle,
490 	    (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL]);
491 
492 	/* tell dcopy we have seen a completion on this channel */
493 	dcopy_device_channel_notify(channel->ic_dcopy_handle, DCOPY_COMPLETION);
494 }
495 
496 
497 /*
498  * ioat_channel_start()
499  */
500 void
ioat_channel_start(ioat_channel_t channel)501 ioat_channel_start(ioat_channel_t channel)
502 {
503 	ioat_chan_dma_desc_t desc;
504 
505 	/* set the first descriptor up as a NULL descriptor */
506 	bzero(&desc, sizeof (desc));
507 	desc.dd_size = 0;
508 	desc.dd_ctrl = IOAT_DESC_CTRL_OP_DMA | IOAT_DESC_DMACTRL_NULL |
509 	    IOAT_DESC_CTRL_CMPL;
510 	desc.dd_next_desc = 0x0;
511 
512 	/* setup the very first descriptor */
513 	ioat_ring_seed(channel, &desc);
514 }
515 
516 
517 /*
518  * ioat_channel_reset()
519  */
520 void
ioat_channel_reset(ioat_channel_t channel)521 ioat_channel_reset(ioat_channel_t channel)
522 {
523 	ioat_state_t *state;
524 
525 	state = channel->ic_state;
526 
527 	/* hit the reset bit */
528 	if (channel->ic_ver == IOAT_CBv1) {
529 		ddi_put8(state->is_reg_handle,
530 		    &channel->ic_regs[IOAT_V1_CHAN_CMD], 0x20);
531 	} else {
532 		ASSERT(channel->ic_ver == IOAT_CBv2);
533 		ddi_put8(state->is_reg_handle,
534 		    &channel->ic_regs[IOAT_V2_CHAN_CMD], 0x20);
535 	}
536 }
537 
538 
539 /*
540  * ioat_completion_alloc()
541  */
542 int
ioat_completion_alloc(ioat_channel_t channel)543 ioat_completion_alloc(ioat_channel_t channel)
544 {
545 	ioat_state_t *state;
546 	size_t real_length;
547 	uint_t cookie_cnt;
548 	int e;
549 
550 
551 	state = channel->ic_state;
552 
553 	/*
554 	 * allocate memory for the completion status, zero it out, and get
555 	 * the paddr. We'll allocate a physically contiguous cache line.
556 	 */
557 	e = ddi_dma_alloc_handle(state->is_dip, &ioat_cmpl_dma_attr,
558 	    DDI_DMA_SLEEP, NULL, &channel->ic_cmpl_dma_handle);
559 	if (e != DDI_SUCCESS) {
560 		goto cmplallocfail_alloc_handle;
561 	}
562 	channel->ic_cmpl_alloc_size = 64;
563 	e = ddi_dma_mem_alloc(channel->ic_cmpl_dma_handle,
564 	    channel->ic_cmpl_alloc_size, &ioat_acc_attr,
565 	    DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
566 	    (caddr_t *)&channel->ic_cmpl, &real_length,
567 	    &channel->ic_cmpl_handle);
568 	if (e != DDI_SUCCESS) {
569 		goto cmplallocfail_mem_alloc;
570 	}
571 	bzero((void *)channel->ic_cmpl, channel->ic_cmpl_alloc_size);
572 	e = ddi_dma_addr_bind_handle(channel->ic_cmpl_dma_handle, NULL,
573 	    (caddr_t)channel->ic_cmpl, channel->ic_cmpl_alloc_size,
574 	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
575 	    &channel->ic_cmpl_cookie, &cookie_cnt);
576 	if (e != DDI_SUCCESS) {
577 		goto cmplallocfail_addr_bind;
578 	}
579 	ASSERT(cookie_cnt == 1);
580 	ASSERT(channel->ic_cmpl_cookie.dmac_size ==
581 	    channel->ic_cmpl_alloc_size);
582 	channel->ic_phys_cmpl = channel->ic_cmpl_cookie.dmac_laddress;
583 
584 	/* write the physical address into the completion address register */
585 	ddi_put32(state->is_reg_handle,
586 	    (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_LO],
587 	    (uint32_t)(channel->ic_phys_cmpl & 0xffffffff));
588 	ddi_put32(state->is_reg_handle,
589 	    (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_HI],
590 	    (uint32_t)(channel->ic_phys_cmpl >> 32));
591 
592 	return (DDI_SUCCESS);
593 
594 cmplallocfail_addr_bind:
595 	ddi_dma_mem_free(&channel->ic_desc_handle);
596 cmplallocfail_mem_alloc:
597 	ddi_dma_free_handle(&channel->ic_desc_dma_handle);
598 cmplallocfail_alloc_handle:
599 	return (DDI_FAILURE);
600 }
601 
602 
603 /*
604  * ioat_completion_free()
605  */
606 void
ioat_completion_free(ioat_channel_t channel)607 ioat_completion_free(ioat_channel_t channel)
608 {
609 	ioat_state_t *state;
610 
611 	state = channel->ic_state;
612 
613 	/* reset the completion address register */
614 	ddi_put32(state->is_reg_handle,
615 	    (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_LO], 0x0);
616 	ddi_put32(state->is_reg_handle,
617 	    (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_HI], 0x0);
618 
619 	/* unbind, then free up the memory, dma handle */
620 	(void) ddi_dma_unbind_handle(channel->ic_cmpl_dma_handle);
621 	ddi_dma_mem_free(&channel->ic_cmpl_handle);
622 	ddi_dma_free_handle(&channel->ic_cmpl_dma_handle);
623 }
624 
625 /*
626  * ioat_ring_alloc()
627  */
628 int
ioat_ring_alloc(ioat_channel_t channel,uint_t desc_cnt)629 ioat_ring_alloc(ioat_channel_t channel, uint_t desc_cnt)
630 {
631 	ioat_channel_ring_t *ring;
632 	ioat_state_t *state;
633 	size_t real_length;
634 	uint_t cookie_cnt;
635 	int e;
636 
637 
638 	state = channel->ic_state;
639 
640 	ring = kmem_zalloc(sizeof (ioat_channel_ring_t), KM_SLEEP);
641 	channel->ic_ring = ring;
642 	ring->cr_chan = channel;
643 	ring->cr_post_cnt = 0;
644 
645 	mutex_init(&ring->cr_cmpl_mutex, NULL, MUTEX_DRIVER,
646 	    channel->ic_state->is_iblock_cookie);
647 	mutex_init(&ring->cr_desc_mutex, NULL, MUTEX_DRIVER,
648 	    channel->ic_state->is_iblock_cookie);
649 
650 	/*
651 	 * allocate memory for the ring, zero it out, and get the paddr.
652 	 * We'll allocate a physically contiguous chunck of memory  which
653 	 * simplifies the completion logic.
654 	 */
655 	e = ddi_dma_alloc_handle(state->is_dip, &ioat_desc_dma_attr,
656 	    DDI_DMA_SLEEP, NULL, &channel->ic_desc_dma_handle);
657 	if (e != DDI_SUCCESS) {
658 		goto ringallocfail_alloc_handle;
659 	}
660 	/*
661 	 * allocate one extra descriptor so we can simplify the empty/full
662 	 * logic. Then round that number up to a whole multiple of 4.
663 	 */
664 	channel->ic_chan_desc_cnt = ((desc_cnt + 1) + 3) & ~0x3;
665 	ring->cr_desc_last = channel->ic_chan_desc_cnt - 1;
666 	channel->ic_desc_alloc_size = channel->ic_chan_desc_cnt *
667 	    sizeof (ioat_chan_desc_t);
668 	e = ddi_dma_mem_alloc(channel->ic_desc_dma_handle,
669 	    channel->ic_desc_alloc_size, &ioat_acc_attr,
670 	    DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
671 	    (caddr_t *)&ring->cr_desc, &real_length, &channel->ic_desc_handle);
672 	if (e != DDI_SUCCESS) {
673 		goto ringallocfail_mem_alloc;
674 	}
675 	bzero(ring->cr_desc, channel->ic_desc_alloc_size);
676 	e = ddi_dma_addr_bind_handle(channel->ic_desc_dma_handle, NULL,
677 	    (caddr_t)ring->cr_desc, channel->ic_desc_alloc_size,
678 	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
679 	    &channel->ic_desc_cookies, &cookie_cnt);
680 	if (e != DDI_SUCCESS) {
681 		goto ringallocfail_addr_bind;
682 	}
683 	ASSERT(cookie_cnt == 1);
684 	ASSERT(channel->ic_desc_cookies.dmac_size ==
685 	    channel->ic_desc_alloc_size);
686 	ring->cr_phys_desc = channel->ic_desc_cookies.dmac_laddress;
687 
688 	/* write the physical address into the chain address register */
689 	if (channel->ic_ver == IOAT_CBv1) {
690 		ddi_put32(state->is_reg_handle,
691 		    (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_LO],
692 		    (uint32_t)(ring->cr_phys_desc & 0xffffffff));
693 		ddi_put32(state->is_reg_handle,
694 		    (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_HI],
695 		    (uint32_t)(ring->cr_phys_desc >> 32));
696 	} else {
697 		ASSERT(channel->ic_ver == IOAT_CBv2);
698 		ddi_put32(state->is_reg_handle,
699 		    (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_LO],
700 		    (uint32_t)(ring->cr_phys_desc & 0xffffffff));
701 		ddi_put32(state->is_reg_handle,
702 		    (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_HI],
703 		    (uint32_t)(ring->cr_phys_desc >> 32));
704 	}
705 
706 	return (DCOPY_SUCCESS);
707 
708 ringallocfail_addr_bind:
709 	ddi_dma_mem_free(&channel->ic_desc_handle);
710 ringallocfail_mem_alloc:
711 	ddi_dma_free_handle(&channel->ic_desc_dma_handle);
712 ringallocfail_alloc_handle:
713 	mutex_destroy(&ring->cr_desc_mutex);
714 	mutex_destroy(&ring->cr_cmpl_mutex);
715 	kmem_free(channel->ic_ring, sizeof (ioat_channel_ring_t));
716 
717 	return (DCOPY_FAILURE);
718 }
719 
720 
721 /*
722  * ioat_ring_free()
723  */
724 void
ioat_ring_free(ioat_channel_t channel)725 ioat_ring_free(ioat_channel_t channel)
726 {
727 	ioat_state_t *state;
728 
729 
730 	state = channel->ic_state;
731 
732 	/* reset the chain address register */
733 	if (channel->ic_ver == IOAT_CBv1) {
734 		ddi_put32(state->is_reg_handle,
735 		    (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_LO], 0x0);
736 		ddi_put32(state->is_reg_handle,
737 		    (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_HI], 0x0);
738 	} else {
739 		ASSERT(channel->ic_ver == IOAT_CBv2);
740 		ddi_put32(state->is_reg_handle,
741 		    (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_LO], 0x0);
742 		ddi_put32(state->is_reg_handle,
743 		    (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_HI], 0x0);
744 	}
745 
746 	/* unbind, then free up the memory, dma handle */
747 	(void) ddi_dma_unbind_handle(channel->ic_desc_dma_handle);
748 	ddi_dma_mem_free(&channel->ic_desc_handle);
749 	ddi_dma_free_handle(&channel->ic_desc_dma_handle);
750 
751 	mutex_destroy(&channel->ic_ring->cr_desc_mutex);
752 	mutex_destroy(&channel->ic_ring->cr_cmpl_mutex);
753 	kmem_free(channel->ic_ring, sizeof (ioat_channel_ring_t));
754 
755 }
756 
757 
758 /*
759  * ioat_ring_seed()
760  *    write the first descriptor in the ring.
761  */
762 void
ioat_ring_seed(ioat_channel_t channel,ioat_chan_dma_desc_t * in_desc)763 ioat_ring_seed(ioat_channel_t channel, ioat_chan_dma_desc_t *in_desc)
764 {
765 	ioat_channel_ring_t *ring;
766 	ioat_chan_dma_desc_t *desc;
767 	ioat_chan_dma_desc_t *prev;
768 	ioat_state_t *state;
769 
770 
771 	state = channel->ic_state;
772 	ring = channel->ic_ring;
773 
774 	/* init the completion state */
775 	ring->cr_cmpl_gen = 0x0;
776 	ring->cr_cmpl_last = 0x0;
777 
778 	/* write in the descriptor and init the descriptor state */
779 	ring->cr_post_cnt++;
780 	channel->ic_ring->cr_desc[0] = *(ioat_chan_desc_t *)in_desc;
781 	ring->cr_desc_gen = 0;
782 	ring->cr_desc_prev = 0;
783 	ring->cr_desc_next = 1;
784 
785 	if (channel->ic_ver == IOAT_CBv1) {
786 		/* hit the start bit */
787 		ddi_put8(state->is_reg_handle,
788 		    &channel->ic_regs[IOAT_V1_CHAN_CMD], 0x1);
789 	} else {
790 		/*
791 		 * if this is CBv2, link the descriptor to an empty
792 		 * descriptor
793 		 */
794 		ASSERT(ring->cr_chan->ic_ver == IOAT_CBv2);
795 		desc = (ioat_chan_dma_desc_t *)
796 		    &ring->cr_desc[ring->cr_desc_next];
797 		prev = (ioat_chan_dma_desc_t *)
798 		    &ring->cr_desc[ring->cr_desc_prev];
799 
800 		desc->dd_ctrl = 0;
801 		desc->dd_next_desc = 0x0;
802 
803 		prev->dd_next_desc = ring->cr_phys_desc +
804 		    (ring->cr_desc_next << 6);
805 
806 		ddi_put16(state->is_reg_handle,
807 		    (uint16_t *)&channel->ic_regs[IOAT_V2_CHAN_CNT],
808 		    (uint16_t)1);
809 	}
810 
811 }
812 
813 /*
814  * ioat_ring_loop()
815  * Make the ring loop for CB v1
816  * This function assume we are in the ring->cr_desc_mutex mutex context
817  */
818 int
ioat_ring_loop(ioat_channel_ring_t * ring,dcopy_cmd_t cmd)819 ioat_ring_loop(ioat_channel_ring_t *ring, dcopy_cmd_t cmd)
820 {
821 	uint64_t count;
822 	ioat_channel_t channel;
823 	ioat_chan_dma_desc_t *curr;
824 	ioat_cmd_private_t *prevpriv;
825 	ioat_cmd_private_t *currpriv;
826 
827 	currpriv = NULL;
828 	channel = ring->cr_chan;
829 	ASSERT(channel->ic_ver == IOAT_CBv1);
830 
831 	/*
832 	 * For each cmd in the command queue, check whether they are continuous
833 	 * in descriptor ring. Return error if not continuous.
834 	 */
835 	for (count = 0, prevpriv = NULL;
836 	    cmd != NULL && count <= channel->ic_chan_desc_cnt;
837 	    prevpriv = currpriv) {
838 		currpriv = cmd->dp_private->pr_device_cmd_private;
839 		if (prevpriv != NULL &&
840 		    currpriv->ip_index + 1 != prevpriv->ip_start &&
841 		    currpriv->ip_index + 1 != prevpriv->ip_start +
842 		    channel->ic_chan_desc_cnt) {
843 			/* Non-continuous, other commands get interleaved */
844 			return (DCOPY_FAILURE);
845 		}
846 		if (currpriv->ip_index < currpriv->ip_start) {
847 			count += channel->ic_chan_desc_cnt
848 			    + currpriv->ip_index - currpriv->ip_start + 1;
849 		} else {
850 			count += currpriv->ip_index - currpriv->ip_start + 1;
851 		}
852 		cmd = currpriv->ip_next;
853 	}
854 	/*
855 	 * Check for too many descriptors which would cause wrap around in
856 	 * descriptor ring. And make sure there is space for cancel operation.
857 	 */
858 	if (count >= channel->ic_chan_desc_cnt) {
859 		return (DCOPY_FAILURE);
860 	}
861 
862 	/* Point next descriptor to header of chain. */
863 	curr = (ioat_chan_dma_desc_t *)&ring->cr_desc[ring->cr_desc_prev];
864 	curr->dd_next_desc = ring->cr_phys_desc + (currpriv->ip_start << 6);
865 
866 	/* sync the last desc */
867 	(void) ddi_dma_sync(channel->ic_desc_dma_handle,
868 	    ring->cr_desc_prev << 6, 64, DDI_DMA_SYNC_FORDEV);
869 
870 	return (DCOPY_SUCCESS);
871 }
872 
873 
874 /*
875  * ioat_cmd_alloc()
876  */
877 int
ioat_cmd_alloc(void * private,int flags,dcopy_cmd_t * cmd)878 ioat_cmd_alloc(void *private, int flags, dcopy_cmd_t *cmd)
879 {
880 	ioat_cmd_private_t *priv;
881 	ioat_channel_t channel;
882 	dcopy_cmd_t oldcmd;
883 	int kmflag;
884 
885 
886 	channel = (ioat_channel_t)private;
887 
888 	if (flags & DCOPY_NOSLEEP) {
889 		kmflag = KM_NOSLEEP;
890 	} else {
891 		kmflag = KM_SLEEP;
892 	}
893 
894 	/* save the command passed incase DCOPY_ALLOC_LINK is set */
895 	oldcmd = *cmd;
896 
897 	*cmd = kmem_cache_alloc(channel->ic_cmd_cache, kmflag);
898 	if (*cmd == NULL) {
899 		return (DCOPY_NORESOURCES);
900 	}
901 
902 	/* setup the dcopy and ioat private state pointers */
903 	(*cmd)->dp_version = DCOPY_CMD_V0;
904 	(*cmd)->dp_cmd = 0;
905 	(*cmd)->dp_private = (struct dcopy_cmd_priv_s *)
906 	    ((uintptr_t)(*cmd) + sizeof (struct dcopy_cmd_s));
907 	(*cmd)->dp_private->pr_device_cmd_private =
908 	    (struct ioat_cmd_private_s *)((uintptr_t)(*cmd)->dp_private +
909 	    sizeof (struct dcopy_cmd_priv_s));
910 
911 	/*
912 	 * if DCOPY_ALLOC_LINK is set, link the old command to the new one
913 	 * just allocated.
914 	 */
915 	priv = (*cmd)->dp_private->pr_device_cmd_private;
916 	if (flags & DCOPY_ALLOC_LINK) {
917 		priv->ip_next = oldcmd;
918 	} else {
919 		priv->ip_next = NULL;
920 	}
921 
922 	return (DCOPY_SUCCESS);
923 }
924 
925 
926 /*
927  * ioat_cmd_free()
928  */
929 void
ioat_cmd_free(void * private,dcopy_cmd_t * cmdp)930 ioat_cmd_free(void *private, dcopy_cmd_t *cmdp)
931 {
932 	ioat_cmd_private_t *priv;
933 	ioat_channel_t channel;
934 	dcopy_cmd_t next;
935 	dcopy_cmd_t cmd;
936 
937 
938 	channel = (ioat_channel_t)private;
939 	cmd = *(cmdp);
940 
941 	/*
942 	 * free all the commands in the chain (see DCOPY_ALLOC_LINK in
943 	 * ioat_cmd_alloc() for more info).
944 	 */
945 	while (cmd != NULL) {
946 		priv = cmd->dp_private->pr_device_cmd_private;
947 		next = priv->ip_next;
948 		kmem_cache_free(channel->ic_cmd_cache, cmd);
949 		cmd = next;
950 	}
951 	*cmdp = NULL;
952 }
953 
954 
955 /*
956  * ioat_cmd_post()
957  */
958 int
ioat_cmd_post(void * private,dcopy_cmd_t cmd)959 ioat_cmd_post(void *private, dcopy_cmd_t cmd)
960 {
961 	ioat_channel_ring_t *ring;
962 	ioat_cmd_private_t *priv;
963 	ioat_channel_t channel;
964 	ioat_state_t *state;
965 	uint64_t dest_paddr;
966 	uint64_t src_paddr;
967 	uint64_t dest_addr;
968 	uint32_t dest_size;
969 	uint64_t src_addr;
970 	uint32_t src_size;
971 	size_t xfer_size;
972 	uint32_t ctrl;
973 	size_t size;
974 	int e;
975 
976 
977 	channel = (ioat_channel_t)private;
978 	priv = cmd->dp_private->pr_device_cmd_private;
979 
980 	state = channel->ic_state;
981 	ring = channel->ic_ring;
982 
983 	/*
984 	 * Special support for DCOPY_CMD_LOOP option, only supported on CBv1.
985 	 * DCOPY_CMD_QUEUE should also be set if DCOPY_CMD_LOOP is set.
986 	 */
987 	if ((cmd->dp_flags & DCOPY_CMD_LOOP) &&
988 	    (channel->ic_ver != IOAT_CBv1 ||
989 	    (cmd->dp_flags & DCOPY_CMD_QUEUE))) {
990 		return (DCOPY_FAILURE);
991 	}
992 
993 	if ((cmd->dp_flags & DCOPY_CMD_NOWAIT) == 0) {
994 		mutex_enter(&ring->cr_desc_mutex);
995 
996 	/*
997 	 * Try to acquire mutex if NOWAIT flag is set.
998 	 * Return failure if failed to acquire mutex.
999 	 */
1000 	} else if (mutex_tryenter(&ring->cr_desc_mutex) == 0) {
1001 		return (DCOPY_FAILURE);
1002 	}
1003 
1004 	/* if the channel has had a fatal failure, return failure */
1005 	if (channel->ic_channel_state == IOAT_CHANNEL_IN_FAILURE) {
1006 		mutex_exit(&ring->cr_desc_mutex);
1007 		return (DCOPY_FAILURE);
1008 	}
1009 
1010 	/* make sure we have space for the descriptors */
1011 	e = ioat_ring_reserve(channel, ring, cmd);
1012 	if (e != DCOPY_SUCCESS) {
1013 		mutex_exit(&ring->cr_desc_mutex);
1014 		return (DCOPY_NORESOURCES);
1015 	}
1016 
1017 	/* if we support DCA, and the DCA flag is set, post a DCA desc */
1018 	if ((channel->ic_ver == IOAT_CBv2) &&
1019 	    (cmd->dp_flags & DCOPY_CMD_DCA)) {
1020 		ioat_cmd_post_dca(ring, cmd->dp_dca_id);
1021 	}
1022 
1023 	/*
1024 	 * the dma copy may have to be broken up into multiple descriptors
1025 	 * since we can't cross a page boundary.
1026 	 */
1027 	ASSERT(cmd->dp_version == DCOPY_CMD_V0);
1028 	ASSERT(cmd->dp_cmd == DCOPY_CMD_COPY);
1029 	src_addr = cmd->dp.copy.cc_source;
1030 	dest_addr = cmd->dp.copy.cc_dest;
1031 	size = cmd->dp.copy.cc_size;
1032 	priv->ip_start = ring->cr_desc_next;
1033 	while (size > 0) {
1034 		src_paddr = pa_to_ma(src_addr);
1035 		dest_paddr = pa_to_ma(dest_addr);
1036 
1037 		/* adjust for any offset into the page */
1038 		if ((src_addr & PAGEOFFSET) == 0) {
1039 			src_size = PAGESIZE;
1040 		} else {
1041 			src_size = PAGESIZE - (src_addr & PAGEOFFSET);
1042 		}
1043 		if ((dest_addr & PAGEOFFSET) == 0) {
1044 			dest_size = PAGESIZE;
1045 		} else {
1046 			dest_size = PAGESIZE - (dest_addr & PAGEOFFSET);
1047 		}
1048 
1049 		/* take the smallest of the three */
1050 		xfer_size = MIN(src_size, dest_size);
1051 		xfer_size = MIN(xfer_size, size);
1052 
1053 		/*
1054 		 * if this is the last descriptor, and we are supposed to
1055 		 * generate a completion, generate a completion. same logic
1056 		 * for interrupt.
1057 		 */
1058 		ctrl = 0;
1059 		if (cmd->dp_flags & DCOPY_CMD_NOSRCSNP) {
1060 			ctrl |= IOAT_DESC_CTRL_NOSRCSNP;
1061 		}
1062 		if (cmd->dp_flags & DCOPY_CMD_NODSTSNP) {
1063 			ctrl |= IOAT_DESC_CTRL_NODSTSNP;
1064 		}
1065 		if (xfer_size == size) {
1066 			if (!(cmd->dp_flags & DCOPY_CMD_NOSTAT)) {
1067 				ctrl |= IOAT_DESC_CTRL_CMPL;
1068 			}
1069 			if ((cmd->dp_flags & DCOPY_CMD_INTR)) {
1070 				ctrl |= IOAT_DESC_CTRL_INTR;
1071 			}
1072 		}
1073 
1074 		ioat_cmd_post_copy(ring, src_paddr, dest_paddr, xfer_size,
1075 		    ctrl);
1076 
1077 		/* go to the next page */
1078 		src_addr += xfer_size;
1079 		dest_addr += xfer_size;
1080 		size -= xfer_size;
1081 	}
1082 
1083 	/* save away the state so we can poll on it. */
1084 	priv->ip_generation = ring->cr_desc_gen_prev;
1085 	priv->ip_index = ring->cr_desc_prev;
1086 
1087 	/* if queue not defined, tell the DMA engine about it */
1088 	if (!(cmd->dp_flags & DCOPY_CMD_QUEUE)) {
1089 		/*
1090 		 * Link the ring to a loop (currently only for FIPE).
1091 		 */
1092 		if (cmd->dp_flags & DCOPY_CMD_LOOP) {
1093 			e = ioat_ring_loop(ring, cmd);
1094 			if (e != DCOPY_SUCCESS) {
1095 				mutex_exit(&ring->cr_desc_mutex);
1096 				return (DCOPY_FAILURE);
1097 			}
1098 		}
1099 
1100 		if (channel->ic_ver == IOAT_CBv1) {
1101 			ddi_put8(state->is_reg_handle,
1102 			    (uint8_t *)&channel->ic_regs[IOAT_V1_CHAN_CMD],
1103 			    0x2);
1104 		} else {
1105 			ASSERT(channel->ic_ver == IOAT_CBv2);
1106 			ddi_put16(state->is_reg_handle,
1107 			    (uint16_t *)&channel->ic_regs[IOAT_V2_CHAN_CNT],
1108 			    (uint16_t)(ring->cr_post_cnt & 0xFFFF));
1109 		}
1110 	}
1111 
1112 	mutex_exit(&ring->cr_desc_mutex);
1113 
1114 	return (DCOPY_SUCCESS);
1115 }
1116 
1117 
1118 /*
1119  * ioat_cmd_post_dca()
1120  */
1121 static void
ioat_cmd_post_dca(ioat_channel_ring_t * ring,uint32_t dca_id)1122 ioat_cmd_post_dca(ioat_channel_ring_t *ring, uint32_t dca_id)
1123 {
1124 	ioat_chan_dca_desc_t *saved_prev;
1125 	ioat_chan_dca_desc_t *desc;
1126 	ioat_chan_dca_desc_t *prev;
1127 	ioat_channel_t channel;
1128 	uint64_t next_desc_phys;
1129 	off_t prev_offset;
1130 	off_t next_offset;
1131 
1132 
1133 	channel = ring->cr_chan;
1134 	desc = (ioat_chan_dca_desc_t *)&ring->cr_desc[ring->cr_desc_next];
1135 	prev = (ioat_chan_dca_desc_t *)&ring->cr_desc[ring->cr_desc_prev];
1136 
1137 	/* keep track of the number of descs posted for cbv2 */
1138 	ring->cr_post_cnt++;
1139 
1140 	/*
1141 	 * post a context change desriptor. If dca has never been used on
1142 	 * this channel, or if the id doesn't match the last id used on this
1143 	 * channel, set CONTEXT_CHANGE bit and dca id, set dca state to active,
1144 	 * and save away the id we're using.
1145 	 */
1146 	desc->dd_ctrl = IOAT_DESC_CTRL_OP_CNTX;
1147 	desc->dd_next_desc = 0x0;
1148 	if (!channel->ic_dca_active || (channel->ic_dca_current != dca_id)) {
1149 		channel->ic_dca_active = B_TRUE;
1150 		channel->ic_dca_current = dca_id;
1151 		desc->dd_ctrl |= IOAT_DESC_CTRL_CNTX_CHNG;
1152 		desc->dd_cntx = dca_id;
1153 	}
1154 
1155 	/*
1156 	 * save next desc and prev offset for when we link the two
1157 	 * descriptors together.
1158 	 */
1159 	saved_prev = prev;
1160 	prev_offset = ring->cr_desc_prev << 6;
1161 	next_offset = ring->cr_desc_next << 6;
1162 	next_desc_phys = ring->cr_phys_desc + next_offset;
1163 
1164 	/* save the current desc_next and desc_last for the completion */
1165 	ring->cr_desc_prev = ring->cr_desc_next;
1166 	ring->cr_desc_gen_prev = ring->cr_desc_gen;
1167 
1168 	/* increment next/gen so it points to the next free desc */
1169 	ring->cr_desc_next++;
1170 	if (ring->cr_desc_next > ring->cr_desc_last) {
1171 		ring->cr_desc_next = 0;
1172 		ring->cr_desc_gen++;
1173 	}
1174 
1175 	/*
1176 	 * if this is CBv2, link the descriptor to an empty descriptor. Since
1177 	 * we always leave on desc empty to detect full, this works out.
1178 	 */
1179 	if (ring->cr_chan->ic_ver == IOAT_CBv2) {
1180 		desc = (ioat_chan_dca_desc_t *)
1181 		    &ring->cr_desc[ring->cr_desc_next];
1182 		prev = (ioat_chan_dca_desc_t *)
1183 		    &ring->cr_desc[ring->cr_desc_prev];
1184 		desc->dd_ctrl = 0;
1185 		desc->dd_next_desc = 0x0;
1186 		(void) ddi_dma_sync(channel->ic_desc_dma_handle,
1187 		    ring->cr_desc_next << 6, 64, DDI_DMA_SYNC_FORDEV);
1188 		prev->dd_next_desc = ring->cr_phys_desc +
1189 		    (ring->cr_desc_next << 6);
1190 	}
1191 
1192 	/* Put the descriptors physical address in the previous descriptor */
1193 	/*LINTED:E_TRUE_LOGICAL_EXPR*/
1194 	ASSERT(sizeof (ioat_chan_dca_desc_t) == 64);
1195 
1196 	/* sync the current desc */
1197 	(void) ddi_dma_sync(channel->ic_desc_dma_handle, next_offset, 64,
1198 	    DDI_DMA_SYNC_FORDEV);
1199 
1200 	/* update the previous desc and sync it too */
1201 	saved_prev->dd_next_desc = next_desc_phys;
1202 	(void) ddi_dma_sync(channel->ic_desc_dma_handle, prev_offset, 64,
1203 	    DDI_DMA_SYNC_FORDEV);
1204 }
1205 
1206 
1207 /*
1208  * ioat_cmd_post_copy()
1209  *
1210  */
1211 static void
ioat_cmd_post_copy(ioat_channel_ring_t * ring,uint64_t src_addr,uint64_t dest_addr,uint32_t size,uint32_t ctrl)1212 ioat_cmd_post_copy(ioat_channel_ring_t *ring, uint64_t src_addr,
1213     uint64_t dest_addr, uint32_t size, uint32_t ctrl)
1214 {
1215 	ioat_chan_dma_desc_t *saved_prev;
1216 	ioat_chan_dma_desc_t *desc;
1217 	ioat_chan_dma_desc_t *prev;
1218 	ioat_channel_t channel;
1219 	uint64_t next_desc_phy;
1220 	off_t prev_offset;
1221 	off_t next_offset;
1222 
1223 
1224 	channel = ring->cr_chan;
1225 	desc = (ioat_chan_dma_desc_t *)&ring->cr_desc[ring->cr_desc_next];
1226 	prev = (ioat_chan_dma_desc_t *)&ring->cr_desc[ring->cr_desc_prev];
1227 
1228 	/* keep track of the number of descs posted for cbv2 */
1229 	ring->cr_post_cnt++;
1230 
1231 	/* write in the DMA desc */
1232 	desc->dd_ctrl = IOAT_DESC_CTRL_OP_DMA | ctrl;
1233 	desc->dd_size = size;
1234 	desc->dd_src_paddr = src_addr;
1235 	desc->dd_dest_paddr = dest_addr;
1236 	desc->dd_next_desc = 0x0;
1237 
1238 	/*
1239 	 * save next desc and prev offset for when we link the two
1240 	 * descriptors together.
1241 	 */
1242 	saved_prev = prev;
1243 	prev_offset = ring->cr_desc_prev << 6;
1244 	next_offset = ring->cr_desc_next << 6;
1245 	next_desc_phy = ring->cr_phys_desc + next_offset;
1246 
1247 	/* increment next/gen so it points to the next free desc */
1248 	ring->cr_desc_prev = ring->cr_desc_next;
1249 	ring->cr_desc_gen_prev = ring->cr_desc_gen;
1250 
1251 	/* increment next/gen so it points to the next free desc */
1252 	ring->cr_desc_next++;
1253 	if (ring->cr_desc_next > ring->cr_desc_last) {
1254 		ring->cr_desc_next = 0;
1255 		ring->cr_desc_gen++;
1256 	}
1257 
1258 	/*
1259 	 * if this is CBv2, link the descriptor to an empty descriptor. Since
1260 	 * we always leave on desc empty to detect full, this works out.
1261 	 */
1262 	if (ring->cr_chan->ic_ver == IOAT_CBv2) {
1263 		desc = (ioat_chan_dma_desc_t *)
1264 		    &ring->cr_desc[ring->cr_desc_next];
1265 		prev = (ioat_chan_dma_desc_t *)
1266 		    &ring->cr_desc[ring->cr_desc_prev];
1267 		desc->dd_size = 0;
1268 		desc->dd_ctrl = 0;
1269 		desc->dd_next_desc = 0x0;
1270 		(void) ddi_dma_sync(channel->ic_desc_dma_handle,
1271 		    ring->cr_desc_next << 6, 64, DDI_DMA_SYNC_FORDEV);
1272 		prev->dd_next_desc = ring->cr_phys_desc +
1273 		    (ring->cr_desc_next << 6);
1274 	}
1275 
1276 	/* Put the descriptors physical address in the previous descriptor */
1277 	/*LINTED:E_TRUE_LOGICAL_EXPR*/
1278 	ASSERT(sizeof (ioat_chan_dma_desc_t) == 64);
1279 
1280 	/* sync the current desc */
1281 	(void) ddi_dma_sync(channel->ic_desc_dma_handle, next_offset, 64,
1282 	    DDI_DMA_SYNC_FORDEV);
1283 
1284 	/* update the previous desc and sync it too */
1285 	saved_prev->dd_next_desc = next_desc_phy;
1286 	(void) ddi_dma_sync(channel->ic_desc_dma_handle, prev_offset, 64,
1287 	    DDI_DMA_SYNC_FORDEV);
1288 }
1289 
1290 
1291 /*
1292  * ioat_cmd_poll()
1293  */
1294 int
ioat_cmd_poll(void * private,dcopy_cmd_t cmd)1295 ioat_cmd_poll(void *private, dcopy_cmd_t cmd)
1296 {
1297 	ioat_channel_ring_t *ring;
1298 	ioat_cmd_private_t *priv;
1299 	ioat_channel_t channel;
1300 	uint64_t generation;
1301 	uint64_t last_cmpl;
1302 
1303 	ASSERT(cmd != NULL);
1304 	channel = (ioat_channel_t)private;
1305 	priv = cmd->dp_private->pr_device_cmd_private;
1306 
1307 	ring = channel->ic_ring;
1308 	ASSERT(ring != NULL);
1309 
1310 	if ((cmd->dp_flags & DCOPY_CMD_NOWAIT) == 0) {
1311 		mutex_enter(&ring->cr_cmpl_mutex);
1312 
1313 	/*
1314 	 * Try to acquire mutex if NOWAIT flag is set.
1315 	 * Return failure if failed to acquire mutex.
1316 	 */
1317 	} else if (mutex_tryenter(&ring->cr_cmpl_mutex) == 0) {
1318 		return (DCOPY_FAILURE);
1319 	}
1320 
1321 	/* if the channel had a fatal failure, fail all polls */
1322 	if ((channel->ic_channel_state == IOAT_CHANNEL_IN_FAILURE) ||
1323 	    IOAT_CMPL_FAILED(channel)) {
1324 		mutex_exit(&ring->cr_cmpl_mutex);
1325 		return (DCOPY_FAILURE);
1326 	}
1327 
1328 	/*
1329 	 * if the current completion is the same as the last time we read one,
1330 	 * post is still pending, nothing further to do. We track completions
1331 	 * as indexes into the ring since post uses VAs and the H/W returns
1332 	 * PAs. We grab a snapshot of generation and last_cmpl in the mutex.
1333 	 */
1334 	(void) ddi_dma_sync(channel->ic_cmpl_dma_handle, 0, 0,
1335 	    DDI_DMA_SYNC_FORCPU);
1336 	last_cmpl = IOAT_CMPL_INDEX(channel);
1337 	if (last_cmpl != ring->cr_cmpl_last) {
1338 		/*
1339 		 * if we wrapped the ring, increment the generation. Store
1340 		 * the last cmpl. This logic assumes a physically contiguous
1341 		 * ring.
1342 		 */
1343 		if (last_cmpl < ring->cr_cmpl_last) {
1344 			ring->cr_cmpl_gen++;
1345 		}
1346 		ring->cr_cmpl_last = last_cmpl;
1347 		generation = ring->cr_cmpl_gen;
1348 
1349 	} else {
1350 		generation = ring->cr_cmpl_gen;
1351 	}
1352 
1353 	mutex_exit(&ring->cr_cmpl_mutex);
1354 
1355 	/*
1356 	 * if cmd isn't passed in, well return.  Useful for updating the
1357 	 * consumer pointer (ring->cr_cmpl_last).
1358 	 */
1359 	if (cmd->dp_flags & DCOPY_CMD_SYNC) {
1360 		return (DCOPY_PENDING);
1361 	}
1362 
1363 	/*
1364 	 * if the post's generation is old, this post has completed. No reason
1365 	 * to go check the last completion. if the generation is the same
1366 	 * and if the post is before or = to the last completion processed,
1367 	 * the post has completed.
1368 	 */
1369 	if (priv->ip_generation < generation) {
1370 		return (DCOPY_COMPLETED);
1371 	} else if ((priv->ip_generation == generation) &&
1372 	    (priv->ip_index <= last_cmpl)) {
1373 		return (DCOPY_COMPLETED);
1374 	}
1375 
1376 	return (DCOPY_PENDING);
1377 }
1378 
1379 
1380 /*
1381  * ioat_ring_reserve()
1382  */
1383 int
ioat_ring_reserve(ioat_channel_t channel,ioat_channel_ring_t * ring,dcopy_cmd_t cmd)1384 ioat_ring_reserve(ioat_channel_t channel, ioat_channel_ring_t *ring,
1385     dcopy_cmd_t cmd)
1386 {
1387 	uint64_t dest_addr;
1388 	uint32_t dest_size;
1389 	uint64_t src_addr;
1390 	uint32_t src_size;
1391 	size_t xfer_size;
1392 	uint64_t desc;
1393 	int num_desc;
1394 	size_t size;
1395 	int i;
1396 
1397 
1398 	/*
1399 	 * figure out how many descriptors we need. This can include a dca
1400 	 * desc and multiple desc for a dma copy.
1401 	 */
1402 	num_desc = 0;
1403 	if ((channel->ic_ver == IOAT_CBv2) &&
1404 	    (cmd->dp_flags & DCOPY_CMD_DCA)) {
1405 		num_desc++;
1406 	}
1407 	src_addr = cmd->dp.copy.cc_source;
1408 	dest_addr = cmd->dp.copy.cc_dest;
1409 	size = cmd->dp.copy.cc_size;
1410 	while (size > 0) {
1411 		num_desc++;
1412 
1413 		/* adjust for any offset into the page */
1414 		if ((src_addr & PAGEOFFSET) == 0) {
1415 			src_size = PAGESIZE;
1416 		} else {
1417 			src_size = PAGESIZE - (src_addr & PAGEOFFSET);
1418 		}
1419 		if ((dest_addr & PAGEOFFSET) == 0) {
1420 			dest_size = PAGESIZE;
1421 		} else {
1422 			dest_size = PAGESIZE - (dest_addr & PAGEOFFSET);
1423 		}
1424 
1425 		/* take the smallest of the three */
1426 		xfer_size = MIN(src_size, dest_size);
1427 		xfer_size = MIN(xfer_size, size);
1428 
1429 		/* go to the next page */
1430 		src_addr += xfer_size;
1431 		dest_addr += xfer_size;
1432 		size -= xfer_size;
1433 	}
1434 
1435 	/* Make sure we have space for these descriptors */
1436 	desc = ring->cr_desc_next;
1437 	for (i = 0; i < num_desc; i++) {
1438 
1439 		/*
1440 		 * if this is the last descriptor in the ring, see if the
1441 		 * last completed descriptor is #0.
1442 		 */
1443 		if (desc == ring->cr_desc_last) {
1444 			if (ring->cr_cmpl_last == 0) {
1445 				/*
1446 				 * if we think the ring is full, update where
1447 				 * the H/W really is and check for full again.
1448 				 */
1449 				cmd->dp_flags |= DCOPY_CMD_SYNC;
1450 				(void) ioat_cmd_poll(channel, cmd);
1451 				cmd->dp_flags &= ~DCOPY_CMD_SYNC;
1452 				if (ring->cr_cmpl_last == 0) {
1453 					return (DCOPY_NORESOURCES);
1454 				}
1455 			}
1456 
1457 			/*
1458 			 * go to the next descriptor which is zero in this
1459 			 * case.
1460 			 */
1461 			desc = 0;
1462 
1463 		/*
1464 		 * if this is not the last descriptor in the ring, see if
1465 		 * the last completion we saw was the next descriptor.
1466 		 */
1467 		} else {
1468 			if ((desc + 1) == ring->cr_cmpl_last) {
1469 				/*
1470 				 * if we think the ring is full, update where
1471 				 * the H/W really is and check for full again.
1472 				 */
1473 				cmd->dp_flags |= DCOPY_CMD_SYNC;
1474 				(void) ioat_cmd_poll(channel, cmd);
1475 				cmd->dp_flags &= ~DCOPY_CMD_SYNC;
1476 				if ((desc + 1) == ring->cr_cmpl_last) {
1477 					return (DCOPY_NORESOURCES);
1478 				}
1479 			}
1480 
1481 			/* go to the next descriptor */
1482 			desc++;
1483 		}
1484 	}
1485 
1486 	return (DCOPY_SUCCESS);
1487 }
1488