xref: /titanic_50/usr/src/uts/sun4v/io/ldc_shm.c (revision 2654012f83cec5dc15b61dfe3e4a4915f186e7a6)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * sun4v LDC Link Layer Shared Memory Routines
31  */
32 #include <sys/types.h>
33 #include <sys/kmem.h>
34 #include <sys/cmn_err.h>
35 #include <sys/ksynch.h>
36 #include <sys/debug.h>
37 #include <sys/cyclic.h>
38 #include <sys/machsystm.h>
39 #include <sys/vm.h>
40 #include <sys/machcpuvar.h>
41 #include <sys/mmu.h>
42 #include <sys/pte.h>
43 #include <vm/hat.h>
44 #include <vm/as.h>
45 #include <vm/hat_sfmmu.h>
46 #include <sys/vm_machparam.h>
47 #include <vm/seg_kmem.h>
48 #include <vm/seg_kpm.h>
49 #include <sys/hypervisor_api.h>
50 #include <sys/ldc.h>
51 #include <sys/ldc_impl.h>
52 
53 /* LDC variables used by shared memory routines */
54 extern ldc_soft_state_t *ldcssp;
55 extern int ldc_max_retries;
56 extern clock_t ldc_delay;
57 
58 #ifdef DEBUG
59 extern int ldcdbg;
60 #endif
61 
62 /* LDC internal functions used by shared memory routines */
63 extern void i_ldc_reset(ldc_chan_t *ldcp, boolean_t force_reset);
64 extern int i_ldc_h2v_error(int h_error);
65 
66 #ifdef DEBUG
67 extern void ldcdebug(int64_t id, const char *fmt, ...);
68 #endif
69 
70 /* Memory synchronization internal functions */
71 static int i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle,
72     uint8_t direction, uint64_t offset, size_t size);
73 static int i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle,
74     uint8_t direction, uint64_t start, uint64_t end);
75 static int i_ldc_mem_map(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie,
76     uint32_t ccount, uint8_t mtype, uint8_t perm, caddr_t *vaddr,
77     caddr_t *raddr);
78 static int i_ldc_mem_bind_handle(ldc_mem_handle_t mhandle, caddr_t vaddr,
79     size_t len, uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie,
80     uint32_t *ccount);
81 
82 /*
83  * LDC framework supports mapping remote domain's memory
84  * either directly or via shadow memory pages. Default
85  * support is currently implemented via shadow copy.
86  * Direct map can be enabled by setting 'ldc_shmem_enabled'
87  */
88 int ldc_shmem_enabled = 0;
89 
90 /*
91  * Use of directly mapped shared memory for LDC descriptor
92  * rings is permitted if this variable is non-zero.
93  */
94 int ldc_dring_shmem_enabled = 1;
95 
96 /*
97  * The major and minor versions required to use directly
98  * mapped shared memory for LDC descriptor rings. The
99  * ldc_dring_shmem_hv_force variable, if set to a non-zero
100  * value, overrides the hypervisor API version check.
101  */
102 static int ldc_dring_shmem_hv_major = 1;
103 static int ldc_dring_shmem_hv_minor = 1;
104 static int ldc_dring_shmem_hv_force = 0;
105 
106 /*
107  * The results of the hypervisor service group API check.
108  * A non-zero value indicates the HV includes support for
109  * descriptor ring shared memory.
110  */
111 static int ldc_dring_shmem_hv_ok = 0;
112 
113 /*
114  * Pages exported for remote access over each channel is
115  * maintained in a table registered with the Hypervisor.
116  * The default number of entries in the table is set to
117  * 'ldc_mtbl_entries'.
118  */
119 uint64_t ldc_maptable_entries = LDC_MTBL_ENTRIES;
120 
121 #define	IDX2COOKIE(idx, pg_szc, pg_shift)				\
122 	(((pg_szc) << LDC_COOKIE_PGSZC_SHIFT) | ((idx) << (pg_shift)))
123 
124 /*
125  * Sets ldc_dring_shmem_hv_ok to a non-zero value if the HV LDC
126  * API version supports directly mapped shared memory or if it has
127  * been explicitly enabled via ldc_dring_shmem_hv_force.
128  */
129 void
130 i_ldc_mem_set_hsvc_vers(uint64_t major, uint64_t minor)
131 {
132 	if ((major == ldc_dring_shmem_hv_major &&
133 	    minor >= ldc_dring_shmem_hv_minor) ||
134 	    (major > ldc_dring_shmem_hv_major) ||
135 	    (ldc_dring_shmem_hv_force != 0)) {
136 		ldc_dring_shmem_hv_ok = 1;
137 	}
138 }
139 
140 /*
141  * Allocate a memory handle for the channel and link it into the list
142  * Also choose which memory table to use if this is the first handle
143  * being assigned to this channel
144  */
145 int
146 ldc_mem_alloc_handle(ldc_handle_t handle, ldc_mem_handle_t *mhandle)
147 {
148 	ldc_chan_t 	*ldcp;
149 	ldc_mhdl_t	*mhdl;
150 
151 	if (handle == NULL) {
152 		DWARN(DBG_ALL_LDCS,
153 		    "ldc_mem_alloc_handle: invalid channel handle\n");
154 		return (EINVAL);
155 	}
156 	ldcp = (ldc_chan_t *)handle;
157 
158 	mutex_enter(&ldcp->lock);
159 
160 	/* check to see if channel is initalized */
161 	if ((ldcp->tstate & ~TS_IN_RESET) < TS_INIT) {
162 		DWARN(ldcp->id,
163 		    "ldc_mem_alloc_handle: (0x%llx) channel not initialized\n",
164 		    ldcp->id);
165 		mutex_exit(&ldcp->lock);
166 		return (EINVAL);
167 	}
168 
169 	/* allocate handle for channel */
170 	mhdl = kmem_cache_alloc(ldcssp->memhdl_cache, KM_SLEEP);
171 
172 	/* initialize the lock */
173 	mutex_init(&mhdl->lock, NULL, MUTEX_DRIVER, NULL);
174 
175 	mhdl->myshadow = B_FALSE;
176 	mhdl->memseg = NULL;
177 	mhdl->ldcp = ldcp;
178 	mhdl->status = LDC_UNBOUND;
179 
180 	/* insert memory handle (@ head) into list */
181 	if (ldcp->mhdl_list == NULL) {
182 		ldcp->mhdl_list = mhdl;
183 		mhdl->next = NULL;
184 	} else {
185 		/* insert @ head */
186 		mhdl->next = ldcp->mhdl_list;
187 		ldcp->mhdl_list = mhdl;
188 	}
189 
190 	/* return the handle */
191 	*mhandle = (ldc_mem_handle_t)mhdl;
192 
193 	mutex_exit(&ldcp->lock);
194 
195 	D1(ldcp->id, "ldc_mem_alloc_handle: (0x%llx) allocated handle 0x%llx\n",
196 	    ldcp->id, mhdl);
197 
198 	return (0);
199 }
200 
201 /*
202  * Free memory handle for the channel and unlink it from the list
203  */
204 int
205 ldc_mem_free_handle(ldc_mem_handle_t mhandle)
206 {
207 	ldc_mhdl_t 	*mhdl, *phdl;
208 	ldc_chan_t 	*ldcp;
209 
210 	if (mhandle == NULL) {
211 		DWARN(DBG_ALL_LDCS,
212 		    "ldc_mem_free_handle: invalid memory handle\n");
213 		return (EINVAL);
214 	}
215 	mhdl = (ldc_mhdl_t *)mhandle;
216 
217 	mutex_enter(&mhdl->lock);
218 
219 	ldcp = mhdl->ldcp;
220 
221 	if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED) {
222 		DWARN(ldcp->id,
223 		    "ldc_mem_free_handle: cannot free, 0x%llx hdl bound\n",
224 		    mhdl);
225 		mutex_exit(&mhdl->lock);
226 		return (EINVAL);
227 	}
228 	mutex_exit(&mhdl->lock);
229 
230 	mutex_enter(&ldcp->mlist_lock);
231 
232 	phdl = ldcp->mhdl_list;
233 
234 	/* first handle */
235 	if (phdl == mhdl) {
236 		ldcp->mhdl_list = mhdl->next;
237 		mutex_destroy(&mhdl->lock);
238 		kmem_cache_free(ldcssp->memhdl_cache, mhdl);
239 
240 		D1(ldcp->id,
241 		    "ldc_mem_free_handle: (0x%llx) freed handle 0x%llx\n",
242 		    ldcp->id, mhdl);
243 	} else {
244 		/* walk the list - unlink and free */
245 		while (phdl != NULL) {
246 			if (phdl->next == mhdl) {
247 				phdl->next = mhdl->next;
248 				mutex_destroy(&mhdl->lock);
249 				kmem_cache_free(ldcssp->memhdl_cache, mhdl);
250 				D1(ldcp->id,
251 				    "ldc_mem_free_handle: (0x%llx) freed "
252 				    "handle 0x%llx\n", ldcp->id, mhdl);
253 				break;
254 			}
255 			phdl = phdl->next;
256 		}
257 	}
258 
259 	if (phdl == NULL) {
260 		DWARN(ldcp->id,
261 		    "ldc_mem_free_handle: invalid handle 0x%llx\n", mhdl);
262 		mutex_exit(&ldcp->mlist_lock);
263 		return (EINVAL);
264 	}
265 
266 	mutex_exit(&ldcp->mlist_lock);
267 
268 	return (0);
269 }
270 
271 /*
272  * Bind a memory handle to a virtual address.
273  * The virtual address is converted to the corresponding real addresses.
274  * Returns pointer to the first ldc_mem_cookie and the total number
275  * of cookies for this virtual address. Other cookies can be obtained
276  * using the ldc_mem_nextcookie() call. If the pages are stored in
277  * consecutive locations in the table, a single cookie corresponding to
278  * the first location is returned. The cookie size spans all the entries.
279  *
280  * If the VA corresponds to a page that is already being exported, reuse
281  * the page and do not export it again. Bump the page's use count.
282  */
283 int
284 ldc_mem_bind_handle(ldc_mem_handle_t mhandle, caddr_t vaddr, size_t len,
285     uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount)
286 {
287 	/*
288 	 * Check if direct shared memory map is enabled, if not change
289 	 * the mapping type to SHADOW_MAP.
290 	 */
291 	if (ldc_shmem_enabled == 0)
292 		mtype = LDC_SHADOW_MAP;
293 
294 	return (i_ldc_mem_bind_handle(mhandle, vaddr, len, mtype, perm,
295 	    cookie, ccount));
296 }
297 
298 static int
299 i_ldc_mem_bind_handle(ldc_mem_handle_t mhandle, caddr_t vaddr, size_t len,
300     uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount)
301 {
302 	ldc_mhdl_t	*mhdl;
303 	ldc_chan_t 	*ldcp;
304 	ldc_mtbl_t	*mtbl;
305 	ldc_memseg_t	*memseg;
306 	ldc_mte_t	tmp_mte;
307 	uint64_t	index, prev_index = 0;
308 	int64_t		cookie_idx;
309 	uintptr_t	raddr, ra_aligned;
310 	uint64_t	psize, poffset, v_offset;
311 	uint64_t	pg_shift, pg_size, pg_size_code, pg_mask;
312 	pgcnt_t		npages;
313 	caddr_t		v_align, addr;
314 	int 		i, rv;
315 
316 	if (mhandle == NULL) {
317 		DWARN(DBG_ALL_LDCS,
318 		    "ldc_mem_bind_handle: invalid memory handle\n");
319 		return (EINVAL);
320 	}
321 	mhdl = (ldc_mhdl_t *)mhandle;
322 	ldcp = mhdl->ldcp;
323 
324 	/* clear count */
325 	*ccount = 0;
326 
327 	mutex_enter(&mhdl->lock);
328 
329 	if (mhdl->status == LDC_BOUND || mhdl->memseg != NULL) {
330 		DWARN(ldcp->id,
331 		    "ldc_mem_bind_handle: (0x%x) handle already bound\n",
332 		    mhandle);
333 		mutex_exit(&mhdl->lock);
334 		return (EINVAL);
335 	}
336 
337 	/* Force address and size to be 8-byte aligned */
338 	if ((((uintptr_t)vaddr | len) & 0x7) != 0) {
339 		DWARN(ldcp->id,
340 		    "ldc_mem_bind_handle: addr/size is not 8-byte aligned\n");
341 		mutex_exit(&mhdl->lock);
342 		return (EINVAL);
343 	}
344 
345 	mutex_enter(&ldcp->lock);
346 
347 	/*
348 	 * If this channel is binding a memory handle for the
349 	 * first time allocate it a memory map table and initialize it
350 	 */
351 	if ((mtbl = ldcp->mtbl) == NULL) {
352 
353 		/* Allocate and initialize the map table structure */
354 		mtbl = kmem_zalloc(sizeof (ldc_mtbl_t), KM_SLEEP);
355 		mtbl->num_entries = mtbl->num_avail = ldc_maptable_entries;
356 		mtbl->size = ldc_maptable_entries * sizeof (ldc_mte_slot_t);
357 		mtbl->next_entry = NULL;
358 		mtbl->contigmem = B_TRUE;
359 
360 		/* Allocate the table itself */
361 		mtbl->table = (ldc_mte_slot_t *)
362 		    contig_mem_alloc_align(mtbl->size, MMU_PAGESIZE);
363 		if (mtbl->table == NULL) {
364 
365 			/* allocate a page of memory using kmem_alloc */
366 			mtbl->table = kmem_alloc(MMU_PAGESIZE, KM_SLEEP);
367 			mtbl->size = MMU_PAGESIZE;
368 			mtbl->contigmem = B_FALSE;
369 			mtbl->num_entries = mtbl->num_avail =
370 			    mtbl->size / sizeof (ldc_mte_slot_t);
371 			DWARN(ldcp->id,
372 			    "ldc_mem_bind_handle: (0x%llx) reduced tbl size "
373 			    "to %lx entries\n", ldcp->id, mtbl->num_entries);
374 		}
375 
376 		/* zero out the memory */
377 		bzero(mtbl->table, mtbl->size);
378 
379 		/* initialize the lock */
380 		mutex_init(&mtbl->lock, NULL, MUTEX_DRIVER, NULL);
381 
382 		/* register table for this channel */
383 		rv = hv_ldc_set_map_table(ldcp->id,
384 		    va_to_pa(mtbl->table), mtbl->num_entries);
385 		if (rv != 0) {
386 			DWARN(DBG_ALL_LDCS,
387 			    "ldc_mem_bind_handle: (0x%lx) err %d mapping tbl",
388 			    ldcp->id, rv);
389 			if (mtbl->contigmem)
390 				contig_mem_free(mtbl->table, mtbl->size);
391 			else
392 				kmem_free(mtbl->table, mtbl->size);
393 			mutex_destroy(&mtbl->lock);
394 			kmem_free(mtbl, sizeof (ldc_mtbl_t));
395 			mutex_exit(&ldcp->lock);
396 			mutex_exit(&mhdl->lock);
397 			return (EIO);
398 		}
399 
400 		ldcp->mtbl = mtbl;
401 
402 		D1(ldcp->id,
403 		    "ldc_mem_bind_handle: (0x%llx) alloc'd map table 0x%llx\n",
404 		    ldcp->id, ldcp->mtbl->table);
405 	}
406 
407 	mutex_exit(&ldcp->lock);
408 
409 	/* FUTURE: get the page size, pgsz code, and shift */
410 	pg_size = MMU_PAGESIZE;
411 	pg_size_code = page_szc(pg_size);
412 	pg_shift = page_get_shift(pg_size_code);
413 	pg_mask = ~(pg_size - 1);
414 
415 	D1(ldcp->id, "ldc_mem_bind_handle: (0x%llx) binding "
416 	    "va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n",
417 	    ldcp->id, vaddr, pg_size, pg_size_code, pg_shift);
418 
419 	/* aligned VA and its offset */
420 	v_align = (caddr_t)(((uintptr_t)vaddr) & ~(pg_size - 1));
421 	v_offset = ((uintptr_t)vaddr) & (pg_size - 1);
422 
423 	npages = (len+v_offset)/pg_size;
424 	npages = ((len+v_offset)%pg_size == 0) ? npages : npages+1;
425 
426 	D1(ldcp->id, "ldc_mem_bind_handle: binding "
427 	    "(0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n",
428 	    ldcp->id, vaddr, v_align, v_offset, npages);
429 
430 	/* lock the memory table - exclusive access to channel */
431 	mutex_enter(&mtbl->lock);
432 
433 	if (npages > mtbl->num_avail) {
434 		D1(ldcp->id, "ldc_mem_bind_handle: (0x%llx) no table entries\n",
435 		    ldcp->id);
436 		mutex_exit(&mtbl->lock);
437 		mutex_exit(&mhdl->lock);
438 		return (ENOMEM);
439 	}
440 
441 	/* Allocate a memseg structure */
442 	memseg = mhdl->memseg =
443 	    kmem_cache_alloc(ldcssp->memseg_cache, KM_SLEEP);
444 
445 	/* Allocate memory to store all pages and cookies */
446 	memseg->pages = kmem_zalloc((sizeof (ldc_page_t) * npages), KM_SLEEP);
447 	memseg->cookies =
448 	    kmem_zalloc((sizeof (ldc_mem_cookie_t) * npages), KM_SLEEP);
449 
450 	D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) processing 0x%llx pages\n",
451 	    ldcp->id, npages);
452 
453 	addr = v_align;
454 
455 	/*
456 	 * Table slots are used in a round-robin manner. The algorithm permits
457 	 * inserting duplicate entries. Slots allocated earlier will typically
458 	 * get freed before we get back to reusing the slot.Inserting duplicate
459 	 * entries should be OK as we only lookup entries using the cookie addr
460 	 * i.e. tbl index, during export, unexport and copy operation.
461 	 *
462 	 * One implementation what was tried was to search for a duplicate
463 	 * page entry first and reuse it. The search overhead is very high and
464 	 * in the vnet case dropped the perf by almost half, 50 to 24 mbps.
465 	 * So it does make sense to avoid searching for duplicates.
466 	 *
467 	 * But during the process of searching for a free slot, if we find a
468 	 * duplicate entry we will go ahead and use it, and bump its use count.
469 	 */
470 
471 	/* index to start searching from */
472 	index = mtbl->next_entry;
473 	cookie_idx = -1;
474 
475 	tmp_mte.ll = 0;	/* initialise fields to 0 */
476 
477 	if (mtype & LDC_DIRECT_MAP) {
478 		tmp_mte.mte_r = (perm & LDC_MEM_R) ? 1 : 0;
479 		tmp_mte.mte_w = (perm & LDC_MEM_W) ? 1 : 0;
480 		tmp_mte.mte_x = (perm & LDC_MEM_X) ? 1 : 0;
481 	}
482 
483 	if (mtype & LDC_SHADOW_MAP) {
484 		tmp_mte.mte_cr = (perm & LDC_MEM_R) ? 1 : 0;
485 		tmp_mte.mte_cw = (perm & LDC_MEM_W) ? 1 : 0;
486 	}
487 
488 	if (mtype & LDC_IO_MAP) {
489 		tmp_mte.mte_ir = (perm & LDC_MEM_R) ? 1 : 0;
490 		tmp_mte.mte_iw = (perm & LDC_MEM_W) ? 1 : 0;
491 	}
492 
493 	D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll);
494 
495 	tmp_mte.mte_pgszc = pg_size_code;
496 
497 	/* initialize each mem table entry */
498 	for (i = 0; i < npages; i++) {
499 
500 		/* check if slot is available in the table */
501 		while (mtbl->table[index].entry.ll != 0) {
502 
503 			index = (index + 1) % mtbl->num_entries;
504 
505 			if (index == mtbl->next_entry) {
506 				/* we have looped around */
507 				DWARN(DBG_ALL_LDCS,
508 				    "ldc_mem_bind_handle: (0x%llx) cannot find "
509 				    "entry\n", ldcp->id);
510 				*ccount = 0;
511 
512 				/* NOTE: free memory, remove previous entries */
513 				/* this shouldnt happen as num_avail was ok */
514 
515 				mutex_exit(&mtbl->lock);
516 				mutex_exit(&mhdl->lock);
517 				return (ENOMEM);
518 			}
519 		}
520 
521 		/* get the real address */
522 		raddr = va_to_pa((void *)addr);
523 		ra_aligned = ((uintptr_t)raddr & pg_mask);
524 
525 		/* build the mte */
526 		tmp_mte.mte_rpfn = ra_aligned >> pg_shift;
527 
528 		D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll);
529 
530 		/* update entry in table */
531 		mtbl->table[index].entry = tmp_mte;
532 
533 		D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) stored MTE 0x%llx"
534 		    " into loc 0x%llx\n", ldcp->id, tmp_mte.ll, index);
535 
536 		/* calculate the size and offset for this export range */
537 		if (i == 0) {
538 			/* first page */
539 			psize = min((pg_size - v_offset), len);
540 			poffset = v_offset;
541 
542 		} else if (i == (npages - 1)) {
543 			/* last page */
544 			psize =	(((uintptr_t)(vaddr + len)) &
545 			    ((uint64_t)(pg_size-1)));
546 			if (psize == 0)
547 				psize = pg_size;
548 			poffset = 0;
549 
550 		} else {
551 			/* middle pages */
552 			psize = pg_size;
553 			poffset = 0;
554 		}
555 
556 		/* store entry for this page */
557 		memseg->pages[i].index = index;
558 		memseg->pages[i].raddr = raddr;
559 		memseg->pages[i].mte = &(mtbl->table[index]);
560 
561 		/* create the cookie */
562 		if (i == 0 || (index != prev_index + 1)) {
563 			cookie_idx++;
564 			memseg->cookies[cookie_idx].addr =
565 			    IDX2COOKIE(index, pg_size_code, pg_shift);
566 			memseg->cookies[cookie_idx].addr |= poffset;
567 			memseg->cookies[cookie_idx].size = psize;
568 
569 		} else {
570 			memseg->cookies[cookie_idx].size += psize;
571 		}
572 
573 		D1(ldcp->id, "ldc_mem_bind_handle: bound "
574 		    "(0x%llx) va=0x%llx, idx=0x%llx, "
575 		    "ra=0x%llx(sz=0x%x,off=0x%x)\n",
576 		    ldcp->id, addr, index, raddr, psize, poffset);
577 
578 		/* decrement number of available entries */
579 		mtbl->num_avail--;
580 
581 		/* increment va by page size */
582 		addr += pg_size;
583 
584 		/* increment index */
585 		prev_index = index;
586 		index = (index + 1) % mtbl->num_entries;
587 
588 		/* save the next slot */
589 		mtbl->next_entry = index;
590 	}
591 
592 	mutex_exit(&mtbl->lock);
593 
594 	/* memory handle = bound */
595 	mhdl->mtype = mtype;
596 	mhdl->perm = perm;
597 	mhdl->status = LDC_BOUND;
598 
599 	/* update memseg_t */
600 	memseg->vaddr = vaddr;
601 	memseg->raddr = memseg->pages[0].raddr;
602 	memseg->size = len;
603 	memseg->npages = npages;
604 	memseg->ncookies = cookie_idx + 1;
605 	memseg->next_cookie = (memseg->ncookies > 1) ? 1 : 0;
606 
607 	/* return count and first cookie */
608 	*ccount = memseg->ncookies;
609 	cookie->addr = memseg->cookies[0].addr;
610 	cookie->size = memseg->cookies[0].size;
611 
612 	D1(ldcp->id,
613 	    "ldc_mem_bind_handle: (0x%llx) bound 0x%llx, va=0x%llx, "
614 	    "pgs=0x%llx cookies=0x%llx\n",
615 	    ldcp->id, mhdl, vaddr, npages, memseg->ncookies);
616 
617 	mutex_exit(&mhdl->lock);
618 	return (0);
619 }
620 
621 /*
622  * Return the next cookie associated with the specified memory handle
623  */
624 int
625 ldc_mem_nextcookie(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie)
626 {
627 	ldc_mhdl_t	*mhdl;
628 	ldc_chan_t 	*ldcp;
629 	ldc_memseg_t	*memseg;
630 
631 	if (mhandle == NULL) {
632 		DWARN(DBG_ALL_LDCS,
633 		    "ldc_mem_nextcookie: invalid memory handle\n");
634 		return (EINVAL);
635 	}
636 	mhdl = (ldc_mhdl_t *)mhandle;
637 
638 	mutex_enter(&mhdl->lock);
639 
640 	ldcp = mhdl->ldcp;
641 	memseg = mhdl->memseg;
642 
643 	if (cookie == 0) {
644 		DWARN(ldcp->id,
645 		    "ldc_mem_nextcookie:(0x%llx) invalid cookie arg\n",
646 		    ldcp->id);
647 		mutex_exit(&mhdl->lock);
648 		return (EINVAL);
649 	}
650 
651 	if (memseg->next_cookie != 0) {
652 		cookie->addr = memseg->cookies[memseg->next_cookie].addr;
653 		cookie->size = memseg->cookies[memseg->next_cookie].size;
654 		memseg->next_cookie++;
655 		if (memseg->next_cookie == memseg->ncookies)
656 			memseg->next_cookie = 0;
657 
658 	} else {
659 		DWARN(ldcp->id,
660 		    "ldc_mem_nextcookie:(0x%llx) no more cookies\n", ldcp->id);
661 		cookie->addr = 0;
662 		cookie->size = 0;
663 		mutex_exit(&mhdl->lock);
664 		return (EINVAL);
665 	}
666 
667 	D1(ldcp->id,
668 	    "ldc_mem_nextcookie: (0x%llx) cookie addr=0x%llx,sz=0x%llx\n",
669 	    ldcp->id, cookie->addr, cookie->size);
670 
671 	mutex_exit(&mhdl->lock);
672 	return (0);
673 }
674 
675 /*
676  * Unbind the virtual memory region associated with the specified
677  * memory handle. Allassociated cookies are freed and the corresponding
678  * RA space is no longer exported.
679  */
680 int
681 ldc_mem_unbind_handle(ldc_mem_handle_t mhandle)
682 {
683 	ldc_mhdl_t	*mhdl;
684 	ldc_chan_t 	*ldcp;
685 	ldc_mtbl_t	*mtbl;
686 	ldc_memseg_t	*memseg;
687 	uint64_t	cookie_addr;
688 	uint64_t	pg_shift, pg_size_code;
689 	int		i, rv, retries;
690 
691 	if (mhandle == NULL) {
692 		DWARN(DBG_ALL_LDCS,
693 		    "ldc_mem_unbind_handle: invalid memory handle\n");
694 		return (EINVAL);
695 	}
696 	mhdl = (ldc_mhdl_t *)mhandle;
697 
698 	mutex_enter(&mhdl->lock);
699 
700 	if (mhdl->status == LDC_UNBOUND) {
701 		DWARN(DBG_ALL_LDCS,
702 		    "ldc_mem_unbind_handle: (0x%x) handle is not bound\n",
703 		    mhandle);
704 		mutex_exit(&mhdl->lock);
705 		return (EINVAL);
706 	}
707 
708 	ldcp = mhdl->ldcp;
709 	mtbl = ldcp->mtbl;
710 
711 	memseg = mhdl->memseg;
712 
713 	/* lock the memory table - exclusive access to channel */
714 	mutex_enter(&mtbl->lock);
715 
716 	/* undo the pages exported */
717 	for (i = 0; i < memseg->npages; i++) {
718 
719 		/* clear the entry from the table */
720 		memseg->pages[i].mte->entry.ll = 0;
721 
722 		/* check for mapped pages, revocation cookie != 0 */
723 		if (memseg->pages[i].mte->cookie) {
724 
725 			pg_size_code = page_szc(MMU_PAGESIZE);
726 			pg_shift = page_get_shift(pg_size_code);
727 			cookie_addr = IDX2COOKIE(memseg->pages[i].index,
728 			    pg_size_code, pg_shift);
729 
730 			D1(ldcp->id, "ldc_mem_unbind_handle: (0x%llx) revoke "
731 			    "cookie 0x%llx, rcookie 0x%llx\n", ldcp->id,
732 			    cookie_addr, memseg->pages[i].mte->cookie);
733 
734 			retries = 0;
735 			do {
736 				rv = hv_ldc_revoke(ldcp->id, cookie_addr,
737 				    memseg->pages[i].mte->cookie);
738 
739 				if (rv != H_EWOULDBLOCK)
740 					break;
741 
742 				drv_usecwait(ldc_delay);
743 
744 			} while (retries++ < ldc_max_retries);
745 
746 			if (rv) {
747 				DWARN(ldcp->id,
748 				    "ldc_mem_unbind_handle: (0x%llx) cannot "
749 				    "revoke mapping, cookie %llx\n", ldcp->id,
750 				    cookie_addr);
751 			}
752 		}
753 
754 		mtbl->num_avail++;
755 	}
756 	mutex_exit(&mtbl->lock);
757 
758 	/* free the allocated memseg and page structures */
759 	kmem_free(memseg->pages, (sizeof (ldc_page_t) * memseg->npages));
760 	kmem_free(memseg->cookies,
761 	    (sizeof (ldc_mem_cookie_t) * memseg->npages));
762 	kmem_cache_free(ldcssp->memseg_cache, memseg);
763 
764 	/* uninitialize the memory handle */
765 	mhdl->memseg = NULL;
766 	mhdl->status = LDC_UNBOUND;
767 
768 	D1(ldcp->id, "ldc_mem_unbind_handle: (0x%llx) unbound handle 0x%llx\n",
769 	    ldcp->id, mhdl);
770 
771 	mutex_exit(&mhdl->lock);
772 	return (0);
773 }
774 
775 /*
776  * Get information about the dring. The base address of the descriptor
777  * ring along with the type and permission are returned back.
778  */
779 int
780 ldc_mem_info(ldc_mem_handle_t mhandle, ldc_mem_info_t *minfo)
781 {
782 	ldc_mhdl_t	*mhdl;
783 
784 	if (mhandle == NULL) {
785 		DWARN(DBG_ALL_LDCS, "ldc_mem_info: invalid memory handle\n");
786 		return (EINVAL);
787 	}
788 	mhdl = (ldc_mhdl_t *)mhandle;
789 
790 	if (minfo == NULL) {
791 		DWARN(DBG_ALL_LDCS, "ldc_mem_info: invalid args\n");
792 		return (EINVAL);
793 	}
794 
795 	mutex_enter(&mhdl->lock);
796 
797 	minfo->status = mhdl->status;
798 	if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED) {
799 		minfo->vaddr = mhdl->memseg->vaddr;
800 		minfo->raddr = mhdl->memseg->raddr;
801 		minfo->mtype = mhdl->mtype;
802 		minfo->perm = mhdl->perm;
803 	}
804 	mutex_exit(&mhdl->lock);
805 
806 	return (0);
807 }
808 
809 /*
810  * Copy data either from or to the client specified virtual address
811  * space to or from the exported memory associated with the cookies.
812  * The direction argument determines whether the data is read from or
813  * written to exported memory.
814  */
815 int
816 ldc_mem_copy(ldc_handle_t handle, caddr_t vaddr, uint64_t off, size_t *size,
817     ldc_mem_cookie_t *cookies, uint32_t ccount, uint8_t direction)
818 {
819 	ldc_chan_t 	*ldcp;
820 	uint64_t	local_voff, local_valign;
821 	uint64_t	cookie_addr, cookie_size;
822 	uint64_t	pg_shift, pg_size, pg_size_code;
823 	uint64_t 	export_caddr, export_poff, export_psize, export_size;
824 	uint64_t	local_ra, local_poff, local_psize;
825 	uint64_t	copy_size, copied_len = 0, total_bal = 0, idx = 0;
826 	pgcnt_t		npages;
827 	size_t		len = *size;
828 	int 		i, rv = 0;
829 
830 	uint64_t	chid;
831 
832 	if (handle == NULL) {
833 		DWARN(DBG_ALL_LDCS, "ldc_mem_copy: invalid channel handle\n");
834 		return (EINVAL);
835 	}
836 	ldcp = (ldc_chan_t *)handle;
837 	chid = ldcp->id;
838 
839 	/* check to see if channel is UP */
840 	if (ldcp->tstate != TS_UP) {
841 		DWARN(chid, "ldc_mem_copy: (0x%llx) channel is not UP\n",
842 		    chid);
843 		return (ECONNRESET);
844 	}
845 
846 	/* Force address and size to be 8-byte aligned */
847 	if ((((uintptr_t)vaddr | len) & 0x7) != 0) {
848 		DWARN(chid,
849 		    "ldc_mem_copy: addr/sz is not 8-byte aligned\n");
850 		return (EINVAL);
851 	}
852 
853 	/* Find the size of the exported memory */
854 	export_size = 0;
855 	for (i = 0; i < ccount; i++)
856 		export_size += cookies[i].size;
857 
858 	/* check to see if offset is valid */
859 	if (off > export_size) {
860 		DWARN(chid,
861 		    "ldc_mem_copy: (0x%llx) start offset > export mem size\n",
862 		    chid);
863 		return (EINVAL);
864 	}
865 
866 	/*
867 	 * Check to see if the export size is smaller than the size we
868 	 * are requesting to copy - if so flag an error
869 	 */
870 	if ((export_size - off) < *size) {
871 		DWARN(chid,
872 		    "ldc_mem_copy: (0x%llx) copy size > export mem size\n",
873 		    chid);
874 		return (EINVAL);
875 	}
876 
877 	total_bal = min(export_size, *size);
878 
879 	/* FUTURE: get the page size, pgsz code, and shift */
880 	pg_size = MMU_PAGESIZE;
881 	pg_size_code = page_szc(pg_size);
882 	pg_shift = page_get_shift(pg_size_code);
883 
884 	D1(chid, "ldc_mem_copy: copying data "
885 	    "(0x%llx) va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n",
886 	    chid, vaddr, pg_size, pg_size_code, pg_shift);
887 
888 	/* aligned VA and its offset */
889 	local_valign = (((uintptr_t)vaddr) & ~(pg_size - 1));
890 	local_voff = ((uintptr_t)vaddr) & (pg_size - 1);
891 
892 	npages = (len+local_voff)/pg_size;
893 	npages = ((len+local_voff)%pg_size == 0) ? npages : npages+1;
894 
895 	D1(chid,
896 	    "ldc_mem_copy: (0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n",
897 	    chid, vaddr, local_valign, local_voff, npages);
898 
899 	local_ra = va_to_pa((void *)local_valign);
900 	local_poff = local_voff;
901 	local_psize = min(len, (pg_size - local_voff));
902 
903 	len -= local_psize;
904 
905 	/*
906 	 * find the first cookie in the list of cookies
907 	 * if the offset passed in is not zero
908 	 */
909 	for (idx = 0; idx < ccount; idx++) {
910 		cookie_size = cookies[idx].size;
911 		if (off < cookie_size)
912 			break;
913 		off -= cookie_size;
914 	}
915 
916 	cookie_addr = cookies[idx].addr + off;
917 	cookie_size = cookies[idx].size - off;
918 
919 	export_caddr = cookie_addr & ~(pg_size - 1);
920 	export_poff = cookie_addr & (pg_size - 1);
921 	export_psize = min(cookie_size, (pg_size - export_poff));
922 
923 	for (;;) {
924 
925 		copy_size = min(export_psize, local_psize);
926 
927 		D1(chid,
928 		    "ldc_mem_copy:(0x%llx) dir=0x%x, caddr=0x%llx,"
929 		    " loc_ra=0x%llx, exp_poff=0x%llx, loc_poff=0x%llx,"
930 		    " exp_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx,"
931 		    " total_bal=0x%llx\n",
932 		    chid, direction, export_caddr, local_ra, export_poff,
933 		    local_poff, export_psize, local_psize, copy_size,
934 		    total_bal);
935 
936 		rv = hv_ldc_copy(chid, direction,
937 		    (export_caddr + export_poff), (local_ra + local_poff),
938 		    copy_size, &copied_len);
939 
940 		if (rv != 0) {
941 			int 		error = EIO;
942 			uint64_t	rx_hd, rx_tl;
943 
944 			DWARN(chid,
945 			    "ldc_mem_copy: (0x%llx) err %d during copy\n",
946 			    (unsigned long long)chid, rv);
947 			DWARN(chid,
948 			    "ldc_mem_copy: (0x%llx) dir=0x%x, caddr=0x%lx, "
949 			    "loc_ra=0x%lx, exp_poff=0x%lx, loc_poff=0x%lx,"
950 			    " exp_psz=0x%lx, loc_psz=0x%lx, copy_sz=0x%lx,"
951 			    " copied_len=0x%lx, total_bal=0x%lx\n",
952 			    chid, direction, export_caddr, local_ra,
953 			    export_poff, local_poff, export_psize, local_psize,
954 			    copy_size, copied_len, total_bal);
955 
956 			*size = *size - total_bal;
957 
958 			/*
959 			 * check if reason for copy error was due to
960 			 * a channel reset. we need to grab the lock
961 			 * just in case we have to do a reset.
962 			 */
963 			mutex_enter(&ldcp->lock);
964 			mutex_enter(&ldcp->tx_lock);
965 
966 			rv = hv_ldc_rx_get_state(ldcp->id,
967 			    &rx_hd, &rx_tl, &(ldcp->link_state));
968 			if (ldcp->link_state == LDC_CHANNEL_DOWN ||
969 			    ldcp->link_state == LDC_CHANNEL_RESET) {
970 				i_ldc_reset(ldcp, B_FALSE);
971 				error = ECONNRESET;
972 			}
973 
974 			mutex_exit(&ldcp->tx_lock);
975 			mutex_exit(&ldcp->lock);
976 
977 			return (error);
978 		}
979 
980 		ASSERT(copied_len <= copy_size);
981 
982 		D2(chid, "ldc_mem_copy: copied=0x%llx\n", copied_len);
983 		export_poff += copied_len;
984 		local_poff += copied_len;
985 		export_psize -= copied_len;
986 		local_psize -= copied_len;
987 		cookie_size -= copied_len;
988 
989 		total_bal -= copied_len;
990 
991 		if (copy_size != copied_len)
992 			continue;
993 
994 		if (export_psize == 0 && total_bal != 0) {
995 
996 			if (cookie_size == 0) {
997 				idx++;
998 				cookie_addr = cookies[idx].addr;
999 				cookie_size = cookies[idx].size;
1000 
1001 				export_caddr = cookie_addr & ~(pg_size - 1);
1002 				export_poff = cookie_addr & (pg_size - 1);
1003 				export_psize =
1004 				    min(cookie_size, (pg_size-export_poff));
1005 			} else {
1006 				export_caddr += pg_size;
1007 				export_poff = 0;
1008 				export_psize = min(cookie_size, pg_size);
1009 			}
1010 		}
1011 
1012 		if (local_psize == 0 && total_bal != 0) {
1013 			local_valign += pg_size;
1014 			local_ra = va_to_pa((void *)local_valign);
1015 			local_poff = 0;
1016 			local_psize = min(pg_size, len);
1017 			len -= local_psize;
1018 		}
1019 
1020 		/* check if we are all done */
1021 		if (total_bal == 0)
1022 			break;
1023 	}
1024 
1025 
1026 	D1(chid,
1027 	    "ldc_mem_copy: (0x%llx) done copying sz=0x%llx\n",
1028 	    chid, *size);
1029 
1030 	return (0);
1031 }
1032 
1033 /*
1034  * Copy data either from or to the client specified virtual address
1035  * space to or from HV physical memory.
1036  *
1037  * The direction argument determines whether the data is read from or
1038  * written to HV memory. direction values are LDC_COPY_IN/OUT similar
1039  * to the ldc_mem_copy interface
1040  */
1041 int
1042 ldc_mem_rdwr_cookie(ldc_handle_t handle, caddr_t vaddr, size_t *size,
1043     caddr_t paddr, uint8_t direction)
1044 {
1045 	ldc_chan_t 	*ldcp;
1046 	uint64_t	local_voff, local_valign;
1047 	uint64_t	pg_shift, pg_size, pg_size_code;
1048 	uint64_t 	target_pa, target_poff, target_psize, target_size;
1049 	uint64_t	local_ra, local_poff, local_psize;
1050 	uint64_t	copy_size, copied_len = 0;
1051 	pgcnt_t		npages;
1052 	size_t		len = *size;
1053 	int 		rv = 0;
1054 
1055 	if (handle == NULL) {
1056 		DWARN(DBG_ALL_LDCS,
1057 		    "ldc_mem_rdwr_cookie: invalid channel handle\n");
1058 		return (EINVAL);
1059 	}
1060 	ldcp = (ldc_chan_t *)handle;
1061 
1062 	mutex_enter(&ldcp->lock);
1063 
1064 	/* check to see if channel is UP */
1065 	if (ldcp->tstate != TS_UP) {
1066 		DWARN(ldcp->id,
1067 		    "ldc_mem_rdwr_cookie: (0x%llx) channel is not UP\n",
1068 		    ldcp->id);
1069 		mutex_exit(&ldcp->lock);
1070 		return (ECONNRESET);
1071 	}
1072 
1073 	/* Force address and size to be 8-byte aligned */
1074 	if ((((uintptr_t)vaddr | len) & 0x7) != 0) {
1075 		DWARN(ldcp->id,
1076 		    "ldc_mem_rdwr_cookie: addr/size is not 8-byte aligned\n");
1077 		mutex_exit(&ldcp->lock);
1078 		return (EINVAL);
1079 	}
1080 
1081 	target_size = *size;
1082 
1083 	/* FUTURE: get the page size, pgsz code, and shift */
1084 	pg_size = MMU_PAGESIZE;
1085 	pg_size_code = page_szc(pg_size);
1086 	pg_shift = page_get_shift(pg_size_code);
1087 
1088 	D1(ldcp->id, "ldc_mem_rdwr_cookie: copying data "
1089 	    "(0x%llx) va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n",
1090 	    ldcp->id, vaddr, pg_size, pg_size_code, pg_shift);
1091 
1092 	/* aligned VA and its offset */
1093 	local_valign = ((uintptr_t)vaddr) & ~(pg_size - 1);
1094 	local_voff = ((uintptr_t)vaddr) & (pg_size - 1);
1095 
1096 	npages = (len + local_voff) / pg_size;
1097 	npages = ((len + local_voff) % pg_size == 0) ? npages : npages+1;
1098 
1099 	D1(ldcp->id, "ldc_mem_rdwr_cookie: (0x%llx) v=0x%llx, "
1100 	    "val=0x%llx,off=0x%x,pgs=0x%x\n",
1101 	    ldcp->id, vaddr, local_valign, local_voff, npages);
1102 
1103 	local_ra = va_to_pa((void *)local_valign);
1104 	local_poff = local_voff;
1105 	local_psize = min(len, (pg_size - local_voff));
1106 
1107 	len -= local_psize;
1108 
1109 	target_pa = ((uintptr_t)paddr) & ~(pg_size - 1);
1110 	target_poff = ((uintptr_t)paddr) & (pg_size - 1);
1111 	target_psize = pg_size - target_poff;
1112 
1113 	for (;;) {
1114 
1115 		copy_size = min(target_psize, local_psize);
1116 
1117 		D1(ldcp->id,
1118 		    "ldc_mem_rdwr_cookie: (0x%llx) dir=0x%x, tar_pa=0x%llx,"
1119 		    " loc_ra=0x%llx, tar_poff=0x%llx, loc_poff=0x%llx,"
1120 		    " tar_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx,"
1121 		    " total_bal=0x%llx\n",
1122 		    ldcp->id, direction, target_pa, local_ra, target_poff,
1123 		    local_poff, target_psize, local_psize, copy_size,
1124 		    target_size);
1125 
1126 		rv = hv_ldc_copy(ldcp->id, direction,
1127 		    (target_pa + target_poff), (local_ra + local_poff),
1128 		    copy_size, &copied_len);
1129 
1130 		if (rv != 0) {
1131 			DWARN(DBG_ALL_LDCS,
1132 			    "ldc_mem_rdwr_cookie: (0x%lx) err %d during copy\n",
1133 			    ldcp->id, rv);
1134 			DWARN(DBG_ALL_LDCS,
1135 			    "ldc_mem_rdwr_cookie: (0x%llx) dir=%lld, "
1136 			    "tar_pa=0x%llx, loc_ra=0x%llx, tar_poff=0x%llx, "
1137 			    "loc_poff=0x%llx, tar_psz=0x%llx, loc_psz=0x%llx, "
1138 			    "copy_sz=0x%llx, total_bal=0x%llx\n",
1139 			    ldcp->id, direction, target_pa, local_ra,
1140 			    target_poff, local_poff, target_psize, local_psize,
1141 			    copy_size, target_size);
1142 
1143 			*size = *size - target_size;
1144 			mutex_exit(&ldcp->lock);
1145 			return (i_ldc_h2v_error(rv));
1146 		}
1147 
1148 		D2(ldcp->id, "ldc_mem_rdwr_cookie: copied=0x%llx\n",
1149 		    copied_len);
1150 		target_poff += copied_len;
1151 		local_poff += copied_len;
1152 		target_psize -= copied_len;
1153 		local_psize -= copied_len;
1154 
1155 		target_size -= copied_len;
1156 
1157 		if (copy_size != copied_len)
1158 			continue;
1159 
1160 		if (target_psize == 0 && target_size != 0) {
1161 			target_pa += pg_size;
1162 			target_poff = 0;
1163 			target_psize = min(pg_size, target_size);
1164 		}
1165 
1166 		if (local_psize == 0 && target_size != 0) {
1167 			local_valign += pg_size;
1168 			local_ra = va_to_pa((void *)local_valign);
1169 			local_poff = 0;
1170 			local_psize = min(pg_size, len);
1171 			len -= local_psize;
1172 		}
1173 
1174 		/* check if we are all done */
1175 		if (target_size == 0)
1176 			break;
1177 	}
1178 
1179 	mutex_exit(&ldcp->lock);
1180 
1181 	D1(ldcp->id, "ldc_mem_rdwr_cookie: (0x%llx) done copying sz=0x%llx\n",
1182 	    ldcp->id, *size);
1183 
1184 	return (0);
1185 }
1186 
1187 /*
1188  * Map an exported memory segment into the local address space. If the
1189  * memory range was exported for direct map access, a HV call is made
1190  * to allocate a RA range. If the map is done via a shadow copy, local
1191  * shadow memory is allocated and the base VA is returned in 'vaddr'. If
1192  * the mapping is a direct map then the RA is returned in 'raddr'.
1193  */
1194 int
1195 ldc_mem_map(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie, uint32_t ccount,
1196     uint8_t mtype, uint8_t perm, caddr_t *vaddr, caddr_t *raddr)
1197 {
1198 	/*
1199 	 * Check if direct map over shared memory is enabled, if not change
1200 	 * the mapping type to SHADOW_MAP.
1201 	 */
1202 	if (ldc_shmem_enabled == 0)
1203 		mtype = LDC_SHADOW_MAP;
1204 
1205 	return (i_ldc_mem_map(mhandle, cookie, ccount, mtype, perm,
1206 	    vaddr, raddr));
1207 }
1208 
1209 static int
1210 i_ldc_mem_map(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie,
1211     uint32_t ccount, uint8_t mtype, uint8_t perm, caddr_t *vaddr,
1212     caddr_t *raddr)
1213 {
1214 
1215 	int		i, j, idx, rv, retries;
1216 	ldc_chan_t 	*ldcp;
1217 	ldc_mhdl_t	*mhdl;
1218 	ldc_memseg_t	*memseg;
1219 	caddr_t		tmpaddr;
1220 	uint64_t	map_perm = perm;
1221 	uint64_t	pg_size, pg_shift, pg_size_code, pg_mask;
1222 	uint64_t	exp_size = 0, base_off, map_size, npages;
1223 	uint64_t	cookie_addr, cookie_off, cookie_size;
1224 	tte_t		ldc_tte;
1225 
1226 	if (mhandle == NULL) {
1227 		DWARN(DBG_ALL_LDCS, "ldc_mem_map: invalid memory handle\n");
1228 		return (EINVAL);
1229 	}
1230 	mhdl = (ldc_mhdl_t *)mhandle;
1231 
1232 	mutex_enter(&mhdl->lock);
1233 
1234 	if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED ||
1235 	    mhdl->memseg != NULL) {
1236 		DWARN(DBG_ALL_LDCS,
1237 		    "ldc_mem_map: (0x%llx) handle bound/mapped\n", mhandle);
1238 		mutex_exit(&mhdl->lock);
1239 		return (EINVAL);
1240 	}
1241 
1242 	ldcp = mhdl->ldcp;
1243 
1244 	mutex_enter(&ldcp->lock);
1245 
1246 	if (ldcp->tstate != TS_UP) {
1247 		DWARN(ldcp->id,
1248 		    "ldc_mem_dring_map: (0x%llx) channel is not UP\n",
1249 		    ldcp->id);
1250 		mutex_exit(&ldcp->lock);
1251 		mutex_exit(&mhdl->lock);
1252 		return (ECONNRESET);
1253 	}
1254 
1255 	if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP|LDC_IO_MAP)) == 0) {
1256 		DWARN(ldcp->id, "ldc_mem_map: invalid map type\n");
1257 		mutex_exit(&ldcp->lock);
1258 		mutex_exit(&mhdl->lock);
1259 		return (EINVAL);
1260 	}
1261 
1262 	D1(ldcp->id, "ldc_mem_map: (0x%llx) cookie = 0x%llx,0x%llx\n",
1263 	    ldcp->id, cookie->addr, cookie->size);
1264 
1265 	/* FUTURE: get the page size, pgsz code, and shift */
1266 	pg_size = MMU_PAGESIZE;
1267 	pg_size_code = page_szc(pg_size);
1268 	pg_shift = page_get_shift(pg_size_code);
1269 	pg_mask = ~(pg_size - 1);
1270 
1271 	/* calculate the number of pages in the exported cookie */
1272 	base_off = cookie[0].addr & (pg_size - 1);
1273 	for (idx = 0; idx < ccount; idx++)
1274 		exp_size += cookie[idx].size;
1275 	map_size = P2ROUNDUP((exp_size + base_off), pg_size);
1276 	npages = (map_size >> pg_shift);
1277 
1278 	/* Allocate memseg structure */
1279 	memseg = mhdl->memseg =
1280 	    kmem_cache_alloc(ldcssp->memseg_cache, KM_SLEEP);
1281 
1282 	/* Allocate memory to store all pages and cookies */
1283 	memseg->pages =	kmem_zalloc((sizeof (ldc_page_t) * npages), KM_SLEEP);
1284 	memseg->cookies =
1285 	    kmem_zalloc((sizeof (ldc_mem_cookie_t) * ccount), KM_SLEEP);
1286 
1287 	D2(ldcp->id, "ldc_mem_map: (0x%llx) exp_size=0x%llx, map_size=0x%llx,"
1288 	    "pages=0x%llx\n", ldcp->id, exp_size, map_size, npages);
1289 
1290 	/*
1291 	 * Check to see if the client is requesting direct or shadow map
1292 	 * If direct map is requested, try to map remote memory first,
1293 	 * and if that fails, revert to shadow map
1294 	 */
1295 	if (mtype == LDC_DIRECT_MAP) {
1296 
1297 		/* Allocate kernel virtual space for mapping */
1298 		memseg->vaddr = vmem_xalloc(heap_arena, map_size,
1299 		    pg_size, 0, 0, NULL, NULL, VM_NOSLEEP);
1300 		if (memseg->vaddr == NULL) {
1301 			DWARN(DBG_ALL_LDCS,
1302 			    "ldc_mem_map: (0x%lx) memory map failed\n",
1303 			    ldcp->id);
1304 			kmem_free(memseg->cookies,
1305 			    (sizeof (ldc_mem_cookie_t) * ccount));
1306 			kmem_free(memseg->pages,
1307 			    (sizeof (ldc_page_t) * npages));
1308 			kmem_cache_free(ldcssp->memseg_cache, memseg);
1309 
1310 			mutex_exit(&ldcp->lock);
1311 			mutex_exit(&mhdl->lock);
1312 			return (ENOMEM);
1313 		}
1314 
1315 		/* Unload previous mapping */
1316 		hat_unload(kas.a_hat, memseg->vaddr, map_size,
1317 		    HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK);
1318 
1319 		/* for each cookie passed in - map into address space */
1320 		idx = 0;
1321 		cookie_size = 0;
1322 		tmpaddr = memseg->vaddr;
1323 
1324 		for (i = 0; i < npages; i++) {
1325 
1326 			if (cookie_size == 0) {
1327 				ASSERT(idx < ccount);
1328 				cookie_addr = cookie[idx].addr & pg_mask;
1329 				cookie_off = cookie[idx].addr & (pg_size - 1);
1330 				cookie_size =
1331 				    P2ROUNDUP((cookie_off + cookie[idx].size),
1332 				    pg_size);
1333 				idx++;
1334 			}
1335 
1336 			D1(ldcp->id, "ldc_mem_map: (0x%llx) mapping "
1337 			    "cookie 0x%llx, bal=0x%llx\n", ldcp->id,
1338 			    cookie_addr, cookie_size);
1339 
1340 			/* map the cookie into address space */
1341 			for (retries = 0; retries < ldc_max_retries;
1342 			    retries++) {
1343 
1344 				rv = hv_ldc_mapin(ldcp->id, cookie_addr,
1345 				    &memseg->pages[i].raddr, &map_perm);
1346 				if (rv != H_EWOULDBLOCK && rv != H_ETOOMANY)
1347 					break;
1348 
1349 				drv_usecwait(ldc_delay);
1350 			}
1351 
1352 			if (rv || memseg->pages[i].raddr == 0) {
1353 				DWARN(ldcp->id,
1354 				    "ldc_mem_map: (0x%llx) hv mapin err %d\n",
1355 				    ldcp->id, rv);
1356 
1357 				/* remove previous mapins */
1358 				hat_unload(kas.a_hat, memseg->vaddr, map_size,
1359 				    HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK);
1360 				for (j = 0; j < i; j++) {
1361 					rv = hv_ldc_unmap(
1362 					    memseg->pages[j].raddr);
1363 					if (rv) {
1364 						DWARN(ldcp->id,
1365 						    "ldc_mem_map: (0x%llx) "
1366 						    "cannot unmap ra=0x%llx\n",
1367 						    ldcp->id,
1368 						    memseg->pages[j].raddr);
1369 					}
1370 				}
1371 
1372 				/* free kernel virtual space */
1373 				vmem_free(heap_arena, (void *)memseg->vaddr,
1374 				    map_size);
1375 
1376 				/* direct map failed - revert to shadow map */
1377 				mtype = LDC_SHADOW_MAP;
1378 				break;
1379 
1380 			} else {
1381 
1382 				D1(ldcp->id,
1383 				    "ldc_mem_map: (0x%llx) vtop map 0x%llx -> "
1384 				    "0x%llx, cookie=0x%llx, perm=0x%llx\n",
1385 				    ldcp->id, tmpaddr, memseg->pages[i].raddr,
1386 				    cookie_addr, perm);
1387 
1388 				/*
1389 				 * NOTE: Calling hat_devload directly, causes it
1390 				 * to look for page_t using the pfn. Since this
1391 				 * addr is greater than the memlist, it treates
1392 				 * it as non-memory
1393 				 */
1394 				sfmmu_memtte(&ldc_tte,
1395 				    (pfn_t)(memseg->pages[i].raddr >> pg_shift),
1396 				    PROT_READ | PROT_WRITE | HAT_NOSYNC, TTE8K);
1397 
1398 				D1(ldcp->id,
1399 				    "ldc_mem_map: (0x%llx) ra 0x%llx -> "
1400 				    "tte 0x%llx\n", ldcp->id,
1401 				    memseg->pages[i].raddr, ldc_tte);
1402 
1403 				sfmmu_tteload(kas.a_hat, &ldc_tte, tmpaddr,
1404 				    NULL, HAT_LOAD_LOCK);
1405 
1406 				cookie_size -= pg_size;
1407 				cookie_addr += pg_size;
1408 				tmpaddr += pg_size;
1409 			}
1410 		}
1411 	}
1412 
1413 	if (mtype == LDC_SHADOW_MAP) {
1414 		if (*vaddr == NULL) {
1415 			memseg->vaddr = kmem_zalloc(exp_size, KM_SLEEP);
1416 			mhdl->myshadow = B_TRUE;
1417 
1418 			D1(ldcp->id, "ldc_mem_map: (0x%llx) allocated "
1419 			    "shadow page va=0x%llx\n", ldcp->id, memseg->vaddr);
1420 		} else {
1421 			/*
1422 			 * Use client supplied memory for memseg->vaddr
1423 			 * WARNING: assuming that client mem is >= exp_size
1424 			 */
1425 			memseg->vaddr = *vaddr;
1426 		}
1427 
1428 		/* Save all page and cookie information */
1429 		for (i = 0, tmpaddr = memseg->vaddr; i < npages; i++) {
1430 			memseg->pages[i].raddr = va_to_pa(tmpaddr);
1431 			tmpaddr += pg_size;
1432 		}
1433 
1434 	}
1435 
1436 	/* save all cookies */
1437 	bcopy(cookie, memseg->cookies, ccount * sizeof (ldc_mem_cookie_t));
1438 
1439 	/* update memseg_t */
1440 	memseg->raddr = memseg->pages[0].raddr;
1441 	memseg->size = (mtype == LDC_SHADOW_MAP) ? exp_size : map_size;
1442 	memseg->npages = npages;
1443 	memseg->ncookies = ccount;
1444 	memseg->next_cookie = 0;
1445 
1446 	/* memory handle = mapped */
1447 	mhdl->mtype = mtype;
1448 	mhdl->perm = perm;
1449 	mhdl->status = LDC_MAPPED;
1450 
1451 	D1(ldcp->id, "ldc_mem_map: (0x%llx) mapped 0x%llx, ra=0x%llx, "
1452 	    "va=0x%llx, pgs=0x%llx cookies=0x%llx\n",
1453 	    ldcp->id, mhdl, memseg->raddr, memseg->vaddr,
1454 	    memseg->npages, memseg->ncookies);
1455 
1456 	if (mtype == LDC_SHADOW_MAP)
1457 		base_off = 0;
1458 	if (raddr)
1459 		*raddr = (caddr_t)(memseg->raddr | base_off);
1460 	if (vaddr)
1461 		*vaddr = (caddr_t)((uintptr_t)memseg->vaddr | base_off);
1462 
1463 	mutex_exit(&ldcp->lock);
1464 	mutex_exit(&mhdl->lock);
1465 	return (0);
1466 }
1467 
1468 /*
1469  * Unmap a memory segment. Free shadow memory (if any).
1470  */
1471 int
1472 ldc_mem_unmap(ldc_mem_handle_t mhandle)
1473 {
1474 	int		i, rv;
1475 	ldc_mhdl_t	*mhdl = (ldc_mhdl_t *)mhandle;
1476 	ldc_chan_t 	*ldcp;
1477 	ldc_memseg_t	*memseg;
1478 
1479 	if (mhdl == 0 || mhdl->status != LDC_MAPPED) {
1480 		DWARN(DBG_ALL_LDCS,
1481 		    "ldc_mem_unmap: (0x%llx) handle is not mapped\n",
1482 		    mhandle);
1483 		return (EINVAL);
1484 	}
1485 
1486 	mutex_enter(&mhdl->lock);
1487 
1488 	ldcp = mhdl->ldcp;
1489 	memseg = mhdl->memseg;
1490 
1491 	D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapping handle 0x%llx\n",
1492 	    ldcp->id, mhdl);
1493 
1494 	/* if we allocated shadow memory - free it */
1495 	if (mhdl->mtype == LDC_SHADOW_MAP && mhdl->myshadow) {
1496 		kmem_free(memseg->vaddr, memseg->size);
1497 	} else if (mhdl->mtype == LDC_DIRECT_MAP) {
1498 
1499 		/* unmap in the case of DIRECT_MAP */
1500 		hat_unload(kas.a_hat, memseg->vaddr, memseg->size,
1501 		    HAT_UNLOAD_UNLOCK);
1502 
1503 		for (i = 0; i < memseg->npages; i++) {
1504 			rv = hv_ldc_unmap(memseg->pages[i].raddr);
1505 			if (rv) {
1506 				DWARN(DBG_ALL_LDCS,
1507 				    "ldc_mem_map: (0x%lx) hv unmap err %d\n",
1508 				    ldcp->id, rv);
1509 			}
1510 		}
1511 
1512 		vmem_free(heap_arena, (void *)memseg->vaddr, memseg->size);
1513 	}
1514 
1515 	/* free the allocated memseg and page structures */
1516 	kmem_free(memseg->pages, (sizeof (ldc_page_t) * memseg->npages));
1517 	kmem_free(memseg->cookies,
1518 	    (sizeof (ldc_mem_cookie_t) * memseg->ncookies));
1519 	kmem_cache_free(ldcssp->memseg_cache, memseg);
1520 
1521 	/* uninitialize the memory handle */
1522 	mhdl->memseg = NULL;
1523 	mhdl->status = LDC_UNBOUND;
1524 
1525 	D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapped handle 0x%llx\n",
1526 	    ldcp->id, mhdl);
1527 
1528 	mutex_exit(&mhdl->lock);
1529 	return (0);
1530 }
1531 
1532 /*
1533  * Internal entry point for LDC mapped memory entry consistency
1534  * semantics. Acquire copies the contents of the remote memory
1535  * into the local shadow copy. The release operation copies the local
1536  * contents into the remote memory. The offset and size specify the
1537  * bounds for the memory range being synchronized.
1538  */
1539 static int
1540 i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle, uint8_t direction,
1541     uint64_t offset, size_t size)
1542 {
1543 	int 		err;
1544 	ldc_mhdl_t	*mhdl;
1545 	ldc_chan_t	*ldcp;
1546 	ldc_memseg_t	*memseg;
1547 	caddr_t		local_vaddr;
1548 	size_t		copy_size;
1549 
1550 	if (mhandle == NULL) {
1551 		DWARN(DBG_ALL_LDCS,
1552 		    "i_ldc_mem_acquire_release: invalid memory handle\n");
1553 		return (EINVAL);
1554 	}
1555 	mhdl = (ldc_mhdl_t *)mhandle;
1556 
1557 	mutex_enter(&mhdl->lock);
1558 
1559 	if (mhdl->status != LDC_MAPPED || mhdl->ldcp == NULL) {
1560 		DWARN(DBG_ALL_LDCS,
1561 		    "i_ldc_mem_acquire_release: not mapped memory\n");
1562 		mutex_exit(&mhdl->lock);
1563 		return (EINVAL);
1564 	}
1565 
1566 	/* do nothing for direct map */
1567 	if (mhdl->mtype == LDC_DIRECT_MAP) {
1568 		mutex_exit(&mhdl->lock);
1569 		return (0);
1570 	}
1571 
1572 	/* do nothing if COPY_IN+MEM_W and COPY_OUT+MEM_R */
1573 	if ((direction == LDC_COPY_IN && (mhdl->perm & LDC_MEM_R) == 0) ||
1574 	    (direction == LDC_COPY_OUT && (mhdl->perm & LDC_MEM_W) == 0)) {
1575 		mutex_exit(&mhdl->lock);
1576 		return (0);
1577 	}
1578 
1579 	if (offset >= mhdl->memseg->size ||
1580 	    (offset + size) > mhdl->memseg->size) {
1581 		DWARN(DBG_ALL_LDCS,
1582 		    "i_ldc_mem_acquire_release: memory out of range\n");
1583 		mutex_exit(&mhdl->lock);
1584 		return (EINVAL);
1585 	}
1586 
1587 	/* get the channel handle and memory segment */
1588 	ldcp = mhdl->ldcp;
1589 	memseg = mhdl->memseg;
1590 
1591 	if (mhdl->mtype == LDC_SHADOW_MAP) {
1592 
1593 		local_vaddr = memseg->vaddr + offset;
1594 		copy_size = size;
1595 
1596 		/* copy to/from remote from/to local memory */
1597 		err = ldc_mem_copy((ldc_handle_t)ldcp, local_vaddr, offset,
1598 		    &copy_size, memseg->cookies, memseg->ncookies,
1599 		    direction);
1600 		if (err || copy_size != size) {
1601 			DWARN(ldcp->id,
1602 			    "i_ldc_mem_acquire_release: copy failed\n");
1603 			mutex_exit(&mhdl->lock);
1604 			return (err);
1605 		}
1606 	}
1607 
1608 	mutex_exit(&mhdl->lock);
1609 
1610 	return (0);
1611 }
1612 
1613 /*
1614  * Ensure that the contents in the remote memory seg are consistent
1615  * with the contents if of local segment
1616  */
1617 int
1618 ldc_mem_acquire(ldc_mem_handle_t mhandle, uint64_t offset, uint64_t size)
1619 {
1620 	return (i_ldc_mem_acquire_release(mhandle, LDC_COPY_IN, offset, size));
1621 }
1622 
1623 
1624 /*
1625  * Ensure that the contents in the local memory seg are consistent
1626  * with the contents if of remote segment
1627  */
1628 int
1629 ldc_mem_release(ldc_mem_handle_t mhandle, uint64_t offset, uint64_t size)
1630 {
1631 	return (i_ldc_mem_acquire_release(mhandle, LDC_COPY_OUT, offset, size));
1632 }
1633 
1634 /*
1635  * Allocate a descriptor ring. The size of each each descriptor
1636  * must be 8-byte aligned and the entire ring should be a multiple
1637  * of MMU_PAGESIZE.
1638  */
1639 int
1640 ldc_mem_dring_create(uint32_t len, uint32_t dsize, ldc_dring_handle_t *dhandle)
1641 {
1642 	ldc_dring_t *dringp;
1643 	size_t size = (dsize * len);
1644 
1645 	D1(DBG_ALL_LDCS, "ldc_mem_dring_create: len=0x%x, size=0x%x\n",
1646 	    len, dsize);
1647 
1648 	if (dhandle == NULL) {
1649 		DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid dhandle\n");
1650 		return (EINVAL);
1651 	}
1652 
1653 	if (len == 0) {
1654 		DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid length\n");
1655 		return (EINVAL);
1656 	}
1657 
1658 	/* descriptor size should be 8-byte aligned */
1659 	if (dsize == 0 || (dsize & 0x7)) {
1660 		DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid size\n");
1661 		return (EINVAL);
1662 	}
1663 
1664 	*dhandle = 0;
1665 
1666 	/* Allocate a desc ring structure */
1667 	dringp = kmem_zalloc(sizeof (ldc_dring_t), KM_SLEEP);
1668 
1669 	/* Initialize dring */
1670 	dringp->length = len;
1671 	dringp->dsize = dsize;
1672 
1673 	/* round off to multiple of pagesize */
1674 	dringp->size = (size & MMU_PAGEMASK);
1675 	if (size & MMU_PAGEOFFSET)
1676 		dringp->size += MMU_PAGESIZE;
1677 
1678 	dringp->status = LDC_UNBOUND;
1679 
1680 	/* allocate descriptor ring memory */
1681 	dringp->base = kmem_zalloc(dringp->size, KM_SLEEP);
1682 
1683 	/* initialize the desc ring lock */
1684 	mutex_init(&dringp->lock, NULL, MUTEX_DRIVER, NULL);
1685 
1686 	/* Add descriptor ring to the head of global list */
1687 	mutex_enter(&ldcssp->lock);
1688 	dringp->next = ldcssp->dring_list;
1689 	ldcssp->dring_list = dringp;
1690 	mutex_exit(&ldcssp->lock);
1691 
1692 	*dhandle = (ldc_dring_handle_t)dringp;
1693 
1694 	D1(DBG_ALL_LDCS, "ldc_mem_dring_create: dring allocated\n");
1695 
1696 	return (0);
1697 }
1698 
1699 
1700 /*
1701  * Destroy a descriptor ring.
1702  */
1703 int
1704 ldc_mem_dring_destroy(ldc_dring_handle_t dhandle)
1705 {
1706 	ldc_dring_t *dringp;
1707 	ldc_dring_t *tmp_dringp;
1708 
1709 	D1(DBG_ALL_LDCS, "ldc_mem_dring_destroy: entered\n");
1710 
1711 	if (dhandle == NULL) {
1712 		DWARN(DBG_ALL_LDCS,
1713 		    "ldc_mem_dring_destroy: invalid desc ring handle\n");
1714 		return (EINVAL);
1715 	}
1716 	dringp = (ldc_dring_t *)dhandle;
1717 
1718 	if (dringp->status == LDC_BOUND) {
1719 		DWARN(DBG_ALL_LDCS,
1720 		    "ldc_mem_dring_destroy: desc ring is bound\n");
1721 		return (EACCES);
1722 	}
1723 
1724 	mutex_enter(&dringp->lock);
1725 	mutex_enter(&ldcssp->lock);
1726 
1727 	/* remove from linked list - if not bound */
1728 	tmp_dringp = ldcssp->dring_list;
1729 	if (tmp_dringp == dringp) {
1730 		ldcssp->dring_list = dringp->next;
1731 		dringp->next = NULL;
1732 
1733 	} else {
1734 		while (tmp_dringp != NULL) {
1735 			if (tmp_dringp->next == dringp) {
1736 				tmp_dringp->next = dringp->next;
1737 				dringp->next = NULL;
1738 				break;
1739 			}
1740 			tmp_dringp = tmp_dringp->next;
1741 		}
1742 		if (tmp_dringp == NULL) {
1743 			DWARN(DBG_ALL_LDCS,
1744 			    "ldc_mem_dring_destroy: invalid descriptor\n");
1745 			mutex_exit(&ldcssp->lock);
1746 			mutex_exit(&dringp->lock);
1747 			return (EINVAL);
1748 		}
1749 	}
1750 
1751 	mutex_exit(&ldcssp->lock);
1752 
1753 	/* free the descriptor ring */
1754 	kmem_free(dringp->base, dringp->size);
1755 
1756 	mutex_exit(&dringp->lock);
1757 
1758 	/* destroy dring lock */
1759 	mutex_destroy(&dringp->lock);
1760 
1761 	/* free desc ring object */
1762 	kmem_free(dringp, sizeof (ldc_dring_t));
1763 
1764 	return (0);
1765 }
1766 
1767 /*
1768  * Bind a previously allocated dring to a channel. The channel should
1769  * be OPEN in order to bind the ring to the channel. Returns back a
1770  * descriptor ring cookie. The descriptor ring is exported for remote
1771  * access by the client at the other end of the channel. An entry for
1772  * dring pages is stored in map table (via call to ldc_mem_bind_handle).
1773  */
1774 int
1775 ldc_mem_dring_bind(ldc_handle_t handle, ldc_dring_handle_t dhandle,
1776     uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount)
1777 {
1778 	int		err;
1779 	ldc_chan_t 	*ldcp;
1780 	ldc_dring_t	*dringp;
1781 	ldc_mem_handle_t mhandle;
1782 
1783 	/* check to see if channel is initalized */
1784 	if (handle == NULL) {
1785 		DWARN(DBG_ALL_LDCS,
1786 		    "ldc_mem_dring_bind: invalid channel handle\n");
1787 		return (EINVAL);
1788 	}
1789 	ldcp = (ldc_chan_t *)handle;
1790 
1791 	if (dhandle == NULL) {
1792 		DWARN(DBG_ALL_LDCS,
1793 		    "ldc_mem_dring_bind: invalid desc ring handle\n");
1794 		return (EINVAL);
1795 	}
1796 	dringp = (ldc_dring_t *)dhandle;
1797 
1798 	if (cookie == NULL) {
1799 		DWARN(ldcp->id,
1800 		    "ldc_mem_dring_bind: invalid cookie arg\n");
1801 		return (EINVAL);
1802 	}
1803 
1804 	/* ensure the mtype is valid */
1805 	if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP)) == 0) {
1806 		DWARN(ldcp->id, "ldc_mem_dring_bind: invalid map type\n");
1807 		return (EINVAL);
1808 	}
1809 
1810 	/* no need to bind as direct map if it's not HV supported or enabled */
1811 	if (!ldc_dring_shmem_hv_ok || !ldc_dring_shmem_enabled) {
1812 		mtype = LDC_SHADOW_MAP;
1813 	}
1814 
1815 	mutex_enter(&dringp->lock);
1816 
1817 	if (dringp->status == LDC_BOUND) {
1818 		DWARN(DBG_ALL_LDCS,
1819 		    "ldc_mem_dring_bind: (0x%llx) descriptor ring is bound\n",
1820 		    ldcp->id);
1821 		mutex_exit(&dringp->lock);
1822 		return (EINVAL);
1823 	}
1824 
1825 	if ((perm & LDC_MEM_RW) == 0) {
1826 		DWARN(DBG_ALL_LDCS,
1827 		    "ldc_mem_dring_bind: invalid permissions\n");
1828 		mutex_exit(&dringp->lock);
1829 		return (EINVAL);
1830 	}
1831 
1832 	if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP|LDC_IO_MAP)) == 0) {
1833 		DWARN(DBG_ALL_LDCS, "ldc_mem_dring_bind: invalid type\n");
1834 		mutex_exit(&dringp->lock);
1835 		return (EINVAL);
1836 	}
1837 
1838 	dringp->ldcp = ldcp;
1839 
1840 	/* create an memory handle */
1841 	err = ldc_mem_alloc_handle(handle, &mhandle);
1842 	if (err || mhandle == NULL) {
1843 		DWARN(DBG_ALL_LDCS,
1844 		    "ldc_mem_dring_bind: (0x%llx) error allocating mhandle\n",
1845 		    ldcp->id);
1846 		mutex_exit(&dringp->lock);
1847 		return (err);
1848 	}
1849 	dringp->mhdl = mhandle;
1850 
1851 	/* bind the descriptor ring to channel */
1852 	err = i_ldc_mem_bind_handle(mhandle, dringp->base, dringp->size,
1853 	    mtype, perm, cookie, ccount);
1854 	if (err) {
1855 		DWARN(ldcp->id,
1856 		    "ldc_mem_dring_bind: (0x%llx) error binding mhandle\n",
1857 		    ldcp->id);
1858 		mutex_exit(&dringp->lock);
1859 		return (err);
1860 	}
1861 
1862 	/*
1863 	 * For now return error if we get more than one cookie
1864 	 * FUTURE: Return multiple cookies ..
1865 	 */
1866 	if (*ccount > 1) {
1867 		(void) ldc_mem_unbind_handle(mhandle);
1868 		(void) ldc_mem_free_handle(mhandle);
1869 
1870 		dringp->ldcp = NULL;
1871 		dringp->mhdl = NULL;
1872 		*ccount = 0;
1873 
1874 		mutex_exit(&dringp->lock);
1875 		return (EAGAIN);
1876 	}
1877 
1878 	/* Add descriptor ring to channel's exported dring list */
1879 	mutex_enter(&ldcp->exp_dlist_lock);
1880 	dringp->ch_next = ldcp->exp_dring_list;
1881 	ldcp->exp_dring_list = dringp;
1882 	mutex_exit(&ldcp->exp_dlist_lock);
1883 
1884 	dringp->status = LDC_BOUND;
1885 
1886 	mutex_exit(&dringp->lock);
1887 
1888 	return (0);
1889 }
1890 
1891 /*
1892  * Return the next cookie associated with the specified dring handle
1893  */
1894 int
1895 ldc_mem_dring_nextcookie(ldc_dring_handle_t dhandle, ldc_mem_cookie_t *cookie)
1896 {
1897 	int		rv = 0;
1898 	ldc_dring_t 	*dringp;
1899 	ldc_chan_t	*ldcp;
1900 
1901 	if (dhandle == NULL) {
1902 		DWARN(DBG_ALL_LDCS,
1903 		    "ldc_mem_dring_nextcookie: invalid desc ring handle\n");
1904 		return (EINVAL);
1905 	}
1906 	dringp = (ldc_dring_t *)dhandle;
1907 	mutex_enter(&dringp->lock);
1908 
1909 	if (dringp->status != LDC_BOUND) {
1910 		DWARN(DBG_ALL_LDCS,
1911 		    "ldc_mem_dring_nextcookie: descriptor ring 0x%llx "
1912 		    "is not bound\n", dringp);
1913 		mutex_exit(&dringp->lock);
1914 		return (EINVAL);
1915 	}
1916 
1917 	ldcp = dringp->ldcp;
1918 
1919 	if (cookie == NULL) {
1920 		DWARN(ldcp->id,
1921 		    "ldc_mem_dring_nextcookie:(0x%llx) invalid cookie arg\n",
1922 		    ldcp->id);
1923 		mutex_exit(&dringp->lock);
1924 		return (EINVAL);
1925 	}
1926 
1927 	rv = ldc_mem_nextcookie((ldc_mem_handle_t)dringp->mhdl, cookie);
1928 	mutex_exit(&dringp->lock);
1929 
1930 	return (rv);
1931 }
1932 
1933 /*
1934  * Unbind a previously bound dring from a channel.
1935  */
1936 int
1937 ldc_mem_dring_unbind(ldc_dring_handle_t dhandle)
1938 {
1939 	ldc_dring_t 	*dringp;
1940 	ldc_dring_t	*tmp_dringp;
1941 	ldc_chan_t	*ldcp;
1942 
1943 	if (dhandle == NULL) {
1944 		DWARN(DBG_ALL_LDCS,
1945 		    "ldc_mem_dring_unbind: invalid desc ring handle\n");
1946 		return (EINVAL);
1947 	}
1948 	dringp = (ldc_dring_t *)dhandle;
1949 
1950 	mutex_enter(&dringp->lock);
1951 
1952 	if (dringp->status == LDC_UNBOUND) {
1953 		DWARN(DBG_ALL_LDCS,
1954 		    "ldc_mem_dring_bind: descriptor ring 0x%llx is unbound\n",
1955 		    dringp);
1956 		mutex_exit(&dringp->lock);
1957 		return (EINVAL);
1958 	}
1959 	ldcp = dringp->ldcp;
1960 
1961 	mutex_enter(&ldcp->exp_dlist_lock);
1962 
1963 	tmp_dringp = ldcp->exp_dring_list;
1964 	if (tmp_dringp == dringp) {
1965 		ldcp->exp_dring_list = dringp->ch_next;
1966 		dringp->ch_next = NULL;
1967 
1968 	} else {
1969 		while (tmp_dringp != NULL) {
1970 			if (tmp_dringp->ch_next == dringp) {
1971 				tmp_dringp->ch_next = dringp->ch_next;
1972 				dringp->ch_next = NULL;
1973 				break;
1974 			}
1975 			tmp_dringp = tmp_dringp->ch_next;
1976 		}
1977 		if (tmp_dringp == NULL) {
1978 			DWARN(DBG_ALL_LDCS,
1979 			    "ldc_mem_dring_unbind: invalid descriptor\n");
1980 			mutex_exit(&ldcp->exp_dlist_lock);
1981 			mutex_exit(&dringp->lock);
1982 			return (EINVAL);
1983 		}
1984 	}
1985 
1986 	mutex_exit(&ldcp->exp_dlist_lock);
1987 
1988 	(void) ldc_mem_unbind_handle((ldc_mem_handle_t)dringp->mhdl);
1989 	(void) ldc_mem_free_handle((ldc_mem_handle_t)dringp->mhdl);
1990 
1991 	dringp->ldcp = NULL;
1992 	dringp->mhdl = NULL;
1993 	dringp->status = LDC_UNBOUND;
1994 
1995 	mutex_exit(&dringp->lock);
1996 
1997 	return (0);
1998 }
1999 
2000 #ifdef	DEBUG
2001 void
2002 i_ldc_mem_inject_dring_clear(ldc_chan_t *ldcp)
2003 {
2004 	ldc_dring_t	*dp;
2005 	ldc_mhdl_t	*mhdl;
2006 	ldc_mtbl_t	*mtbl;
2007 	ldc_memseg_t	*memseg;
2008 	uint64_t	cookie_addr;
2009 	uint64_t	pg_shift, pg_size_code;
2010 	int		i, rv, retries;
2011 
2012 	/* has a map table been allocated? */
2013 	if ((mtbl = ldcp->mtbl) == NULL)
2014 		return;
2015 
2016 	/* lock the memory table - exclusive access to channel */
2017 	mutex_enter(&mtbl->lock);
2018 
2019 	/* lock the exported dring list */
2020 	mutex_enter(&ldcp->exp_dlist_lock);
2021 
2022 	for (dp = ldcp->exp_dring_list; dp != NULL; dp = dp->ch_next) {
2023 		if ((mhdl = (ldc_mhdl_t *)dp->mhdl) == NULL)
2024 			continue;
2025 
2026 		if ((memseg = mhdl->memseg) == NULL)
2027 			continue;
2028 
2029 		/* undo the pages exported */
2030 		for (i = 0; i < memseg->npages; i++) {
2031 
2032 			/* clear the entry from the table */
2033 			memseg->pages[i].mte->entry.ll = 0;
2034 
2035 			pg_size_code = page_szc(MMU_PAGESIZE);
2036 			pg_shift = page_get_shift(pg_size_code);
2037 			cookie_addr = IDX2COOKIE(memseg->pages[i].index,
2038 			    pg_size_code, pg_shift);
2039 
2040 			retries = 0;
2041 			do {
2042 				rv = hv_ldc_revoke(ldcp->id, cookie_addr,
2043 				    memseg->pages[i].mte->cookie);
2044 
2045 				if (rv != H_EWOULDBLOCK)
2046 					break;
2047 
2048 				drv_usecwait(ldc_delay);
2049 
2050 			} while (retries++ < ldc_max_retries);
2051 
2052 			if (rv != 0) {
2053 				DWARN(ldcp->id,
2054 				    "i_ldc_mem_inject_dring_clear(): "
2055 				    "hv_ldc_revoke failed: "
2056 				    "channel: 0x%lx, cookie addr: 0x%p,"
2057 				    "cookie: 0x%lx, rv: %d",
2058 				    ldcp->id, cookie_addr,
2059 				    memseg->pages[i].mte->cookie, rv);
2060 			}
2061 
2062 			mtbl->num_avail++;
2063 		}
2064 	}
2065 
2066 	mutex_exit(&ldcp->exp_dlist_lock);
2067 	mutex_exit(&mtbl->lock);
2068 }
2069 #endif
2070 
2071 /*
2072  * Get information about the dring. The base address of the descriptor
2073  * ring along with the type and permission are returned back.
2074  */
2075 int
2076 ldc_mem_dring_info(ldc_dring_handle_t dhandle, ldc_mem_info_t *minfo)
2077 {
2078 	ldc_dring_t	*dringp;
2079 	int		rv;
2080 
2081 	if (dhandle == NULL) {
2082 		DWARN(DBG_ALL_LDCS,
2083 		    "ldc_mem_dring_info: invalid desc ring handle\n");
2084 		return (EINVAL);
2085 	}
2086 	dringp = (ldc_dring_t *)dhandle;
2087 
2088 	mutex_enter(&dringp->lock);
2089 
2090 	if (dringp->mhdl) {
2091 		rv = ldc_mem_info(dringp->mhdl, minfo);
2092 		if (rv) {
2093 			DWARN(DBG_ALL_LDCS,
2094 			    "ldc_mem_dring_info: error reading mem info\n");
2095 			mutex_exit(&dringp->lock);
2096 			return (rv);
2097 		}
2098 	} else {
2099 		minfo->vaddr = dringp->base;
2100 		minfo->raddr = NULL;
2101 		minfo->status = dringp->status;
2102 	}
2103 
2104 	mutex_exit(&dringp->lock);
2105 
2106 	return (0);
2107 }
2108 
2109 /*
2110  * Map an exported descriptor ring into the local address space. If the
2111  * descriptor ring was exported for direct map access, a HV call is made
2112  * to allocate a RA range. If the map is done via a shadow copy, local
2113  * shadow memory is allocated.
2114  */
2115 int
2116 ldc_mem_dring_map(ldc_handle_t handle, ldc_mem_cookie_t *cookie,
2117     uint32_t ccount, uint32_t len, uint32_t dsize, uint8_t mtype,
2118     ldc_dring_handle_t *dhandle)
2119 {
2120 	int		err;
2121 	ldc_chan_t 	*ldcp = (ldc_chan_t *)handle;
2122 	ldc_mem_handle_t mhandle;
2123 	ldc_dring_t	*dringp;
2124 	size_t		dring_size;
2125 
2126 	if (dhandle == NULL) {
2127 		DWARN(DBG_ALL_LDCS,
2128 		    "ldc_mem_dring_map: invalid dhandle\n");
2129 		return (EINVAL);
2130 	}
2131 
2132 	/* check to see if channel is initalized */
2133 	if (handle == NULL) {
2134 		DWARN(DBG_ALL_LDCS,
2135 		    "ldc_mem_dring_map: invalid channel handle\n");
2136 		return (EINVAL);
2137 	}
2138 	ldcp = (ldc_chan_t *)handle;
2139 
2140 	if (cookie == NULL) {
2141 		DWARN(ldcp->id,
2142 		    "ldc_mem_dring_map: (0x%llx) invalid cookie\n",
2143 		    ldcp->id);
2144 		return (EINVAL);
2145 	}
2146 
2147 	/* FUTURE: For now we support only one cookie per dring */
2148 	ASSERT(ccount == 1);
2149 
2150 	if (cookie->size < (dsize * len)) {
2151 		DWARN(ldcp->id,
2152 		    "ldc_mem_dring_map: (0x%llx) invalid dsize/len\n",
2153 		    ldcp->id);
2154 		return (EINVAL);
2155 	}
2156 
2157 	/* ensure the mtype is valid */
2158 	if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP)) == 0) {
2159 		DWARN(ldcp->id, "ldc_mem_dring_map: invalid map type\n");
2160 		return (EINVAL);
2161 	}
2162 
2163 	/* do not attempt direct map if it's not HV supported or enabled */
2164 	if (!ldc_dring_shmem_hv_ok || !ldc_dring_shmem_enabled) {
2165 		mtype = LDC_SHADOW_MAP;
2166 	}
2167 
2168 	*dhandle = 0;
2169 
2170 	/* Allocate an dring structure */
2171 	dringp = kmem_zalloc(sizeof (ldc_dring_t), KM_SLEEP);
2172 
2173 	D1(ldcp->id,
2174 	    "ldc_mem_dring_map: 0x%x,0x%x,0x%x,0x%llx,0x%llx\n",
2175 	    mtype, len, dsize, cookie->addr, cookie->size);
2176 
2177 	/* Initialize dring */
2178 	dringp->length = len;
2179 	dringp->dsize = dsize;
2180 
2181 	/* round of to multiple of page size */
2182 	dring_size = len * dsize;
2183 	dringp->size = (dring_size & MMU_PAGEMASK);
2184 	if (dring_size & MMU_PAGEOFFSET)
2185 		dringp->size += MMU_PAGESIZE;
2186 
2187 	dringp->ldcp = ldcp;
2188 
2189 	/* create an memory handle */
2190 	err = ldc_mem_alloc_handle(handle, &mhandle);
2191 	if (err || mhandle == NULL) {
2192 		DWARN(DBG_ALL_LDCS,
2193 		    "ldc_mem_dring_map: cannot alloc hdl err=%d\n",
2194 		    err);
2195 		kmem_free(dringp, sizeof (ldc_dring_t));
2196 		return (ENOMEM);
2197 	}
2198 
2199 	dringp->mhdl = mhandle;
2200 	dringp->base = NULL;
2201 
2202 	/* map the dring into local memory */
2203 	err = i_ldc_mem_map(mhandle, cookie, ccount, mtype, LDC_MEM_RW,
2204 	    &(dringp->base), NULL);
2205 	if (err || dringp->base == NULL) {
2206 		DWARN(DBG_ALL_LDCS,
2207 		    "ldc_mem_dring_map: cannot map desc ring err=%d\n", err);
2208 		(void) ldc_mem_free_handle(mhandle);
2209 		kmem_free(dringp, sizeof (ldc_dring_t));
2210 		return (ENOMEM);
2211 	}
2212 
2213 	/* initialize the desc ring lock */
2214 	mutex_init(&dringp->lock, NULL, MUTEX_DRIVER, NULL);
2215 
2216 	/* Add descriptor ring to channel's imported dring list */
2217 	mutex_enter(&ldcp->imp_dlist_lock);
2218 	dringp->ch_next = ldcp->imp_dring_list;
2219 	ldcp->imp_dring_list = dringp;
2220 	mutex_exit(&ldcp->imp_dlist_lock);
2221 
2222 	dringp->status = LDC_MAPPED;
2223 
2224 	*dhandle = (ldc_dring_handle_t)dringp;
2225 
2226 	return (0);
2227 }
2228 
2229 /*
2230  * Unmap a descriptor ring. Free shadow memory (if any).
2231  */
2232 int
2233 ldc_mem_dring_unmap(ldc_dring_handle_t dhandle)
2234 {
2235 	ldc_dring_t 	*dringp;
2236 	ldc_dring_t	*tmp_dringp;
2237 	ldc_chan_t	*ldcp;
2238 
2239 	if (dhandle == NULL) {
2240 		DWARN(DBG_ALL_LDCS,
2241 		    "ldc_mem_dring_unmap: invalid desc ring handle\n");
2242 		return (EINVAL);
2243 	}
2244 	dringp = (ldc_dring_t *)dhandle;
2245 
2246 	if (dringp->status != LDC_MAPPED) {
2247 		DWARN(DBG_ALL_LDCS,
2248 		    "ldc_mem_dring_unmap: not a mapped desc ring\n");
2249 		return (EINVAL);
2250 	}
2251 
2252 	mutex_enter(&dringp->lock);
2253 
2254 	ldcp = dringp->ldcp;
2255 
2256 	mutex_enter(&ldcp->imp_dlist_lock);
2257 
2258 	/* find and unlink the desc ring from channel import list */
2259 	tmp_dringp = ldcp->imp_dring_list;
2260 	if (tmp_dringp == dringp) {
2261 		ldcp->imp_dring_list = dringp->ch_next;
2262 		dringp->ch_next = NULL;
2263 
2264 	} else {
2265 		while (tmp_dringp != NULL) {
2266 			if (tmp_dringp->ch_next == dringp) {
2267 				tmp_dringp->ch_next = dringp->ch_next;
2268 				dringp->ch_next = NULL;
2269 				break;
2270 			}
2271 			tmp_dringp = tmp_dringp->ch_next;
2272 		}
2273 		if (tmp_dringp == NULL) {
2274 			DWARN(DBG_ALL_LDCS,
2275 			    "ldc_mem_dring_unmap: invalid descriptor\n");
2276 			mutex_exit(&ldcp->imp_dlist_lock);
2277 			mutex_exit(&dringp->lock);
2278 			return (EINVAL);
2279 		}
2280 	}
2281 
2282 	mutex_exit(&ldcp->imp_dlist_lock);
2283 
2284 	/* do a LDC memory handle unmap and free */
2285 	(void) ldc_mem_unmap(dringp->mhdl);
2286 	(void) ldc_mem_free_handle((ldc_mem_handle_t)dringp->mhdl);
2287 
2288 	dringp->status = 0;
2289 	dringp->ldcp = NULL;
2290 
2291 	mutex_exit(&dringp->lock);
2292 
2293 	/* destroy dring lock */
2294 	mutex_destroy(&dringp->lock);
2295 
2296 	/* free desc ring object */
2297 	kmem_free(dringp, sizeof (ldc_dring_t));
2298 
2299 	return (0);
2300 }
2301 
2302 /*
2303  * Internal entry point for descriptor ring access entry consistency
2304  * semantics. Acquire copies the contents of the remote descriptor ring
2305  * into the local shadow copy. The release operation copies the local
2306  * contents into the remote dring. The start and end locations specify
2307  * bounds for the entries being synchronized.
2308  */
2309 static int
2310 i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle,
2311     uint8_t direction, uint64_t start, uint64_t end)
2312 {
2313 	int 			err;
2314 	ldc_dring_t		*dringp;
2315 	ldc_chan_t		*ldcp;
2316 	ldc_mhdl_t		*mhdl;
2317 	uint64_t		soff;
2318 	size_t			copy_size;
2319 
2320 	if (dhandle == NULL) {
2321 		DWARN(DBG_ALL_LDCS,
2322 		    "i_ldc_dring_acquire_release: invalid desc ring handle\n");
2323 		return (EINVAL);
2324 	}
2325 	dringp = (ldc_dring_t *)dhandle;
2326 	mutex_enter(&dringp->lock);
2327 
2328 	if (dringp->status != LDC_MAPPED || dringp->ldcp == NULL) {
2329 		DWARN(DBG_ALL_LDCS,
2330 		    "i_ldc_dring_acquire_release: not a mapped desc ring\n");
2331 		mutex_exit(&dringp->lock);
2332 		return (EINVAL);
2333 	}
2334 
2335 	if (start >= dringp->length || end >= dringp->length) {
2336 		DWARN(DBG_ALL_LDCS,
2337 		    "i_ldc_dring_acquire_release: index out of range\n");
2338 		mutex_exit(&dringp->lock);
2339 		return (EINVAL);
2340 	}
2341 
2342 	mhdl = (ldc_mhdl_t *)dringp->mhdl;
2343 	if (mhdl == NULL) {
2344 		DWARN(DBG_ALL_LDCS,
2345 		    "i_ldc_dring_acquire_release: invalid memory handle\n");
2346 		mutex_exit(&dringp->lock);
2347 		return (EINVAL);
2348 	}
2349 
2350 	if (mhdl->mtype != LDC_SHADOW_MAP) {
2351 		DWARN(DBG_ALL_LDCS,
2352 		    "i_ldc_dring_acquire_release: invalid mtype: %d\n",
2353 		    mhdl->mtype);
2354 		mutex_exit(&dringp->lock);
2355 		return (EINVAL);
2356 	}
2357 
2358 	/* get the channel handle */
2359 	ldcp = dringp->ldcp;
2360 
2361 	copy_size = (start <= end) ? (((end - start) + 1) * dringp->dsize) :
2362 	    ((dringp->length - start) * dringp->dsize);
2363 
2364 	/* Calculate the relative offset for the first desc */
2365 	soff = (start * dringp->dsize);
2366 
2367 	/* copy to/from remote from/to local memory */
2368 	D1(ldcp->id, "i_ldc_dring_acquire_release: c1 off=0x%llx sz=0x%llx\n",
2369 	    soff, copy_size);
2370 	err = i_ldc_mem_acquire_release((ldc_mem_handle_t)dringp->mhdl,
2371 	    direction, soff, copy_size);
2372 	if (err) {
2373 		DWARN(ldcp->id,
2374 		    "i_ldc_dring_acquire_release: copy failed\n");
2375 		mutex_exit(&dringp->lock);
2376 		return (err);
2377 	}
2378 
2379 	/* do the balance */
2380 	if (start > end) {
2381 		copy_size = ((end + 1) * dringp->dsize);
2382 		soff = 0;
2383 
2384 		/* copy to/from remote from/to local memory */
2385 		D1(ldcp->id, "i_ldc_dring_acquire_release: c2 "
2386 		    "off=0x%llx sz=0x%llx\n", soff, copy_size);
2387 		err = i_ldc_mem_acquire_release((ldc_mem_handle_t)dringp->mhdl,
2388 		    direction, soff, copy_size);
2389 		if (err) {
2390 			DWARN(ldcp->id,
2391 			    "i_ldc_dring_acquire_release: copy failed\n");
2392 			mutex_exit(&dringp->lock);
2393 			return (err);
2394 		}
2395 	}
2396 
2397 	mutex_exit(&dringp->lock);
2398 
2399 	return (0);
2400 }
2401 
2402 /*
2403  * Ensure that the contents in the local dring are consistent
2404  * with the contents if of remote dring
2405  */
2406 int
2407 ldc_mem_dring_acquire(ldc_dring_handle_t dhandle, uint64_t start, uint64_t end)
2408 {
2409 	return (i_ldc_dring_acquire_release(dhandle, LDC_COPY_IN, start, end));
2410 }
2411 
2412 /*
2413  * Ensure that the contents in the remote dring are consistent
2414  * with the contents if of local dring
2415  */
2416 int
2417 ldc_mem_dring_release(ldc_dring_handle_t dhandle, uint64_t start, uint64_t end)
2418 {
2419 	return (i_ldc_dring_acquire_release(dhandle, LDC_COPY_OUT, start, end));
2420 }
2421