1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * sun4v LDC Link Layer Shared Memory Routines 31 */ 32 #include <sys/types.h> 33 #include <sys/kmem.h> 34 #include <sys/cmn_err.h> 35 #include <sys/ksynch.h> 36 #include <sys/debug.h> 37 #include <sys/cyclic.h> 38 #include <sys/machsystm.h> 39 #include <sys/vm.h> 40 #include <sys/machcpuvar.h> 41 #include <sys/mmu.h> 42 #include <sys/pte.h> 43 #include <vm/hat.h> 44 #include <vm/as.h> 45 #include <vm/hat_sfmmu.h> 46 #include <sys/vm_machparam.h> 47 #include <vm/seg_kmem.h> 48 #include <vm/seg_kpm.h> 49 #include <sys/hypervisor_api.h> 50 #include <sys/ldc.h> 51 #include <sys/ldc_impl.h> 52 53 /* LDC variables used by shared memory routines */ 54 extern ldc_soft_state_t *ldcssp; 55 extern int ldc_max_retries; 56 extern clock_t ldc_delay; 57 58 #ifdef DEBUG 59 extern int ldcdbg; 60 #endif 61 62 /* LDC internal functions used by shared memory routines */ 63 extern void i_ldc_reset(ldc_chan_t *ldcp, boolean_t force_reset); 64 extern int i_ldc_h2v_error(int h_error); 65 66 #ifdef DEBUG 67 extern void ldcdebug(int64_t id, const char *fmt, ...); 68 #endif 69 70 /* Memory synchronization internal functions */ 71 static int i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle, 72 uint8_t direction, uint64_t offset, size_t size); 73 static int i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle, 74 uint8_t direction, uint64_t start, uint64_t end); 75 static int i_ldc_mem_map(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie, 76 uint32_t ccount, uint8_t mtype, uint8_t perm, caddr_t *vaddr, 77 caddr_t *raddr); 78 static int i_ldc_mem_bind_handle(ldc_mem_handle_t mhandle, caddr_t vaddr, 79 size_t len, uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, 80 uint32_t *ccount); 81 82 /* 83 * LDC framework supports mapping remote domain's memory 84 * either directly or via shadow memory pages. Default 85 * support is currently implemented via shadow copy. 86 * Direct map can be enabled by setting 'ldc_shmem_enabled' 87 */ 88 int ldc_shmem_enabled = 0; 89 90 /* 91 * Use of directly mapped shared memory for LDC descriptor 92 * rings is permitted if this variable is non-zero. 93 */ 94 int ldc_dring_shmem_enabled = 1; 95 96 /* 97 * The major and minor versions required to use directly 98 * mapped shared memory for LDC descriptor rings. The 99 * ldc_dring_shmem_hv_force variable, if set to a non-zero 100 * value, overrides the hypervisor API version check. 101 */ 102 static int ldc_dring_shmem_hv_major = 1; 103 static int ldc_dring_shmem_hv_minor = 1; 104 static int ldc_dring_shmem_hv_force = 0; 105 106 /* 107 * The results of the hypervisor service group API check. 108 * A non-zero value indicates the HV includes support for 109 * descriptor ring shared memory. 110 */ 111 static int ldc_dring_shmem_hv_ok = 0; 112 113 /* 114 * Pages exported for remote access over each channel is 115 * maintained in a table registered with the Hypervisor. 116 * The default number of entries in the table is set to 117 * 'ldc_mtbl_entries'. 118 */ 119 uint64_t ldc_maptable_entries = LDC_MTBL_ENTRIES; 120 121 #define IDX2COOKIE(idx, pg_szc, pg_shift) \ 122 (((pg_szc) << LDC_COOKIE_PGSZC_SHIFT) | ((idx) << (pg_shift))) 123 124 /* 125 * Sets ldc_dring_shmem_hv_ok to a non-zero value if the HV LDC 126 * API version supports directly mapped shared memory or if it has 127 * been explicitly enabled via ldc_dring_shmem_hv_force. 128 */ 129 void 130 i_ldc_mem_set_hsvc_vers(uint64_t major, uint64_t minor) 131 { 132 if ((major == ldc_dring_shmem_hv_major && 133 minor >= ldc_dring_shmem_hv_minor) || 134 (major > ldc_dring_shmem_hv_major) || 135 (ldc_dring_shmem_hv_force != 0)) { 136 ldc_dring_shmem_hv_ok = 1; 137 } 138 } 139 140 /* 141 * Allocate a memory handle for the channel and link it into the list 142 * Also choose which memory table to use if this is the first handle 143 * being assigned to this channel 144 */ 145 int 146 ldc_mem_alloc_handle(ldc_handle_t handle, ldc_mem_handle_t *mhandle) 147 { 148 ldc_chan_t *ldcp; 149 ldc_mhdl_t *mhdl; 150 151 if (handle == NULL) { 152 DWARN(DBG_ALL_LDCS, 153 "ldc_mem_alloc_handle: invalid channel handle\n"); 154 return (EINVAL); 155 } 156 ldcp = (ldc_chan_t *)handle; 157 158 mutex_enter(&ldcp->lock); 159 160 /* check to see if channel is initalized */ 161 if ((ldcp->tstate & ~TS_IN_RESET) < TS_INIT) { 162 DWARN(ldcp->id, 163 "ldc_mem_alloc_handle: (0x%llx) channel not initialized\n", 164 ldcp->id); 165 mutex_exit(&ldcp->lock); 166 return (EINVAL); 167 } 168 169 /* allocate handle for channel */ 170 mhdl = kmem_cache_alloc(ldcssp->memhdl_cache, KM_SLEEP); 171 172 /* initialize the lock */ 173 mutex_init(&mhdl->lock, NULL, MUTEX_DRIVER, NULL); 174 175 mhdl->myshadow = B_FALSE; 176 mhdl->memseg = NULL; 177 mhdl->ldcp = ldcp; 178 mhdl->status = LDC_UNBOUND; 179 180 /* insert memory handle (@ head) into list */ 181 if (ldcp->mhdl_list == NULL) { 182 ldcp->mhdl_list = mhdl; 183 mhdl->next = NULL; 184 } else { 185 /* insert @ head */ 186 mhdl->next = ldcp->mhdl_list; 187 ldcp->mhdl_list = mhdl; 188 } 189 190 /* return the handle */ 191 *mhandle = (ldc_mem_handle_t)mhdl; 192 193 mutex_exit(&ldcp->lock); 194 195 D1(ldcp->id, "ldc_mem_alloc_handle: (0x%llx) allocated handle 0x%llx\n", 196 ldcp->id, mhdl); 197 198 return (0); 199 } 200 201 /* 202 * Free memory handle for the channel and unlink it from the list 203 */ 204 int 205 ldc_mem_free_handle(ldc_mem_handle_t mhandle) 206 { 207 ldc_mhdl_t *mhdl, *phdl; 208 ldc_chan_t *ldcp; 209 210 if (mhandle == NULL) { 211 DWARN(DBG_ALL_LDCS, 212 "ldc_mem_free_handle: invalid memory handle\n"); 213 return (EINVAL); 214 } 215 mhdl = (ldc_mhdl_t *)mhandle; 216 217 mutex_enter(&mhdl->lock); 218 219 ldcp = mhdl->ldcp; 220 221 if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED) { 222 DWARN(ldcp->id, 223 "ldc_mem_free_handle: cannot free, 0x%llx hdl bound\n", 224 mhdl); 225 mutex_exit(&mhdl->lock); 226 return (EINVAL); 227 } 228 mutex_exit(&mhdl->lock); 229 230 mutex_enter(&ldcp->mlist_lock); 231 232 phdl = ldcp->mhdl_list; 233 234 /* first handle */ 235 if (phdl == mhdl) { 236 ldcp->mhdl_list = mhdl->next; 237 mutex_destroy(&mhdl->lock); 238 kmem_cache_free(ldcssp->memhdl_cache, mhdl); 239 240 D1(ldcp->id, 241 "ldc_mem_free_handle: (0x%llx) freed handle 0x%llx\n", 242 ldcp->id, mhdl); 243 } else { 244 /* walk the list - unlink and free */ 245 while (phdl != NULL) { 246 if (phdl->next == mhdl) { 247 phdl->next = mhdl->next; 248 mutex_destroy(&mhdl->lock); 249 kmem_cache_free(ldcssp->memhdl_cache, mhdl); 250 D1(ldcp->id, 251 "ldc_mem_free_handle: (0x%llx) freed " 252 "handle 0x%llx\n", ldcp->id, mhdl); 253 break; 254 } 255 phdl = phdl->next; 256 } 257 } 258 259 if (phdl == NULL) { 260 DWARN(ldcp->id, 261 "ldc_mem_free_handle: invalid handle 0x%llx\n", mhdl); 262 mutex_exit(&ldcp->mlist_lock); 263 return (EINVAL); 264 } 265 266 mutex_exit(&ldcp->mlist_lock); 267 268 return (0); 269 } 270 271 /* 272 * Bind a memory handle to a virtual address. 273 * The virtual address is converted to the corresponding real addresses. 274 * Returns pointer to the first ldc_mem_cookie and the total number 275 * of cookies for this virtual address. Other cookies can be obtained 276 * using the ldc_mem_nextcookie() call. If the pages are stored in 277 * consecutive locations in the table, a single cookie corresponding to 278 * the first location is returned. The cookie size spans all the entries. 279 * 280 * If the VA corresponds to a page that is already being exported, reuse 281 * the page and do not export it again. Bump the page's use count. 282 */ 283 int 284 ldc_mem_bind_handle(ldc_mem_handle_t mhandle, caddr_t vaddr, size_t len, 285 uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount) 286 { 287 /* 288 * Check if direct shared memory map is enabled, if not change 289 * the mapping type to SHADOW_MAP. 290 */ 291 if (ldc_shmem_enabled == 0) 292 mtype = LDC_SHADOW_MAP; 293 294 return (i_ldc_mem_bind_handle(mhandle, vaddr, len, mtype, perm, 295 cookie, ccount)); 296 } 297 298 static int 299 i_ldc_mem_bind_handle(ldc_mem_handle_t mhandle, caddr_t vaddr, size_t len, 300 uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount) 301 { 302 ldc_mhdl_t *mhdl; 303 ldc_chan_t *ldcp; 304 ldc_mtbl_t *mtbl; 305 ldc_memseg_t *memseg; 306 ldc_mte_t tmp_mte; 307 uint64_t index, prev_index = 0; 308 int64_t cookie_idx; 309 uintptr_t raddr, ra_aligned; 310 uint64_t psize, poffset, v_offset; 311 uint64_t pg_shift, pg_size, pg_size_code, pg_mask; 312 pgcnt_t npages; 313 caddr_t v_align, addr; 314 int i, rv; 315 316 if (mhandle == NULL) { 317 DWARN(DBG_ALL_LDCS, 318 "ldc_mem_bind_handle: invalid memory handle\n"); 319 return (EINVAL); 320 } 321 mhdl = (ldc_mhdl_t *)mhandle; 322 ldcp = mhdl->ldcp; 323 324 /* clear count */ 325 *ccount = 0; 326 327 mutex_enter(&mhdl->lock); 328 329 if (mhdl->status == LDC_BOUND || mhdl->memseg != NULL) { 330 DWARN(ldcp->id, 331 "ldc_mem_bind_handle: (0x%x) handle already bound\n", 332 mhandle); 333 mutex_exit(&mhdl->lock); 334 return (EINVAL); 335 } 336 337 /* Force address and size to be 8-byte aligned */ 338 if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 339 DWARN(ldcp->id, 340 "ldc_mem_bind_handle: addr/size is not 8-byte aligned\n"); 341 mutex_exit(&mhdl->lock); 342 return (EINVAL); 343 } 344 345 mutex_enter(&ldcp->lock); 346 347 /* 348 * If this channel is binding a memory handle for the 349 * first time allocate it a memory map table and initialize it 350 */ 351 if ((mtbl = ldcp->mtbl) == NULL) { 352 353 /* Allocate and initialize the map table structure */ 354 mtbl = kmem_zalloc(sizeof (ldc_mtbl_t), KM_SLEEP); 355 mtbl->num_entries = mtbl->num_avail = ldc_maptable_entries; 356 mtbl->size = ldc_maptable_entries * sizeof (ldc_mte_slot_t); 357 mtbl->next_entry = NULL; 358 mtbl->contigmem = B_TRUE; 359 360 /* Allocate the table itself */ 361 mtbl->table = (ldc_mte_slot_t *) 362 contig_mem_alloc_align(mtbl->size, MMU_PAGESIZE); 363 if (mtbl->table == NULL) { 364 365 /* allocate a page of memory using kmem_alloc */ 366 mtbl->table = kmem_alloc(MMU_PAGESIZE, KM_SLEEP); 367 mtbl->size = MMU_PAGESIZE; 368 mtbl->contigmem = B_FALSE; 369 mtbl->num_entries = mtbl->num_avail = 370 mtbl->size / sizeof (ldc_mte_slot_t); 371 DWARN(ldcp->id, 372 "ldc_mem_bind_handle: (0x%llx) reduced tbl size " 373 "to %lx entries\n", ldcp->id, mtbl->num_entries); 374 } 375 376 /* zero out the memory */ 377 bzero(mtbl->table, mtbl->size); 378 379 /* initialize the lock */ 380 mutex_init(&mtbl->lock, NULL, MUTEX_DRIVER, NULL); 381 382 /* register table for this channel */ 383 rv = hv_ldc_set_map_table(ldcp->id, 384 va_to_pa(mtbl->table), mtbl->num_entries); 385 if (rv != 0) { 386 DWARN(DBG_ALL_LDCS, 387 "ldc_mem_bind_handle: (0x%lx) err %d mapping tbl", 388 ldcp->id, rv); 389 if (mtbl->contigmem) 390 contig_mem_free(mtbl->table, mtbl->size); 391 else 392 kmem_free(mtbl->table, mtbl->size); 393 mutex_destroy(&mtbl->lock); 394 kmem_free(mtbl, sizeof (ldc_mtbl_t)); 395 mutex_exit(&ldcp->lock); 396 mutex_exit(&mhdl->lock); 397 return (EIO); 398 } 399 400 ldcp->mtbl = mtbl; 401 402 D1(ldcp->id, 403 "ldc_mem_bind_handle: (0x%llx) alloc'd map table 0x%llx\n", 404 ldcp->id, ldcp->mtbl->table); 405 } 406 407 mutex_exit(&ldcp->lock); 408 409 /* FUTURE: get the page size, pgsz code, and shift */ 410 pg_size = MMU_PAGESIZE; 411 pg_size_code = page_szc(pg_size); 412 pg_shift = page_get_shift(pg_size_code); 413 pg_mask = ~(pg_size - 1); 414 415 D1(ldcp->id, "ldc_mem_bind_handle: (0x%llx) binding " 416 "va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 417 ldcp->id, vaddr, pg_size, pg_size_code, pg_shift); 418 419 /* aligned VA and its offset */ 420 v_align = (caddr_t)(((uintptr_t)vaddr) & ~(pg_size - 1)); 421 v_offset = ((uintptr_t)vaddr) & (pg_size - 1); 422 423 npages = (len+v_offset)/pg_size; 424 npages = ((len+v_offset)%pg_size == 0) ? npages : npages+1; 425 426 D1(ldcp->id, "ldc_mem_bind_handle: binding " 427 "(0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n", 428 ldcp->id, vaddr, v_align, v_offset, npages); 429 430 /* lock the memory table - exclusive access to channel */ 431 mutex_enter(&mtbl->lock); 432 433 if (npages > mtbl->num_avail) { 434 D1(ldcp->id, "ldc_mem_bind_handle: (0x%llx) no table entries\n", 435 ldcp->id); 436 mutex_exit(&mtbl->lock); 437 mutex_exit(&mhdl->lock); 438 return (ENOMEM); 439 } 440 441 /* Allocate a memseg structure */ 442 memseg = mhdl->memseg = 443 kmem_cache_alloc(ldcssp->memseg_cache, KM_SLEEP); 444 445 /* Allocate memory to store all pages and cookies */ 446 memseg->pages = kmem_zalloc((sizeof (ldc_page_t) * npages), KM_SLEEP); 447 memseg->cookies = 448 kmem_zalloc((sizeof (ldc_mem_cookie_t) * npages), KM_SLEEP); 449 450 D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) processing 0x%llx pages\n", 451 ldcp->id, npages); 452 453 addr = v_align; 454 455 /* 456 * Table slots are used in a round-robin manner. The algorithm permits 457 * inserting duplicate entries. Slots allocated earlier will typically 458 * get freed before we get back to reusing the slot.Inserting duplicate 459 * entries should be OK as we only lookup entries using the cookie addr 460 * i.e. tbl index, during export, unexport and copy operation. 461 * 462 * One implementation what was tried was to search for a duplicate 463 * page entry first and reuse it. The search overhead is very high and 464 * in the vnet case dropped the perf by almost half, 50 to 24 mbps. 465 * So it does make sense to avoid searching for duplicates. 466 * 467 * But during the process of searching for a free slot, if we find a 468 * duplicate entry we will go ahead and use it, and bump its use count. 469 */ 470 471 /* index to start searching from */ 472 index = mtbl->next_entry; 473 cookie_idx = -1; 474 475 tmp_mte.ll = 0; /* initialise fields to 0 */ 476 477 if (mtype & LDC_DIRECT_MAP) { 478 tmp_mte.mte_r = (perm & LDC_MEM_R) ? 1 : 0; 479 tmp_mte.mte_w = (perm & LDC_MEM_W) ? 1 : 0; 480 tmp_mte.mte_x = (perm & LDC_MEM_X) ? 1 : 0; 481 } 482 483 if (mtype & LDC_SHADOW_MAP) { 484 tmp_mte.mte_cr = (perm & LDC_MEM_R) ? 1 : 0; 485 tmp_mte.mte_cw = (perm & LDC_MEM_W) ? 1 : 0; 486 } 487 488 if (mtype & LDC_IO_MAP) { 489 tmp_mte.mte_ir = (perm & LDC_MEM_R) ? 1 : 0; 490 tmp_mte.mte_iw = (perm & LDC_MEM_W) ? 1 : 0; 491 } 492 493 D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll); 494 495 tmp_mte.mte_pgszc = pg_size_code; 496 497 /* initialize each mem table entry */ 498 for (i = 0; i < npages; i++) { 499 500 /* check if slot is available in the table */ 501 while (mtbl->table[index].entry.ll != 0) { 502 503 index = (index + 1) % mtbl->num_entries; 504 505 if (index == mtbl->next_entry) { 506 /* we have looped around */ 507 DWARN(DBG_ALL_LDCS, 508 "ldc_mem_bind_handle: (0x%llx) cannot find " 509 "entry\n", ldcp->id); 510 *ccount = 0; 511 512 /* NOTE: free memory, remove previous entries */ 513 /* this shouldnt happen as num_avail was ok */ 514 515 mutex_exit(&mtbl->lock); 516 mutex_exit(&mhdl->lock); 517 return (ENOMEM); 518 } 519 } 520 521 /* get the real address */ 522 raddr = va_to_pa((void *)addr); 523 ra_aligned = ((uintptr_t)raddr & pg_mask); 524 525 /* build the mte */ 526 tmp_mte.mte_rpfn = ra_aligned >> pg_shift; 527 528 D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll); 529 530 /* update entry in table */ 531 mtbl->table[index].entry = tmp_mte; 532 533 D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) stored MTE 0x%llx" 534 " into loc 0x%llx\n", ldcp->id, tmp_mte.ll, index); 535 536 /* calculate the size and offset for this export range */ 537 if (i == 0) { 538 /* first page */ 539 psize = min((pg_size - v_offset), len); 540 poffset = v_offset; 541 542 } else if (i == (npages - 1)) { 543 /* last page */ 544 psize = (((uintptr_t)(vaddr + len)) & 545 ((uint64_t)(pg_size-1))); 546 if (psize == 0) 547 psize = pg_size; 548 poffset = 0; 549 550 } else { 551 /* middle pages */ 552 psize = pg_size; 553 poffset = 0; 554 } 555 556 /* store entry for this page */ 557 memseg->pages[i].index = index; 558 memseg->pages[i].raddr = raddr; 559 memseg->pages[i].mte = &(mtbl->table[index]); 560 561 /* create the cookie */ 562 if (i == 0 || (index != prev_index + 1)) { 563 cookie_idx++; 564 memseg->cookies[cookie_idx].addr = 565 IDX2COOKIE(index, pg_size_code, pg_shift); 566 memseg->cookies[cookie_idx].addr |= poffset; 567 memseg->cookies[cookie_idx].size = psize; 568 569 } else { 570 memseg->cookies[cookie_idx].size += psize; 571 } 572 573 D1(ldcp->id, "ldc_mem_bind_handle: bound " 574 "(0x%llx) va=0x%llx, idx=0x%llx, " 575 "ra=0x%llx(sz=0x%x,off=0x%x)\n", 576 ldcp->id, addr, index, raddr, psize, poffset); 577 578 /* decrement number of available entries */ 579 mtbl->num_avail--; 580 581 /* increment va by page size */ 582 addr += pg_size; 583 584 /* increment index */ 585 prev_index = index; 586 index = (index + 1) % mtbl->num_entries; 587 588 /* save the next slot */ 589 mtbl->next_entry = index; 590 } 591 592 mutex_exit(&mtbl->lock); 593 594 /* memory handle = bound */ 595 mhdl->mtype = mtype; 596 mhdl->perm = perm; 597 mhdl->status = LDC_BOUND; 598 599 /* update memseg_t */ 600 memseg->vaddr = vaddr; 601 memseg->raddr = memseg->pages[0].raddr; 602 memseg->size = len; 603 memseg->npages = npages; 604 memseg->ncookies = cookie_idx + 1; 605 memseg->next_cookie = (memseg->ncookies > 1) ? 1 : 0; 606 607 /* return count and first cookie */ 608 *ccount = memseg->ncookies; 609 cookie->addr = memseg->cookies[0].addr; 610 cookie->size = memseg->cookies[0].size; 611 612 D1(ldcp->id, 613 "ldc_mem_bind_handle: (0x%llx) bound 0x%llx, va=0x%llx, " 614 "pgs=0x%llx cookies=0x%llx\n", 615 ldcp->id, mhdl, vaddr, npages, memseg->ncookies); 616 617 mutex_exit(&mhdl->lock); 618 return (0); 619 } 620 621 /* 622 * Return the next cookie associated with the specified memory handle 623 */ 624 int 625 ldc_mem_nextcookie(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie) 626 { 627 ldc_mhdl_t *mhdl; 628 ldc_chan_t *ldcp; 629 ldc_memseg_t *memseg; 630 631 if (mhandle == NULL) { 632 DWARN(DBG_ALL_LDCS, 633 "ldc_mem_nextcookie: invalid memory handle\n"); 634 return (EINVAL); 635 } 636 mhdl = (ldc_mhdl_t *)mhandle; 637 638 mutex_enter(&mhdl->lock); 639 640 ldcp = mhdl->ldcp; 641 memseg = mhdl->memseg; 642 643 if (cookie == 0) { 644 DWARN(ldcp->id, 645 "ldc_mem_nextcookie:(0x%llx) invalid cookie arg\n", 646 ldcp->id); 647 mutex_exit(&mhdl->lock); 648 return (EINVAL); 649 } 650 651 if (memseg->next_cookie != 0) { 652 cookie->addr = memseg->cookies[memseg->next_cookie].addr; 653 cookie->size = memseg->cookies[memseg->next_cookie].size; 654 memseg->next_cookie++; 655 if (memseg->next_cookie == memseg->ncookies) 656 memseg->next_cookie = 0; 657 658 } else { 659 DWARN(ldcp->id, 660 "ldc_mem_nextcookie:(0x%llx) no more cookies\n", ldcp->id); 661 cookie->addr = 0; 662 cookie->size = 0; 663 mutex_exit(&mhdl->lock); 664 return (EINVAL); 665 } 666 667 D1(ldcp->id, 668 "ldc_mem_nextcookie: (0x%llx) cookie addr=0x%llx,sz=0x%llx\n", 669 ldcp->id, cookie->addr, cookie->size); 670 671 mutex_exit(&mhdl->lock); 672 return (0); 673 } 674 675 /* 676 * Unbind the virtual memory region associated with the specified 677 * memory handle. Allassociated cookies are freed and the corresponding 678 * RA space is no longer exported. 679 */ 680 int 681 ldc_mem_unbind_handle(ldc_mem_handle_t mhandle) 682 { 683 ldc_mhdl_t *mhdl; 684 ldc_chan_t *ldcp; 685 ldc_mtbl_t *mtbl; 686 ldc_memseg_t *memseg; 687 uint64_t cookie_addr; 688 uint64_t pg_shift, pg_size_code; 689 int i, rv, retries; 690 691 if (mhandle == NULL) { 692 DWARN(DBG_ALL_LDCS, 693 "ldc_mem_unbind_handle: invalid memory handle\n"); 694 return (EINVAL); 695 } 696 mhdl = (ldc_mhdl_t *)mhandle; 697 698 mutex_enter(&mhdl->lock); 699 700 if (mhdl->status == LDC_UNBOUND) { 701 DWARN(DBG_ALL_LDCS, 702 "ldc_mem_unbind_handle: (0x%x) handle is not bound\n", 703 mhandle); 704 mutex_exit(&mhdl->lock); 705 return (EINVAL); 706 } 707 708 ldcp = mhdl->ldcp; 709 mtbl = ldcp->mtbl; 710 711 memseg = mhdl->memseg; 712 713 /* lock the memory table - exclusive access to channel */ 714 mutex_enter(&mtbl->lock); 715 716 /* undo the pages exported */ 717 for (i = 0; i < memseg->npages; i++) { 718 719 /* clear the entry from the table */ 720 memseg->pages[i].mte->entry.ll = 0; 721 722 /* check for mapped pages, revocation cookie != 0 */ 723 if (memseg->pages[i].mte->cookie) { 724 725 pg_size_code = page_szc(MMU_PAGESIZE); 726 pg_shift = page_get_shift(pg_size_code); 727 cookie_addr = IDX2COOKIE(memseg->pages[i].index, 728 pg_size_code, pg_shift); 729 730 D1(ldcp->id, "ldc_mem_unbind_handle: (0x%llx) revoke " 731 "cookie 0x%llx, rcookie 0x%llx\n", ldcp->id, 732 cookie_addr, memseg->pages[i].mte->cookie); 733 734 retries = 0; 735 do { 736 rv = hv_ldc_revoke(ldcp->id, cookie_addr, 737 memseg->pages[i].mte->cookie); 738 739 if (rv != H_EWOULDBLOCK) 740 break; 741 742 drv_usecwait(ldc_delay); 743 744 } while (retries++ < ldc_max_retries); 745 746 if (rv) { 747 DWARN(ldcp->id, 748 "ldc_mem_unbind_handle: (0x%llx) cannot " 749 "revoke mapping, cookie %llx\n", ldcp->id, 750 cookie_addr); 751 } 752 } 753 754 mtbl->num_avail++; 755 } 756 mutex_exit(&mtbl->lock); 757 758 /* free the allocated memseg and page structures */ 759 kmem_free(memseg->pages, (sizeof (ldc_page_t) * memseg->npages)); 760 kmem_free(memseg->cookies, 761 (sizeof (ldc_mem_cookie_t) * memseg->npages)); 762 kmem_cache_free(ldcssp->memseg_cache, memseg); 763 764 /* uninitialize the memory handle */ 765 mhdl->memseg = NULL; 766 mhdl->status = LDC_UNBOUND; 767 768 D1(ldcp->id, "ldc_mem_unbind_handle: (0x%llx) unbound handle 0x%llx\n", 769 ldcp->id, mhdl); 770 771 mutex_exit(&mhdl->lock); 772 return (0); 773 } 774 775 /* 776 * Get information about the dring. The base address of the descriptor 777 * ring along with the type and permission are returned back. 778 */ 779 int 780 ldc_mem_info(ldc_mem_handle_t mhandle, ldc_mem_info_t *minfo) 781 { 782 ldc_mhdl_t *mhdl; 783 784 if (mhandle == NULL) { 785 DWARN(DBG_ALL_LDCS, "ldc_mem_info: invalid memory handle\n"); 786 return (EINVAL); 787 } 788 mhdl = (ldc_mhdl_t *)mhandle; 789 790 if (minfo == NULL) { 791 DWARN(DBG_ALL_LDCS, "ldc_mem_info: invalid args\n"); 792 return (EINVAL); 793 } 794 795 mutex_enter(&mhdl->lock); 796 797 minfo->status = mhdl->status; 798 if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED) { 799 minfo->vaddr = mhdl->memseg->vaddr; 800 minfo->raddr = mhdl->memseg->raddr; 801 minfo->mtype = mhdl->mtype; 802 minfo->perm = mhdl->perm; 803 } 804 mutex_exit(&mhdl->lock); 805 806 return (0); 807 } 808 809 /* 810 * Copy data either from or to the client specified virtual address 811 * space to or from the exported memory associated with the cookies. 812 * The direction argument determines whether the data is read from or 813 * written to exported memory. 814 */ 815 int 816 ldc_mem_copy(ldc_handle_t handle, caddr_t vaddr, uint64_t off, size_t *size, 817 ldc_mem_cookie_t *cookies, uint32_t ccount, uint8_t direction) 818 { 819 ldc_chan_t *ldcp; 820 uint64_t local_voff, local_valign; 821 uint64_t cookie_addr, cookie_size; 822 uint64_t pg_shift, pg_size, pg_size_code; 823 uint64_t export_caddr, export_poff, export_psize, export_size; 824 uint64_t local_ra, local_poff, local_psize; 825 uint64_t copy_size, copied_len = 0, total_bal = 0, idx = 0; 826 pgcnt_t npages; 827 size_t len = *size; 828 int i, rv = 0; 829 830 uint64_t chid; 831 832 if (handle == NULL) { 833 DWARN(DBG_ALL_LDCS, "ldc_mem_copy: invalid channel handle\n"); 834 return (EINVAL); 835 } 836 ldcp = (ldc_chan_t *)handle; 837 chid = ldcp->id; 838 839 /* check to see if channel is UP */ 840 if (ldcp->tstate != TS_UP) { 841 DWARN(chid, "ldc_mem_copy: (0x%llx) channel is not UP\n", 842 chid); 843 return (ECONNRESET); 844 } 845 846 /* Force address and size to be 8-byte aligned */ 847 if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 848 DWARN(chid, 849 "ldc_mem_copy: addr/sz is not 8-byte aligned\n"); 850 return (EINVAL); 851 } 852 853 /* Find the size of the exported memory */ 854 export_size = 0; 855 for (i = 0; i < ccount; i++) 856 export_size += cookies[i].size; 857 858 /* check to see if offset is valid */ 859 if (off > export_size) { 860 DWARN(chid, 861 "ldc_mem_copy: (0x%llx) start offset > export mem size\n", 862 chid); 863 return (EINVAL); 864 } 865 866 /* 867 * Check to see if the export size is smaller than the size we 868 * are requesting to copy - if so flag an error 869 */ 870 if ((export_size - off) < *size) { 871 DWARN(chid, 872 "ldc_mem_copy: (0x%llx) copy size > export mem size\n", 873 chid); 874 return (EINVAL); 875 } 876 877 total_bal = min(export_size, *size); 878 879 /* FUTURE: get the page size, pgsz code, and shift */ 880 pg_size = MMU_PAGESIZE; 881 pg_size_code = page_szc(pg_size); 882 pg_shift = page_get_shift(pg_size_code); 883 884 D1(chid, "ldc_mem_copy: copying data " 885 "(0x%llx) va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 886 chid, vaddr, pg_size, pg_size_code, pg_shift); 887 888 /* aligned VA and its offset */ 889 local_valign = (((uintptr_t)vaddr) & ~(pg_size - 1)); 890 local_voff = ((uintptr_t)vaddr) & (pg_size - 1); 891 892 npages = (len+local_voff)/pg_size; 893 npages = ((len+local_voff)%pg_size == 0) ? npages : npages+1; 894 895 D1(chid, 896 "ldc_mem_copy: (0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n", 897 chid, vaddr, local_valign, local_voff, npages); 898 899 local_ra = va_to_pa((void *)local_valign); 900 local_poff = local_voff; 901 local_psize = min(len, (pg_size - local_voff)); 902 903 len -= local_psize; 904 905 /* 906 * find the first cookie in the list of cookies 907 * if the offset passed in is not zero 908 */ 909 for (idx = 0; idx < ccount; idx++) { 910 cookie_size = cookies[idx].size; 911 if (off < cookie_size) 912 break; 913 off -= cookie_size; 914 } 915 916 cookie_addr = cookies[idx].addr + off; 917 cookie_size = cookies[idx].size - off; 918 919 export_caddr = cookie_addr & ~(pg_size - 1); 920 export_poff = cookie_addr & (pg_size - 1); 921 export_psize = min(cookie_size, (pg_size - export_poff)); 922 923 for (;;) { 924 925 copy_size = min(export_psize, local_psize); 926 927 D1(chid, 928 "ldc_mem_copy:(0x%llx) dir=0x%x, caddr=0x%llx," 929 " loc_ra=0x%llx, exp_poff=0x%llx, loc_poff=0x%llx," 930 " exp_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx," 931 " total_bal=0x%llx\n", 932 chid, direction, export_caddr, local_ra, export_poff, 933 local_poff, export_psize, local_psize, copy_size, 934 total_bal); 935 936 rv = hv_ldc_copy(chid, direction, 937 (export_caddr + export_poff), (local_ra + local_poff), 938 copy_size, &copied_len); 939 940 if (rv != 0) { 941 int error = EIO; 942 uint64_t rx_hd, rx_tl; 943 944 DWARN(chid, 945 "ldc_mem_copy: (0x%llx) err %d during copy\n", 946 (unsigned long long)chid, rv); 947 DWARN(chid, 948 "ldc_mem_copy: (0x%llx) dir=0x%x, caddr=0x%lx, " 949 "loc_ra=0x%lx, exp_poff=0x%lx, loc_poff=0x%lx," 950 " exp_psz=0x%lx, loc_psz=0x%lx, copy_sz=0x%lx," 951 " copied_len=0x%lx, total_bal=0x%lx\n", 952 chid, direction, export_caddr, local_ra, 953 export_poff, local_poff, export_psize, local_psize, 954 copy_size, copied_len, total_bal); 955 956 *size = *size - total_bal; 957 958 /* 959 * check if reason for copy error was due to 960 * a channel reset. we need to grab the lock 961 * just in case we have to do a reset. 962 */ 963 mutex_enter(&ldcp->lock); 964 mutex_enter(&ldcp->tx_lock); 965 966 rv = hv_ldc_rx_get_state(ldcp->id, 967 &rx_hd, &rx_tl, &(ldcp->link_state)); 968 if (ldcp->link_state == LDC_CHANNEL_DOWN || 969 ldcp->link_state == LDC_CHANNEL_RESET) { 970 i_ldc_reset(ldcp, B_FALSE); 971 error = ECONNRESET; 972 } 973 974 mutex_exit(&ldcp->tx_lock); 975 mutex_exit(&ldcp->lock); 976 977 return (error); 978 } 979 980 ASSERT(copied_len <= copy_size); 981 982 D2(chid, "ldc_mem_copy: copied=0x%llx\n", copied_len); 983 export_poff += copied_len; 984 local_poff += copied_len; 985 export_psize -= copied_len; 986 local_psize -= copied_len; 987 cookie_size -= copied_len; 988 989 total_bal -= copied_len; 990 991 if (copy_size != copied_len) 992 continue; 993 994 if (export_psize == 0 && total_bal != 0) { 995 996 if (cookie_size == 0) { 997 idx++; 998 cookie_addr = cookies[idx].addr; 999 cookie_size = cookies[idx].size; 1000 1001 export_caddr = cookie_addr & ~(pg_size - 1); 1002 export_poff = cookie_addr & (pg_size - 1); 1003 export_psize = 1004 min(cookie_size, (pg_size-export_poff)); 1005 } else { 1006 export_caddr += pg_size; 1007 export_poff = 0; 1008 export_psize = min(cookie_size, pg_size); 1009 } 1010 } 1011 1012 if (local_psize == 0 && total_bal != 0) { 1013 local_valign += pg_size; 1014 local_ra = va_to_pa((void *)local_valign); 1015 local_poff = 0; 1016 local_psize = min(pg_size, len); 1017 len -= local_psize; 1018 } 1019 1020 /* check if we are all done */ 1021 if (total_bal == 0) 1022 break; 1023 } 1024 1025 1026 D1(chid, 1027 "ldc_mem_copy: (0x%llx) done copying sz=0x%llx\n", 1028 chid, *size); 1029 1030 return (0); 1031 } 1032 1033 /* 1034 * Copy data either from or to the client specified virtual address 1035 * space to or from HV physical memory. 1036 * 1037 * The direction argument determines whether the data is read from or 1038 * written to HV memory. direction values are LDC_COPY_IN/OUT similar 1039 * to the ldc_mem_copy interface 1040 */ 1041 int 1042 ldc_mem_rdwr_cookie(ldc_handle_t handle, caddr_t vaddr, size_t *size, 1043 caddr_t paddr, uint8_t direction) 1044 { 1045 ldc_chan_t *ldcp; 1046 uint64_t local_voff, local_valign; 1047 uint64_t pg_shift, pg_size, pg_size_code; 1048 uint64_t target_pa, target_poff, target_psize, target_size; 1049 uint64_t local_ra, local_poff, local_psize; 1050 uint64_t copy_size, copied_len = 0; 1051 pgcnt_t npages; 1052 size_t len = *size; 1053 int rv = 0; 1054 1055 if (handle == NULL) { 1056 DWARN(DBG_ALL_LDCS, 1057 "ldc_mem_rdwr_cookie: invalid channel handle\n"); 1058 return (EINVAL); 1059 } 1060 ldcp = (ldc_chan_t *)handle; 1061 1062 mutex_enter(&ldcp->lock); 1063 1064 /* check to see if channel is UP */ 1065 if (ldcp->tstate != TS_UP) { 1066 DWARN(ldcp->id, 1067 "ldc_mem_rdwr_cookie: (0x%llx) channel is not UP\n", 1068 ldcp->id); 1069 mutex_exit(&ldcp->lock); 1070 return (ECONNRESET); 1071 } 1072 1073 /* Force address and size to be 8-byte aligned */ 1074 if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 1075 DWARN(ldcp->id, 1076 "ldc_mem_rdwr_cookie: addr/size is not 8-byte aligned\n"); 1077 mutex_exit(&ldcp->lock); 1078 return (EINVAL); 1079 } 1080 1081 target_size = *size; 1082 1083 /* FUTURE: get the page size, pgsz code, and shift */ 1084 pg_size = MMU_PAGESIZE; 1085 pg_size_code = page_szc(pg_size); 1086 pg_shift = page_get_shift(pg_size_code); 1087 1088 D1(ldcp->id, "ldc_mem_rdwr_cookie: copying data " 1089 "(0x%llx) va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 1090 ldcp->id, vaddr, pg_size, pg_size_code, pg_shift); 1091 1092 /* aligned VA and its offset */ 1093 local_valign = ((uintptr_t)vaddr) & ~(pg_size - 1); 1094 local_voff = ((uintptr_t)vaddr) & (pg_size - 1); 1095 1096 npages = (len + local_voff) / pg_size; 1097 npages = ((len + local_voff) % pg_size == 0) ? npages : npages+1; 1098 1099 D1(ldcp->id, "ldc_mem_rdwr_cookie: (0x%llx) v=0x%llx, " 1100 "val=0x%llx,off=0x%x,pgs=0x%x\n", 1101 ldcp->id, vaddr, local_valign, local_voff, npages); 1102 1103 local_ra = va_to_pa((void *)local_valign); 1104 local_poff = local_voff; 1105 local_psize = min(len, (pg_size - local_voff)); 1106 1107 len -= local_psize; 1108 1109 target_pa = ((uintptr_t)paddr) & ~(pg_size - 1); 1110 target_poff = ((uintptr_t)paddr) & (pg_size - 1); 1111 target_psize = pg_size - target_poff; 1112 1113 for (;;) { 1114 1115 copy_size = min(target_psize, local_psize); 1116 1117 D1(ldcp->id, 1118 "ldc_mem_rdwr_cookie: (0x%llx) dir=0x%x, tar_pa=0x%llx," 1119 " loc_ra=0x%llx, tar_poff=0x%llx, loc_poff=0x%llx," 1120 " tar_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx," 1121 " total_bal=0x%llx\n", 1122 ldcp->id, direction, target_pa, local_ra, target_poff, 1123 local_poff, target_psize, local_psize, copy_size, 1124 target_size); 1125 1126 rv = hv_ldc_copy(ldcp->id, direction, 1127 (target_pa + target_poff), (local_ra + local_poff), 1128 copy_size, &copied_len); 1129 1130 if (rv != 0) { 1131 DWARN(DBG_ALL_LDCS, 1132 "ldc_mem_rdwr_cookie: (0x%lx) err %d during copy\n", 1133 ldcp->id, rv); 1134 DWARN(DBG_ALL_LDCS, 1135 "ldc_mem_rdwr_cookie: (0x%llx) dir=%lld, " 1136 "tar_pa=0x%llx, loc_ra=0x%llx, tar_poff=0x%llx, " 1137 "loc_poff=0x%llx, tar_psz=0x%llx, loc_psz=0x%llx, " 1138 "copy_sz=0x%llx, total_bal=0x%llx\n", 1139 ldcp->id, direction, target_pa, local_ra, 1140 target_poff, local_poff, target_psize, local_psize, 1141 copy_size, target_size); 1142 1143 *size = *size - target_size; 1144 mutex_exit(&ldcp->lock); 1145 return (i_ldc_h2v_error(rv)); 1146 } 1147 1148 D2(ldcp->id, "ldc_mem_rdwr_cookie: copied=0x%llx\n", 1149 copied_len); 1150 target_poff += copied_len; 1151 local_poff += copied_len; 1152 target_psize -= copied_len; 1153 local_psize -= copied_len; 1154 1155 target_size -= copied_len; 1156 1157 if (copy_size != copied_len) 1158 continue; 1159 1160 if (target_psize == 0 && target_size != 0) { 1161 target_pa += pg_size; 1162 target_poff = 0; 1163 target_psize = min(pg_size, target_size); 1164 } 1165 1166 if (local_psize == 0 && target_size != 0) { 1167 local_valign += pg_size; 1168 local_ra = va_to_pa((void *)local_valign); 1169 local_poff = 0; 1170 local_psize = min(pg_size, len); 1171 len -= local_psize; 1172 } 1173 1174 /* check if we are all done */ 1175 if (target_size == 0) 1176 break; 1177 } 1178 1179 mutex_exit(&ldcp->lock); 1180 1181 D1(ldcp->id, "ldc_mem_rdwr_cookie: (0x%llx) done copying sz=0x%llx\n", 1182 ldcp->id, *size); 1183 1184 return (0); 1185 } 1186 1187 /* 1188 * Map an exported memory segment into the local address space. If the 1189 * memory range was exported for direct map access, a HV call is made 1190 * to allocate a RA range. If the map is done via a shadow copy, local 1191 * shadow memory is allocated and the base VA is returned in 'vaddr'. If 1192 * the mapping is a direct map then the RA is returned in 'raddr'. 1193 */ 1194 int 1195 ldc_mem_map(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie, uint32_t ccount, 1196 uint8_t mtype, uint8_t perm, caddr_t *vaddr, caddr_t *raddr) 1197 { 1198 /* 1199 * Check if direct map over shared memory is enabled, if not change 1200 * the mapping type to SHADOW_MAP. 1201 */ 1202 if (ldc_shmem_enabled == 0) 1203 mtype = LDC_SHADOW_MAP; 1204 1205 return (i_ldc_mem_map(mhandle, cookie, ccount, mtype, perm, 1206 vaddr, raddr)); 1207 } 1208 1209 static int 1210 i_ldc_mem_map(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie, 1211 uint32_t ccount, uint8_t mtype, uint8_t perm, caddr_t *vaddr, 1212 caddr_t *raddr) 1213 { 1214 1215 int i, j, idx, rv, retries; 1216 ldc_chan_t *ldcp; 1217 ldc_mhdl_t *mhdl; 1218 ldc_memseg_t *memseg; 1219 caddr_t tmpaddr; 1220 uint64_t map_perm = perm; 1221 uint64_t pg_size, pg_shift, pg_size_code, pg_mask; 1222 uint64_t exp_size = 0, base_off, map_size, npages; 1223 uint64_t cookie_addr, cookie_off, cookie_size; 1224 tte_t ldc_tte; 1225 1226 if (mhandle == NULL) { 1227 DWARN(DBG_ALL_LDCS, "ldc_mem_map: invalid memory handle\n"); 1228 return (EINVAL); 1229 } 1230 mhdl = (ldc_mhdl_t *)mhandle; 1231 1232 mutex_enter(&mhdl->lock); 1233 1234 if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED || 1235 mhdl->memseg != NULL) { 1236 DWARN(DBG_ALL_LDCS, 1237 "ldc_mem_map: (0x%llx) handle bound/mapped\n", mhandle); 1238 mutex_exit(&mhdl->lock); 1239 return (EINVAL); 1240 } 1241 1242 ldcp = mhdl->ldcp; 1243 1244 mutex_enter(&ldcp->lock); 1245 1246 if (ldcp->tstate != TS_UP) { 1247 DWARN(ldcp->id, 1248 "ldc_mem_dring_map: (0x%llx) channel is not UP\n", 1249 ldcp->id); 1250 mutex_exit(&ldcp->lock); 1251 mutex_exit(&mhdl->lock); 1252 return (ECONNRESET); 1253 } 1254 1255 if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP|LDC_IO_MAP)) == 0) { 1256 DWARN(ldcp->id, "ldc_mem_map: invalid map type\n"); 1257 mutex_exit(&ldcp->lock); 1258 mutex_exit(&mhdl->lock); 1259 return (EINVAL); 1260 } 1261 1262 D1(ldcp->id, "ldc_mem_map: (0x%llx) cookie = 0x%llx,0x%llx\n", 1263 ldcp->id, cookie->addr, cookie->size); 1264 1265 /* FUTURE: get the page size, pgsz code, and shift */ 1266 pg_size = MMU_PAGESIZE; 1267 pg_size_code = page_szc(pg_size); 1268 pg_shift = page_get_shift(pg_size_code); 1269 pg_mask = ~(pg_size - 1); 1270 1271 /* calculate the number of pages in the exported cookie */ 1272 base_off = cookie[0].addr & (pg_size - 1); 1273 for (idx = 0; idx < ccount; idx++) 1274 exp_size += cookie[idx].size; 1275 map_size = P2ROUNDUP((exp_size + base_off), pg_size); 1276 npages = (map_size >> pg_shift); 1277 1278 /* Allocate memseg structure */ 1279 memseg = mhdl->memseg = 1280 kmem_cache_alloc(ldcssp->memseg_cache, KM_SLEEP); 1281 1282 /* Allocate memory to store all pages and cookies */ 1283 memseg->pages = kmem_zalloc((sizeof (ldc_page_t) * npages), KM_SLEEP); 1284 memseg->cookies = 1285 kmem_zalloc((sizeof (ldc_mem_cookie_t) * ccount), KM_SLEEP); 1286 1287 D2(ldcp->id, "ldc_mem_map: (0x%llx) exp_size=0x%llx, map_size=0x%llx," 1288 "pages=0x%llx\n", ldcp->id, exp_size, map_size, npages); 1289 1290 /* 1291 * Check to see if the client is requesting direct or shadow map 1292 * If direct map is requested, try to map remote memory first, 1293 * and if that fails, revert to shadow map 1294 */ 1295 if (mtype == LDC_DIRECT_MAP) { 1296 1297 /* Allocate kernel virtual space for mapping */ 1298 memseg->vaddr = vmem_xalloc(heap_arena, map_size, 1299 pg_size, 0, 0, NULL, NULL, VM_NOSLEEP); 1300 if (memseg->vaddr == NULL) { 1301 DWARN(DBG_ALL_LDCS, 1302 "ldc_mem_map: (0x%lx) memory map failed\n", 1303 ldcp->id); 1304 kmem_free(memseg->cookies, 1305 (sizeof (ldc_mem_cookie_t) * ccount)); 1306 kmem_free(memseg->pages, 1307 (sizeof (ldc_page_t) * npages)); 1308 kmem_cache_free(ldcssp->memseg_cache, memseg); 1309 1310 mutex_exit(&ldcp->lock); 1311 mutex_exit(&mhdl->lock); 1312 return (ENOMEM); 1313 } 1314 1315 /* Unload previous mapping */ 1316 hat_unload(kas.a_hat, memseg->vaddr, map_size, 1317 HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK); 1318 1319 /* for each cookie passed in - map into address space */ 1320 idx = 0; 1321 cookie_size = 0; 1322 tmpaddr = memseg->vaddr; 1323 1324 for (i = 0; i < npages; i++) { 1325 1326 if (cookie_size == 0) { 1327 ASSERT(idx < ccount); 1328 cookie_addr = cookie[idx].addr & pg_mask; 1329 cookie_off = cookie[idx].addr & (pg_size - 1); 1330 cookie_size = 1331 P2ROUNDUP((cookie_off + cookie[idx].size), 1332 pg_size); 1333 idx++; 1334 } 1335 1336 D1(ldcp->id, "ldc_mem_map: (0x%llx) mapping " 1337 "cookie 0x%llx, bal=0x%llx\n", ldcp->id, 1338 cookie_addr, cookie_size); 1339 1340 /* map the cookie into address space */ 1341 for (retries = 0; retries < ldc_max_retries; 1342 retries++) { 1343 1344 rv = hv_ldc_mapin(ldcp->id, cookie_addr, 1345 &memseg->pages[i].raddr, &map_perm); 1346 if (rv != H_EWOULDBLOCK && rv != H_ETOOMANY) 1347 break; 1348 1349 drv_usecwait(ldc_delay); 1350 } 1351 1352 if (rv || memseg->pages[i].raddr == 0) { 1353 DWARN(ldcp->id, 1354 "ldc_mem_map: (0x%llx) hv mapin err %d\n", 1355 ldcp->id, rv); 1356 1357 /* remove previous mapins */ 1358 hat_unload(kas.a_hat, memseg->vaddr, map_size, 1359 HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK); 1360 for (j = 0; j < i; j++) { 1361 rv = hv_ldc_unmap( 1362 memseg->pages[j].raddr); 1363 if (rv) { 1364 DWARN(ldcp->id, 1365 "ldc_mem_map: (0x%llx) " 1366 "cannot unmap ra=0x%llx\n", 1367 ldcp->id, 1368 memseg->pages[j].raddr); 1369 } 1370 } 1371 1372 /* free kernel virtual space */ 1373 vmem_free(heap_arena, (void *)memseg->vaddr, 1374 map_size); 1375 1376 /* direct map failed - revert to shadow map */ 1377 mtype = LDC_SHADOW_MAP; 1378 break; 1379 1380 } else { 1381 1382 D1(ldcp->id, 1383 "ldc_mem_map: (0x%llx) vtop map 0x%llx -> " 1384 "0x%llx, cookie=0x%llx, perm=0x%llx\n", 1385 ldcp->id, tmpaddr, memseg->pages[i].raddr, 1386 cookie_addr, perm); 1387 1388 /* 1389 * NOTE: Calling hat_devload directly, causes it 1390 * to look for page_t using the pfn. Since this 1391 * addr is greater than the memlist, it treates 1392 * it as non-memory 1393 */ 1394 sfmmu_memtte(&ldc_tte, 1395 (pfn_t)(memseg->pages[i].raddr >> pg_shift), 1396 PROT_READ | PROT_WRITE | HAT_NOSYNC, TTE8K); 1397 1398 D1(ldcp->id, 1399 "ldc_mem_map: (0x%llx) ra 0x%llx -> " 1400 "tte 0x%llx\n", ldcp->id, 1401 memseg->pages[i].raddr, ldc_tte); 1402 1403 sfmmu_tteload(kas.a_hat, &ldc_tte, tmpaddr, 1404 NULL, HAT_LOAD_LOCK); 1405 1406 cookie_size -= pg_size; 1407 cookie_addr += pg_size; 1408 tmpaddr += pg_size; 1409 } 1410 } 1411 } 1412 1413 if (mtype == LDC_SHADOW_MAP) { 1414 if (*vaddr == NULL) { 1415 memseg->vaddr = kmem_zalloc(exp_size, KM_SLEEP); 1416 mhdl->myshadow = B_TRUE; 1417 1418 D1(ldcp->id, "ldc_mem_map: (0x%llx) allocated " 1419 "shadow page va=0x%llx\n", ldcp->id, memseg->vaddr); 1420 } else { 1421 /* 1422 * Use client supplied memory for memseg->vaddr 1423 * WARNING: assuming that client mem is >= exp_size 1424 */ 1425 memseg->vaddr = *vaddr; 1426 } 1427 1428 /* Save all page and cookie information */ 1429 for (i = 0, tmpaddr = memseg->vaddr; i < npages; i++) { 1430 memseg->pages[i].raddr = va_to_pa(tmpaddr); 1431 tmpaddr += pg_size; 1432 } 1433 1434 } 1435 1436 /* save all cookies */ 1437 bcopy(cookie, memseg->cookies, ccount * sizeof (ldc_mem_cookie_t)); 1438 1439 /* update memseg_t */ 1440 memseg->raddr = memseg->pages[0].raddr; 1441 memseg->size = (mtype == LDC_SHADOW_MAP) ? exp_size : map_size; 1442 memseg->npages = npages; 1443 memseg->ncookies = ccount; 1444 memseg->next_cookie = 0; 1445 1446 /* memory handle = mapped */ 1447 mhdl->mtype = mtype; 1448 mhdl->perm = perm; 1449 mhdl->status = LDC_MAPPED; 1450 1451 D1(ldcp->id, "ldc_mem_map: (0x%llx) mapped 0x%llx, ra=0x%llx, " 1452 "va=0x%llx, pgs=0x%llx cookies=0x%llx\n", 1453 ldcp->id, mhdl, memseg->raddr, memseg->vaddr, 1454 memseg->npages, memseg->ncookies); 1455 1456 if (mtype == LDC_SHADOW_MAP) 1457 base_off = 0; 1458 if (raddr) 1459 *raddr = (caddr_t)(memseg->raddr | base_off); 1460 if (vaddr) 1461 *vaddr = (caddr_t)((uintptr_t)memseg->vaddr | base_off); 1462 1463 mutex_exit(&ldcp->lock); 1464 mutex_exit(&mhdl->lock); 1465 return (0); 1466 } 1467 1468 /* 1469 * Unmap a memory segment. Free shadow memory (if any). 1470 */ 1471 int 1472 ldc_mem_unmap(ldc_mem_handle_t mhandle) 1473 { 1474 int i, rv; 1475 ldc_mhdl_t *mhdl = (ldc_mhdl_t *)mhandle; 1476 ldc_chan_t *ldcp; 1477 ldc_memseg_t *memseg; 1478 1479 if (mhdl == 0 || mhdl->status != LDC_MAPPED) { 1480 DWARN(DBG_ALL_LDCS, 1481 "ldc_mem_unmap: (0x%llx) handle is not mapped\n", 1482 mhandle); 1483 return (EINVAL); 1484 } 1485 1486 mutex_enter(&mhdl->lock); 1487 1488 ldcp = mhdl->ldcp; 1489 memseg = mhdl->memseg; 1490 1491 D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapping handle 0x%llx\n", 1492 ldcp->id, mhdl); 1493 1494 /* if we allocated shadow memory - free it */ 1495 if (mhdl->mtype == LDC_SHADOW_MAP && mhdl->myshadow) { 1496 kmem_free(memseg->vaddr, memseg->size); 1497 } else if (mhdl->mtype == LDC_DIRECT_MAP) { 1498 1499 /* unmap in the case of DIRECT_MAP */ 1500 hat_unload(kas.a_hat, memseg->vaddr, memseg->size, 1501 HAT_UNLOAD_UNLOCK); 1502 1503 for (i = 0; i < memseg->npages; i++) { 1504 rv = hv_ldc_unmap(memseg->pages[i].raddr); 1505 if (rv) { 1506 DWARN(DBG_ALL_LDCS, 1507 "ldc_mem_map: (0x%lx) hv unmap err %d\n", 1508 ldcp->id, rv); 1509 } 1510 } 1511 1512 vmem_free(heap_arena, (void *)memseg->vaddr, memseg->size); 1513 } 1514 1515 /* free the allocated memseg and page structures */ 1516 kmem_free(memseg->pages, (sizeof (ldc_page_t) * memseg->npages)); 1517 kmem_free(memseg->cookies, 1518 (sizeof (ldc_mem_cookie_t) * memseg->ncookies)); 1519 kmem_cache_free(ldcssp->memseg_cache, memseg); 1520 1521 /* uninitialize the memory handle */ 1522 mhdl->memseg = NULL; 1523 mhdl->status = LDC_UNBOUND; 1524 1525 D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapped handle 0x%llx\n", 1526 ldcp->id, mhdl); 1527 1528 mutex_exit(&mhdl->lock); 1529 return (0); 1530 } 1531 1532 /* 1533 * Internal entry point for LDC mapped memory entry consistency 1534 * semantics. Acquire copies the contents of the remote memory 1535 * into the local shadow copy. The release operation copies the local 1536 * contents into the remote memory. The offset and size specify the 1537 * bounds for the memory range being synchronized. 1538 */ 1539 static int 1540 i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle, uint8_t direction, 1541 uint64_t offset, size_t size) 1542 { 1543 int err; 1544 ldc_mhdl_t *mhdl; 1545 ldc_chan_t *ldcp; 1546 ldc_memseg_t *memseg; 1547 caddr_t local_vaddr; 1548 size_t copy_size; 1549 1550 if (mhandle == NULL) { 1551 DWARN(DBG_ALL_LDCS, 1552 "i_ldc_mem_acquire_release: invalid memory handle\n"); 1553 return (EINVAL); 1554 } 1555 mhdl = (ldc_mhdl_t *)mhandle; 1556 1557 mutex_enter(&mhdl->lock); 1558 1559 if (mhdl->status != LDC_MAPPED || mhdl->ldcp == NULL) { 1560 DWARN(DBG_ALL_LDCS, 1561 "i_ldc_mem_acquire_release: not mapped memory\n"); 1562 mutex_exit(&mhdl->lock); 1563 return (EINVAL); 1564 } 1565 1566 /* do nothing for direct map */ 1567 if (mhdl->mtype == LDC_DIRECT_MAP) { 1568 mutex_exit(&mhdl->lock); 1569 return (0); 1570 } 1571 1572 /* do nothing if COPY_IN+MEM_W and COPY_OUT+MEM_R */ 1573 if ((direction == LDC_COPY_IN && (mhdl->perm & LDC_MEM_R) == 0) || 1574 (direction == LDC_COPY_OUT && (mhdl->perm & LDC_MEM_W) == 0)) { 1575 mutex_exit(&mhdl->lock); 1576 return (0); 1577 } 1578 1579 if (offset >= mhdl->memseg->size || 1580 (offset + size) > mhdl->memseg->size) { 1581 DWARN(DBG_ALL_LDCS, 1582 "i_ldc_mem_acquire_release: memory out of range\n"); 1583 mutex_exit(&mhdl->lock); 1584 return (EINVAL); 1585 } 1586 1587 /* get the channel handle and memory segment */ 1588 ldcp = mhdl->ldcp; 1589 memseg = mhdl->memseg; 1590 1591 if (mhdl->mtype == LDC_SHADOW_MAP) { 1592 1593 local_vaddr = memseg->vaddr + offset; 1594 copy_size = size; 1595 1596 /* copy to/from remote from/to local memory */ 1597 err = ldc_mem_copy((ldc_handle_t)ldcp, local_vaddr, offset, 1598 ©_size, memseg->cookies, memseg->ncookies, 1599 direction); 1600 if (err || copy_size != size) { 1601 DWARN(ldcp->id, 1602 "i_ldc_mem_acquire_release: copy failed\n"); 1603 mutex_exit(&mhdl->lock); 1604 return (err); 1605 } 1606 } 1607 1608 mutex_exit(&mhdl->lock); 1609 1610 return (0); 1611 } 1612 1613 /* 1614 * Ensure that the contents in the remote memory seg are consistent 1615 * with the contents if of local segment 1616 */ 1617 int 1618 ldc_mem_acquire(ldc_mem_handle_t mhandle, uint64_t offset, uint64_t size) 1619 { 1620 return (i_ldc_mem_acquire_release(mhandle, LDC_COPY_IN, offset, size)); 1621 } 1622 1623 1624 /* 1625 * Ensure that the contents in the local memory seg are consistent 1626 * with the contents if of remote segment 1627 */ 1628 int 1629 ldc_mem_release(ldc_mem_handle_t mhandle, uint64_t offset, uint64_t size) 1630 { 1631 return (i_ldc_mem_acquire_release(mhandle, LDC_COPY_OUT, offset, size)); 1632 } 1633 1634 /* 1635 * Allocate a descriptor ring. The size of each each descriptor 1636 * must be 8-byte aligned and the entire ring should be a multiple 1637 * of MMU_PAGESIZE. 1638 */ 1639 int 1640 ldc_mem_dring_create(uint32_t len, uint32_t dsize, ldc_dring_handle_t *dhandle) 1641 { 1642 ldc_dring_t *dringp; 1643 size_t size = (dsize * len); 1644 1645 D1(DBG_ALL_LDCS, "ldc_mem_dring_create: len=0x%x, size=0x%x\n", 1646 len, dsize); 1647 1648 if (dhandle == NULL) { 1649 DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid dhandle\n"); 1650 return (EINVAL); 1651 } 1652 1653 if (len == 0) { 1654 DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid length\n"); 1655 return (EINVAL); 1656 } 1657 1658 /* descriptor size should be 8-byte aligned */ 1659 if (dsize == 0 || (dsize & 0x7)) { 1660 DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid size\n"); 1661 return (EINVAL); 1662 } 1663 1664 *dhandle = 0; 1665 1666 /* Allocate a desc ring structure */ 1667 dringp = kmem_zalloc(sizeof (ldc_dring_t), KM_SLEEP); 1668 1669 /* Initialize dring */ 1670 dringp->length = len; 1671 dringp->dsize = dsize; 1672 1673 /* round off to multiple of pagesize */ 1674 dringp->size = (size & MMU_PAGEMASK); 1675 if (size & MMU_PAGEOFFSET) 1676 dringp->size += MMU_PAGESIZE; 1677 1678 dringp->status = LDC_UNBOUND; 1679 1680 /* allocate descriptor ring memory */ 1681 dringp->base = kmem_zalloc(dringp->size, KM_SLEEP); 1682 1683 /* initialize the desc ring lock */ 1684 mutex_init(&dringp->lock, NULL, MUTEX_DRIVER, NULL); 1685 1686 /* Add descriptor ring to the head of global list */ 1687 mutex_enter(&ldcssp->lock); 1688 dringp->next = ldcssp->dring_list; 1689 ldcssp->dring_list = dringp; 1690 mutex_exit(&ldcssp->lock); 1691 1692 *dhandle = (ldc_dring_handle_t)dringp; 1693 1694 D1(DBG_ALL_LDCS, "ldc_mem_dring_create: dring allocated\n"); 1695 1696 return (0); 1697 } 1698 1699 1700 /* 1701 * Destroy a descriptor ring. 1702 */ 1703 int 1704 ldc_mem_dring_destroy(ldc_dring_handle_t dhandle) 1705 { 1706 ldc_dring_t *dringp; 1707 ldc_dring_t *tmp_dringp; 1708 1709 D1(DBG_ALL_LDCS, "ldc_mem_dring_destroy: entered\n"); 1710 1711 if (dhandle == NULL) { 1712 DWARN(DBG_ALL_LDCS, 1713 "ldc_mem_dring_destroy: invalid desc ring handle\n"); 1714 return (EINVAL); 1715 } 1716 dringp = (ldc_dring_t *)dhandle; 1717 1718 if (dringp->status == LDC_BOUND) { 1719 DWARN(DBG_ALL_LDCS, 1720 "ldc_mem_dring_destroy: desc ring is bound\n"); 1721 return (EACCES); 1722 } 1723 1724 mutex_enter(&dringp->lock); 1725 mutex_enter(&ldcssp->lock); 1726 1727 /* remove from linked list - if not bound */ 1728 tmp_dringp = ldcssp->dring_list; 1729 if (tmp_dringp == dringp) { 1730 ldcssp->dring_list = dringp->next; 1731 dringp->next = NULL; 1732 1733 } else { 1734 while (tmp_dringp != NULL) { 1735 if (tmp_dringp->next == dringp) { 1736 tmp_dringp->next = dringp->next; 1737 dringp->next = NULL; 1738 break; 1739 } 1740 tmp_dringp = tmp_dringp->next; 1741 } 1742 if (tmp_dringp == NULL) { 1743 DWARN(DBG_ALL_LDCS, 1744 "ldc_mem_dring_destroy: invalid descriptor\n"); 1745 mutex_exit(&ldcssp->lock); 1746 mutex_exit(&dringp->lock); 1747 return (EINVAL); 1748 } 1749 } 1750 1751 mutex_exit(&ldcssp->lock); 1752 1753 /* free the descriptor ring */ 1754 kmem_free(dringp->base, dringp->size); 1755 1756 mutex_exit(&dringp->lock); 1757 1758 /* destroy dring lock */ 1759 mutex_destroy(&dringp->lock); 1760 1761 /* free desc ring object */ 1762 kmem_free(dringp, sizeof (ldc_dring_t)); 1763 1764 return (0); 1765 } 1766 1767 /* 1768 * Bind a previously allocated dring to a channel. The channel should 1769 * be OPEN in order to bind the ring to the channel. Returns back a 1770 * descriptor ring cookie. The descriptor ring is exported for remote 1771 * access by the client at the other end of the channel. An entry for 1772 * dring pages is stored in map table (via call to ldc_mem_bind_handle). 1773 */ 1774 int 1775 ldc_mem_dring_bind(ldc_handle_t handle, ldc_dring_handle_t dhandle, 1776 uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount) 1777 { 1778 int err; 1779 ldc_chan_t *ldcp; 1780 ldc_dring_t *dringp; 1781 ldc_mem_handle_t mhandle; 1782 1783 /* check to see if channel is initalized */ 1784 if (handle == NULL) { 1785 DWARN(DBG_ALL_LDCS, 1786 "ldc_mem_dring_bind: invalid channel handle\n"); 1787 return (EINVAL); 1788 } 1789 ldcp = (ldc_chan_t *)handle; 1790 1791 if (dhandle == NULL) { 1792 DWARN(DBG_ALL_LDCS, 1793 "ldc_mem_dring_bind: invalid desc ring handle\n"); 1794 return (EINVAL); 1795 } 1796 dringp = (ldc_dring_t *)dhandle; 1797 1798 if (cookie == NULL) { 1799 DWARN(ldcp->id, 1800 "ldc_mem_dring_bind: invalid cookie arg\n"); 1801 return (EINVAL); 1802 } 1803 1804 /* ensure the mtype is valid */ 1805 if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP)) == 0) { 1806 DWARN(ldcp->id, "ldc_mem_dring_bind: invalid map type\n"); 1807 return (EINVAL); 1808 } 1809 1810 /* no need to bind as direct map if it's not HV supported or enabled */ 1811 if (!ldc_dring_shmem_hv_ok || !ldc_dring_shmem_enabled) { 1812 mtype = LDC_SHADOW_MAP; 1813 } 1814 1815 mutex_enter(&dringp->lock); 1816 1817 if (dringp->status == LDC_BOUND) { 1818 DWARN(DBG_ALL_LDCS, 1819 "ldc_mem_dring_bind: (0x%llx) descriptor ring is bound\n", 1820 ldcp->id); 1821 mutex_exit(&dringp->lock); 1822 return (EINVAL); 1823 } 1824 1825 if ((perm & LDC_MEM_RW) == 0) { 1826 DWARN(DBG_ALL_LDCS, 1827 "ldc_mem_dring_bind: invalid permissions\n"); 1828 mutex_exit(&dringp->lock); 1829 return (EINVAL); 1830 } 1831 1832 if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP|LDC_IO_MAP)) == 0) { 1833 DWARN(DBG_ALL_LDCS, "ldc_mem_dring_bind: invalid type\n"); 1834 mutex_exit(&dringp->lock); 1835 return (EINVAL); 1836 } 1837 1838 dringp->ldcp = ldcp; 1839 1840 /* create an memory handle */ 1841 err = ldc_mem_alloc_handle(handle, &mhandle); 1842 if (err || mhandle == NULL) { 1843 DWARN(DBG_ALL_LDCS, 1844 "ldc_mem_dring_bind: (0x%llx) error allocating mhandle\n", 1845 ldcp->id); 1846 mutex_exit(&dringp->lock); 1847 return (err); 1848 } 1849 dringp->mhdl = mhandle; 1850 1851 /* bind the descriptor ring to channel */ 1852 err = i_ldc_mem_bind_handle(mhandle, dringp->base, dringp->size, 1853 mtype, perm, cookie, ccount); 1854 if (err) { 1855 DWARN(ldcp->id, 1856 "ldc_mem_dring_bind: (0x%llx) error binding mhandle\n", 1857 ldcp->id); 1858 mutex_exit(&dringp->lock); 1859 return (err); 1860 } 1861 1862 /* 1863 * For now return error if we get more than one cookie 1864 * FUTURE: Return multiple cookies .. 1865 */ 1866 if (*ccount > 1) { 1867 (void) ldc_mem_unbind_handle(mhandle); 1868 (void) ldc_mem_free_handle(mhandle); 1869 1870 dringp->ldcp = NULL; 1871 dringp->mhdl = NULL; 1872 *ccount = 0; 1873 1874 mutex_exit(&dringp->lock); 1875 return (EAGAIN); 1876 } 1877 1878 /* Add descriptor ring to channel's exported dring list */ 1879 mutex_enter(&ldcp->exp_dlist_lock); 1880 dringp->ch_next = ldcp->exp_dring_list; 1881 ldcp->exp_dring_list = dringp; 1882 mutex_exit(&ldcp->exp_dlist_lock); 1883 1884 dringp->status = LDC_BOUND; 1885 1886 mutex_exit(&dringp->lock); 1887 1888 return (0); 1889 } 1890 1891 /* 1892 * Return the next cookie associated with the specified dring handle 1893 */ 1894 int 1895 ldc_mem_dring_nextcookie(ldc_dring_handle_t dhandle, ldc_mem_cookie_t *cookie) 1896 { 1897 int rv = 0; 1898 ldc_dring_t *dringp; 1899 ldc_chan_t *ldcp; 1900 1901 if (dhandle == NULL) { 1902 DWARN(DBG_ALL_LDCS, 1903 "ldc_mem_dring_nextcookie: invalid desc ring handle\n"); 1904 return (EINVAL); 1905 } 1906 dringp = (ldc_dring_t *)dhandle; 1907 mutex_enter(&dringp->lock); 1908 1909 if (dringp->status != LDC_BOUND) { 1910 DWARN(DBG_ALL_LDCS, 1911 "ldc_mem_dring_nextcookie: descriptor ring 0x%llx " 1912 "is not bound\n", dringp); 1913 mutex_exit(&dringp->lock); 1914 return (EINVAL); 1915 } 1916 1917 ldcp = dringp->ldcp; 1918 1919 if (cookie == NULL) { 1920 DWARN(ldcp->id, 1921 "ldc_mem_dring_nextcookie:(0x%llx) invalid cookie arg\n", 1922 ldcp->id); 1923 mutex_exit(&dringp->lock); 1924 return (EINVAL); 1925 } 1926 1927 rv = ldc_mem_nextcookie((ldc_mem_handle_t)dringp->mhdl, cookie); 1928 mutex_exit(&dringp->lock); 1929 1930 return (rv); 1931 } 1932 1933 /* 1934 * Unbind a previously bound dring from a channel. 1935 */ 1936 int 1937 ldc_mem_dring_unbind(ldc_dring_handle_t dhandle) 1938 { 1939 ldc_dring_t *dringp; 1940 ldc_dring_t *tmp_dringp; 1941 ldc_chan_t *ldcp; 1942 1943 if (dhandle == NULL) { 1944 DWARN(DBG_ALL_LDCS, 1945 "ldc_mem_dring_unbind: invalid desc ring handle\n"); 1946 return (EINVAL); 1947 } 1948 dringp = (ldc_dring_t *)dhandle; 1949 1950 mutex_enter(&dringp->lock); 1951 1952 if (dringp->status == LDC_UNBOUND) { 1953 DWARN(DBG_ALL_LDCS, 1954 "ldc_mem_dring_bind: descriptor ring 0x%llx is unbound\n", 1955 dringp); 1956 mutex_exit(&dringp->lock); 1957 return (EINVAL); 1958 } 1959 ldcp = dringp->ldcp; 1960 1961 mutex_enter(&ldcp->exp_dlist_lock); 1962 1963 tmp_dringp = ldcp->exp_dring_list; 1964 if (tmp_dringp == dringp) { 1965 ldcp->exp_dring_list = dringp->ch_next; 1966 dringp->ch_next = NULL; 1967 1968 } else { 1969 while (tmp_dringp != NULL) { 1970 if (tmp_dringp->ch_next == dringp) { 1971 tmp_dringp->ch_next = dringp->ch_next; 1972 dringp->ch_next = NULL; 1973 break; 1974 } 1975 tmp_dringp = tmp_dringp->ch_next; 1976 } 1977 if (tmp_dringp == NULL) { 1978 DWARN(DBG_ALL_LDCS, 1979 "ldc_mem_dring_unbind: invalid descriptor\n"); 1980 mutex_exit(&ldcp->exp_dlist_lock); 1981 mutex_exit(&dringp->lock); 1982 return (EINVAL); 1983 } 1984 } 1985 1986 mutex_exit(&ldcp->exp_dlist_lock); 1987 1988 (void) ldc_mem_unbind_handle((ldc_mem_handle_t)dringp->mhdl); 1989 (void) ldc_mem_free_handle((ldc_mem_handle_t)dringp->mhdl); 1990 1991 dringp->ldcp = NULL; 1992 dringp->mhdl = NULL; 1993 dringp->status = LDC_UNBOUND; 1994 1995 mutex_exit(&dringp->lock); 1996 1997 return (0); 1998 } 1999 2000 #ifdef DEBUG 2001 void 2002 i_ldc_mem_inject_dring_clear(ldc_chan_t *ldcp) 2003 { 2004 ldc_dring_t *dp; 2005 ldc_mhdl_t *mhdl; 2006 ldc_mtbl_t *mtbl; 2007 ldc_memseg_t *memseg; 2008 uint64_t cookie_addr; 2009 uint64_t pg_shift, pg_size_code; 2010 int i, rv, retries; 2011 2012 /* has a map table been allocated? */ 2013 if ((mtbl = ldcp->mtbl) == NULL) 2014 return; 2015 2016 /* lock the memory table - exclusive access to channel */ 2017 mutex_enter(&mtbl->lock); 2018 2019 /* lock the exported dring list */ 2020 mutex_enter(&ldcp->exp_dlist_lock); 2021 2022 for (dp = ldcp->exp_dring_list; dp != NULL; dp = dp->ch_next) { 2023 if ((mhdl = (ldc_mhdl_t *)dp->mhdl) == NULL) 2024 continue; 2025 2026 if ((memseg = mhdl->memseg) == NULL) 2027 continue; 2028 2029 /* undo the pages exported */ 2030 for (i = 0; i < memseg->npages; i++) { 2031 2032 /* clear the entry from the table */ 2033 memseg->pages[i].mte->entry.ll = 0; 2034 2035 pg_size_code = page_szc(MMU_PAGESIZE); 2036 pg_shift = page_get_shift(pg_size_code); 2037 cookie_addr = IDX2COOKIE(memseg->pages[i].index, 2038 pg_size_code, pg_shift); 2039 2040 retries = 0; 2041 do { 2042 rv = hv_ldc_revoke(ldcp->id, cookie_addr, 2043 memseg->pages[i].mte->cookie); 2044 2045 if (rv != H_EWOULDBLOCK) 2046 break; 2047 2048 drv_usecwait(ldc_delay); 2049 2050 } while (retries++ < ldc_max_retries); 2051 2052 if (rv != 0) { 2053 DWARN(ldcp->id, 2054 "i_ldc_mem_inject_dring_clear(): " 2055 "hv_ldc_revoke failed: " 2056 "channel: 0x%lx, cookie addr: 0x%p," 2057 "cookie: 0x%lx, rv: %d", 2058 ldcp->id, cookie_addr, 2059 memseg->pages[i].mte->cookie, rv); 2060 } 2061 2062 mtbl->num_avail++; 2063 } 2064 } 2065 2066 mutex_exit(&ldcp->exp_dlist_lock); 2067 mutex_exit(&mtbl->lock); 2068 } 2069 #endif 2070 2071 /* 2072 * Get information about the dring. The base address of the descriptor 2073 * ring along with the type and permission are returned back. 2074 */ 2075 int 2076 ldc_mem_dring_info(ldc_dring_handle_t dhandle, ldc_mem_info_t *minfo) 2077 { 2078 ldc_dring_t *dringp; 2079 int rv; 2080 2081 if (dhandle == NULL) { 2082 DWARN(DBG_ALL_LDCS, 2083 "ldc_mem_dring_info: invalid desc ring handle\n"); 2084 return (EINVAL); 2085 } 2086 dringp = (ldc_dring_t *)dhandle; 2087 2088 mutex_enter(&dringp->lock); 2089 2090 if (dringp->mhdl) { 2091 rv = ldc_mem_info(dringp->mhdl, minfo); 2092 if (rv) { 2093 DWARN(DBG_ALL_LDCS, 2094 "ldc_mem_dring_info: error reading mem info\n"); 2095 mutex_exit(&dringp->lock); 2096 return (rv); 2097 } 2098 } else { 2099 minfo->vaddr = dringp->base; 2100 minfo->raddr = NULL; 2101 minfo->status = dringp->status; 2102 } 2103 2104 mutex_exit(&dringp->lock); 2105 2106 return (0); 2107 } 2108 2109 /* 2110 * Map an exported descriptor ring into the local address space. If the 2111 * descriptor ring was exported for direct map access, a HV call is made 2112 * to allocate a RA range. If the map is done via a shadow copy, local 2113 * shadow memory is allocated. 2114 */ 2115 int 2116 ldc_mem_dring_map(ldc_handle_t handle, ldc_mem_cookie_t *cookie, 2117 uint32_t ccount, uint32_t len, uint32_t dsize, uint8_t mtype, 2118 ldc_dring_handle_t *dhandle) 2119 { 2120 int err; 2121 ldc_chan_t *ldcp = (ldc_chan_t *)handle; 2122 ldc_mem_handle_t mhandle; 2123 ldc_dring_t *dringp; 2124 size_t dring_size; 2125 2126 if (dhandle == NULL) { 2127 DWARN(DBG_ALL_LDCS, 2128 "ldc_mem_dring_map: invalid dhandle\n"); 2129 return (EINVAL); 2130 } 2131 2132 /* check to see if channel is initalized */ 2133 if (handle == NULL) { 2134 DWARN(DBG_ALL_LDCS, 2135 "ldc_mem_dring_map: invalid channel handle\n"); 2136 return (EINVAL); 2137 } 2138 ldcp = (ldc_chan_t *)handle; 2139 2140 if (cookie == NULL) { 2141 DWARN(ldcp->id, 2142 "ldc_mem_dring_map: (0x%llx) invalid cookie\n", 2143 ldcp->id); 2144 return (EINVAL); 2145 } 2146 2147 /* FUTURE: For now we support only one cookie per dring */ 2148 ASSERT(ccount == 1); 2149 2150 if (cookie->size < (dsize * len)) { 2151 DWARN(ldcp->id, 2152 "ldc_mem_dring_map: (0x%llx) invalid dsize/len\n", 2153 ldcp->id); 2154 return (EINVAL); 2155 } 2156 2157 /* ensure the mtype is valid */ 2158 if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP)) == 0) { 2159 DWARN(ldcp->id, "ldc_mem_dring_map: invalid map type\n"); 2160 return (EINVAL); 2161 } 2162 2163 /* do not attempt direct map if it's not HV supported or enabled */ 2164 if (!ldc_dring_shmem_hv_ok || !ldc_dring_shmem_enabled) { 2165 mtype = LDC_SHADOW_MAP; 2166 } 2167 2168 *dhandle = 0; 2169 2170 /* Allocate an dring structure */ 2171 dringp = kmem_zalloc(sizeof (ldc_dring_t), KM_SLEEP); 2172 2173 D1(ldcp->id, 2174 "ldc_mem_dring_map: 0x%x,0x%x,0x%x,0x%llx,0x%llx\n", 2175 mtype, len, dsize, cookie->addr, cookie->size); 2176 2177 /* Initialize dring */ 2178 dringp->length = len; 2179 dringp->dsize = dsize; 2180 2181 /* round of to multiple of page size */ 2182 dring_size = len * dsize; 2183 dringp->size = (dring_size & MMU_PAGEMASK); 2184 if (dring_size & MMU_PAGEOFFSET) 2185 dringp->size += MMU_PAGESIZE; 2186 2187 dringp->ldcp = ldcp; 2188 2189 /* create an memory handle */ 2190 err = ldc_mem_alloc_handle(handle, &mhandle); 2191 if (err || mhandle == NULL) { 2192 DWARN(DBG_ALL_LDCS, 2193 "ldc_mem_dring_map: cannot alloc hdl err=%d\n", 2194 err); 2195 kmem_free(dringp, sizeof (ldc_dring_t)); 2196 return (ENOMEM); 2197 } 2198 2199 dringp->mhdl = mhandle; 2200 dringp->base = NULL; 2201 2202 /* map the dring into local memory */ 2203 err = i_ldc_mem_map(mhandle, cookie, ccount, mtype, LDC_MEM_RW, 2204 &(dringp->base), NULL); 2205 if (err || dringp->base == NULL) { 2206 DWARN(DBG_ALL_LDCS, 2207 "ldc_mem_dring_map: cannot map desc ring err=%d\n", err); 2208 (void) ldc_mem_free_handle(mhandle); 2209 kmem_free(dringp, sizeof (ldc_dring_t)); 2210 return (ENOMEM); 2211 } 2212 2213 /* initialize the desc ring lock */ 2214 mutex_init(&dringp->lock, NULL, MUTEX_DRIVER, NULL); 2215 2216 /* Add descriptor ring to channel's imported dring list */ 2217 mutex_enter(&ldcp->imp_dlist_lock); 2218 dringp->ch_next = ldcp->imp_dring_list; 2219 ldcp->imp_dring_list = dringp; 2220 mutex_exit(&ldcp->imp_dlist_lock); 2221 2222 dringp->status = LDC_MAPPED; 2223 2224 *dhandle = (ldc_dring_handle_t)dringp; 2225 2226 return (0); 2227 } 2228 2229 /* 2230 * Unmap a descriptor ring. Free shadow memory (if any). 2231 */ 2232 int 2233 ldc_mem_dring_unmap(ldc_dring_handle_t dhandle) 2234 { 2235 ldc_dring_t *dringp; 2236 ldc_dring_t *tmp_dringp; 2237 ldc_chan_t *ldcp; 2238 2239 if (dhandle == NULL) { 2240 DWARN(DBG_ALL_LDCS, 2241 "ldc_mem_dring_unmap: invalid desc ring handle\n"); 2242 return (EINVAL); 2243 } 2244 dringp = (ldc_dring_t *)dhandle; 2245 2246 if (dringp->status != LDC_MAPPED) { 2247 DWARN(DBG_ALL_LDCS, 2248 "ldc_mem_dring_unmap: not a mapped desc ring\n"); 2249 return (EINVAL); 2250 } 2251 2252 mutex_enter(&dringp->lock); 2253 2254 ldcp = dringp->ldcp; 2255 2256 mutex_enter(&ldcp->imp_dlist_lock); 2257 2258 /* find and unlink the desc ring from channel import list */ 2259 tmp_dringp = ldcp->imp_dring_list; 2260 if (tmp_dringp == dringp) { 2261 ldcp->imp_dring_list = dringp->ch_next; 2262 dringp->ch_next = NULL; 2263 2264 } else { 2265 while (tmp_dringp != NULL) { 2266 if (tmp_dringp->ch_next == dringp) { 2267 tmp_dringp->ch_next = dringp->ch_next; 2268 dringp->ch_next = NULL; 2269 break; 2270 } 2271 tmp_dringp = tmp_dringp->ch_next; 2272 } 2273 if (tmp_dringp == NULL) { 2274 DWARN(DBG_ALL_LDCS, 2275 "ldc_mem_dring_unmap: invalid descriptor\n"); 2276 mutex_exit(&ldcp->imp_dlist_lock); 2277 mutex_exit(&dringp->lock); 2278 return (EINVAL); 2279 } 2280 } 2281 2282 mutex_exit(&ldcp->imp_dlist_lock); 2283 2284 /* do a LDC memory handle unmap and free */ 2285 (void) ldc_mem_unmap(dringp->mhdl); 2286 (void) ldc_mem_free_handle((ldc_mem_handle_t)dringp->mhdl); 2287 2288 dringp->status = 0; 2289 dringp->ldcp = NULL; 2290 2291 mutex_exit(&dringp->lock); 2292 2293 /* destroy dring lock */ 2294 mutex_destroy(&dringp->lock); 2295 2296 /* free desc ring object */ 2297 kmem_free(dringp, sizeof (ldc_dring_t)); 2298 2299 return (0); 2300 } 2301 2302 /* 2303 * Internal entry point for descriptor ring access entry consistency 2304 * semantics. Acquire copies the contents of the remote descriptor ring 2305 * into the local shadow copy. The release operation copies the local 2306 * contents into the remote dring. The start and end locations specify 2307 * bounds for the entries being synchronized. 2308 */ 2309 static int 2310 i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle, 2311 uint8_t direction, uint64_t start, uint64_t end) 2312 { 2313 int err; 2314 ldc_dring_t *dringp; 2315 ldc_chan_t *ldcp; 2316 ldc_mhdl_t *mhdl; 2317 uint64_t soff; 2318 size_t copy_size; 2319 2320 if (dhandle == NULL) { 2321 DWARN(DBG_ALL_LDCS, 2322 "i_ldc_dring_acquire_release: invalid desc ring handle\n"); 2323 return (EINVAL); 2324 } 2325 dringp = (ldc_dring_t *)dhandle; 2326 mutex_enter(&dringp->lock); 2327 2328 if (dringp->status != LDC_MAPPED || dringp->ldcp == NULL) { 2329 DWARN(DBG_ALL_LDCS, 2330 "i_ldc_dring_acquire_release: not a mapped desc ring\n"); 2331 mutex_exit(&dringp->lock); 2332 return (EINVAL); 2333 } 2334 2335 if (start >= dringp->length || end >= dringp->length) { 2336 DWARN(DBG_ALL_LDCS, 2337 "i_ldc_dring_acquire_release: index out of range\n"); 2338 mutex_exit(&dringp->lock); 2339 return (EINVAL); 2340 } 2341 2342 mhdl = (ldc_mhdl_t *)dringp->mhdl; 2343 if (mhdl == NULL) { 2344 DWARN(DBG_ALL_LDCS, 2345 "i_ldc_dring_acquire_release: invalid memory handle\n"); 2346 mutex_exit(&dringp->lock); 2347 return (EINVAL); 2348 } 2349 2350 if (mhdl->mtype != LDC_SHADOW_MAP) { 2351 DWARN(DBG_ALL_LDCS, 2352 "i_ldc_dring_acquire_release: invalid mtype: %d\n", 2353 mhdl->mtype); 2354 mutex_exit(&dringp->lock); 2355 return (EINVAL); 2356 } 2357 2358 /* get the channel handle */ 2359 ldcp = dringp->ldcp; 2360 2361 copy_size = (start <= end) ? (((end - start) + 1) * dringp->dsize) : 2362 ((dringp->length - start) * dringp->dsize); 2363 2364 /* Calculate the relative offset for the first desc */ 2365 soff = (start * dringp->dsize); 2366 2367 /* copy to/from remote from/to local memory */ 2368 D1(ldcp->id, "i_ldc_dring_acquire_release: c1 off=0x%llx sz=0x%llx\n", 2369 soff, copy_size); 2370 err = i_ldc_mem_acquire_release((ldc_mem_handle_t)dringp->mhdl, 2371 direction, soff, copy_size); 2372 if (err) { 2373 DWARN(ldcp->id, 2374 "i_ldc_dring_acquire_release: copy failed\n"); 2375 mutex_exit(&dringp->lock); 2376 return (err); 2377 } 2378 2379 /* do the balance */ 2380 if (start > end) { 2381 copy_size = ((end + 1) * dringp->dsize); 2382 soff = 0; 2383 2384 /* copy to/from remote from/to local memory */ 2385 D1(ldcp->id, "i_ldc_dring_acquire_release: c2 " 2386 "off=0x%llx sz=0x%llx\n", soff, copy_size); 2387 err = i_ldc_mem_acquire_release((ldc_mem_handle_t)dringp->mhdl, 2388 direction, soff, copy_size); 2389 if (err) { 2390 DWARN(ldcp->id, 2391 "i_ldc_dring_acquire_release: copy failed\n"); 2392 mutex_exit(&dringp->lock); 2393 return (err); 2394 } 2395 } 2396 2397 mutex_exit(&dringp->lock); 2398 2399 return (0); 2400 } 2401 2402 /* 2403 * Ensure that the contents in the local dring are consistent 2404 * with the contents if of remote dring 2405 */ 2406 int 2407 ldc_mem_dring_acquire(ldc_dring_handle_t dhandle, uint64_t start, uint64_t end) 2408 { 2409 return (i_ldc_dring_acquire_release(dhandle, LDC_COPY_IN, start, end)); 2410 } 2411 2412 /* 2413 * Ensure that the contents in the remote dring are consistent 2414 * with the contents if of local dring 2415 */ 2416 int 2417 ldc_mem_dring_release(ldc_dring_handle_t dhandle, uint64_t start, uint64_t end) 2418 { 2419 return (i_ldc_dring_acquire_release(dhandle, LDC_COPY_OUT, start, end)); 2420 } 2421