1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * gnttab.c 29 * 30 * Granting foreign access to our memory reservation. 31 * 32 * Copyright (c) 2005-2006, Christopher Clark 33 * Copyright (c) 2004-2005, K A Fraser 34 * 35 * This program is free software; you can redistribute it and/or 36 * modify it under the terms of the GNU General Public License version 2 37 * as published by the Free Software Foundation; or, when distributed 38 * separately from the Linux kernel or incorporated into other 39 * software packages, subject to the following license: 40 * 41 * Permission is hereby granted, free of charge, to any person obtaining a copy 42 * of this source file (the "Software"), to deal in the Software without 43 * restriction, including without limitation the rights to use, copy, modify, 44 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 45 * and to permit persons to whom the Software is furnished to do so, subject to 46 * the following conditions: 47 * 48 * The above copyright notice and this permission notice shall be included in 49 * all copies or substantial portions of the Software. 50 * 51 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 52 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 53 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 54 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 55 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 56 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 57 * IN THE SOFTWARE. 58 */ 59 60 #include <sys/types.h> 61 #include <sys/archsystm.h> 62 #ifdef XPV_HVM_DRIVER 63 #include <sys/xpv_support.h> 64 #include <sys/mman.h> 65 #include <vm/hat.h> 66 #endif 67 #include <sys/hypervisor.h> 68 #include <sys/gnttab.h> 69 #include <sys/sysmacros.h> 70 #include <sys/machsystm.h> 71 #include <sys/systm.h> 72 #include <sys/mutex.h> 73 #include <sys/atomic.h> 74 #include <sys/spl.h> 75 #include <sys/condvar.h> 76 #include <sys/cpuvar.h> 77 #include <sys/taskq.h> 78 #include <sys/panic.h> 79 #include <sys/cmn_err.h> 80 #include <sys/promif.h> 81 #include <sys/cpu.h> 82 #include <sys/vmem.h> 83 #include <vm/hat_i86.h> 84 #include <sys/bootconf.h> 85 #include <sys/bootsvcs.h> 86 #ifndef XPV_HVM_DRIVER 87 #include <sys/bootinfo.h> 88 #include <sys/multiboot.h> 89 #include <vm/kboot_mmu.h> 90 #endif 91 #include <sys/bootvfs.h> 92 #include <sys/bootprops.h> 93 #include <vm/seg_kmem.h> 94 #include <sys/mman.h> 95 96 /* Globals */ 97 98 static grant_ref_t **gnttab_list; 99 static uint_t nr_grant_frames; 100 static int gnttab_free_count; 101 static grant_ref_t gnttab_free_head; 102 static kmutex_t gnttab_list_lock; 103 static grant_entry_t *shared; 104 static struct gnttab_free_callback *gnttab_free_callback_list; 105 106 /* Macros */ 107 108 #define GT_PGADDR(i) ((uintptr_t)shared + ((i) << MMU_PAGESHIFT)) 109 #define VALID_GRANT_REF(r) ((r) < (nr_grant_frames * GREFS_PER_GRANT_FRAME)) 110 #define RPP (PAGESIZE / sizeof (grant_ref_t)) 111 #define GNTTAB_ENTRY(entry) (gnttab_list[(entry) / RPP][(entry) % RPP]) 112 #define CMPXCHG(t, c, n) atomic_cas_16((t), (c), (n)) 113 /* External tools reserve first few grant table entries. */ 114 #define NR_RESERVED_ENTRIES 8 115 #define GNTTAB_LIST_END 0xffffffff 116 #define GREFS_PER_GRANT_FRAME (PAGESIZE / sizeof (grant_entry_t)) 117 118 /* Implementation */ 119 120 static uint_t 121 max_nr_grant_frames(void) 122 { 123 struct gnttab_query_size query; 124 int rc; 125 126 query.dom = DOMID_SELF; 127 128 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1); 129 if ((rc < 0) || (query.status != GNTST_okay)) 130 return (4); /* Legacy max supported number of frames */ 131 132 ASSERT(query.max_nr_frames); 133 return (query.max_nr_frames); 134 } 135 136 static void 137 do_free_callbacks(void) 138 { 139 struct gnttab_free_callback *callback, *next; 140 141 callback = gnttab_free_callback_list; 142 gnttab_free_callback_list = NULL; 143 144 while (callback != NULL) { 145 next = callback->next; 146 if (gnttab_free_count >= callback->count) { 147 callback->next = NULL; 148 callback->fn(callback->arg); 149 } else { 150 callback->next = gnttab_free_callback_list; 151 gnttab_free_callback_list = callback; 152 } 153 callback = next; 154 } 155 } 156 157 static void 158 check_free_callbacks(void) 159 { 160 if (gnttab_free_callback_list) 161 do_free_callbacks(); 162 } 163 164 static int 165 grow_gnttab_list(uint_t more_frames) 166 { 167 uint_t new_nr_grant_frames, extra_entries, i; 168 uint_t nr_glist_frames, new_nr_glist_frames; 169 170 ASSERT(MUTEX_HELD(&gnttab_list_lock)); 171 172 new_nr_grant_frames = nr_grant_frames + more_frames; 173 extra_entries = more_frames * GREFS_PER_GRANT_FRAME; 174 175 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) 176 / RPP; 177 new_nr_glist_frames = (new_nr_grant_frames * GREFS_PER_GRANT_FRAME 178 + RPP - 1) / RPP; 179 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) 180 gnttab_list[i] = kmem_alloc(PAGESIZE, KM_SLEEP); 181 182 for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames; 183 i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++) 184 GNTTAB_ENTRY(i) = i + 1; 185 186 GNTTAB_ENTRY(i) = gnttab_free_head; 187 gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames; 188 gnttab_free_count += extra_entries; 189 190 nr_grant_frames = new_nr_grant_frames; 191 192 check_free_callbacks(); 193 194 return (0); 195 } 196 197 static int 198 gnttab_expand(uint_t req_entries) 199 { 200 uint_t cur, extra; 201 202 ASSERT(MUTEX_HELD(&gnttab_list_lock)); 203 204 cur = nr_grant_frames; 205 extra = ((req_entries + (GREFS_PER_GRANT_FRAME - 1)) / 206 GREFS_PER_GRANT_FRAME); 207 if (cur + extra > max_nr_grant_frames()) 208 return (-1); 209 210 return (grow_gnttab_list(extra)); 211 } 212 213 static int 214 get_free_entries(int count) 215 { 216 int ref, rc; 217 grant_ref_t head; 218 219 mutex_enter(&gnttab_list_lock); 220 if (gnttab_free_count < count && 221 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) { 222 mutex_exit(&gnttab_list_lock); 223 return (rc); 224 } 225 ref = head = gnttab_free_head; 226 gnttab_free_count -= count; 227 while (count-- > 1) 228 head = GNTTAB_ENTRY(head); 229 gnttab_free_head = GNTTAB_ENTRY(head); 230 GNTTAB_ENTRY(head) = GNTTAB_LIST_END; 231 mutex_exit(&gnttab_list_lock); 232 return (ref); 233 } 234 235 static void 236 put_free_entry(grant_ref_t ref) 237 { 238 ASSERT(VALID_GRANT_REF(ref)); 239 240 mutex_enter(&gnttab_list_lock); 241 GNTTAB_ENTRY(ref) = gnttab_free_head; 242 gnttab_free_head = ref; 243 gnttab_free_count++; 244 check_free_callbacks(); 245 mutex_exit(&gnttab_list_lock); 246 } 247 248 /* 249 * Public grant-issuing interface functions 250 */ 251 252 int 253 gnttab_grant_foreign_access(domid_t domid, gnttab_frame_t frame, int readonly) 254 { 255 int ref; 256 257 if ((ref = get_free_entries(1)) == -1) 258 return (-1); 259 260 ASSERT(VALID_GRANT_REF(ref)); 261 262 shared[ref].frame = frame; 263 shared[ref].domid = domid; 264 membar_producer(); 265 shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0); 266 267 return (ref); 268 } 269 270 void 271 gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid, 272 gnttab_frame_t frame, int readonly) 273 { 274 ASSERT(VALID_GRANT_REF(ref)); 275 276 shared[ref].frame = frame; 277 shared[ref].domid = domid; 278 membar_producer(); 279 shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0); 280 } 281 282 283 int 284 gnttab_query_foreign_access(grant_ref_t ref) 285 { 286 uint16_t nflags; 287 288 ASSERT(VALID_GRANT_REF(ref)); 289 290 nflags = shared[ref].flags; 291 292 return (nflags & (GTF_reading|GTF_writing)); 293 } 294 295 /* ARGSUSED */ 296 int 297 gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly) 298 { 299 uint16_t flags, nflags; 300 301 ASSERT(VALID_GRANT_REF(ref)); 302 303 nflags = shared[ref].flags; 304 do { 305 if ((flags = nflags) & (GTF_reading|GTF_writing)) { 306 cmn_err(CE_WARN, "g.e. still in use!"); 307 return (0); 308 } 309 } while ((nflags = CMPXCHG(&shared[ref].flags, flags, 0)) != flags); 310 311 return (1); 312 } 313 314 void 315 gnttab_end_foreign_access(grant_ref_t ref, int readonly, gnttab_frame_t page) 316 { 317 ASSERT(VALID_GRANT_REF(ref)); 318 319 if (gnttab_end_foreign_access_ref(ref, readonly)) { 320 put_free_entry(ref); 321 /* 322 * XXPV - we don't support freeing a page here 323 */ 324 if (page != 0) { 325 cmn_err(CE_WARN, 326 "gnttab_end_foreign_access_ref: using unsupported free_page interface"); 327 /* free_page(page); */ 328 } 329 } else { 330 /* 331 * XXX This needs to be fixed so that the ref and page are 332 * placed on a list to be freed up later. 333 */ 334 cmn_err(CE_WARN, "leaking g.e. and page still in use!"); 335 } 336 } 337 338 int 339 gnttab_grant_foreign_transfer(domid_t domid, pfn_t pfn) 340 { 341 int ref; 342 343 if ((ref = get_free_entries(1)) == -1) 344 return (-1); 345 346 ASSERT(VALID_GRANT_REF(ref)); 347 348 gnttab_grant_foreign_transfer_ref(ref, domid, pfn); 349 350 return (ref); 351 } 352 353 void 354 gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid, pfn_t pfn) 355 { 356 ASSERT(VALID_GRANT_REF(ref)); 357 358 shared[ref].frame = pfn; 359 shared[ref].domid = domid; 360 membar_producer(); 361 shared[ref].flags = GTF_accept_transfer; 362 } 363 364 gnttab_frame_t 365 gnttab_end_foreign_transfer_ref(grant_ref_t ref) 366 { 367 gnttab_frame_t frame; 368 uint16_t flags; 369 370 ASSERT(VALID_GRANT_REF(ref)); 371 372 /* 373 * If a transfer is not even yet started, try to reclaim the grant 374 * reference and return failure (== 0). 375 */ 376 while (!((flags = shared[ref].flags) & GTF_transfer_committed)) { 377 if (CMPXCHG(&shared[ref].flags, flags, 0) == flags) 378 return (0); 379 (void) HYPERVISOR_yield(); 380 } 381 382 /* If a transfer is in progress then wait until it is completed. */ 383 while (!(flags & GTF_transfer_completed)) { 384 flags = shared[ref].flags; 385 (void) HYPERVISOR_yield(); 386 } 387 388 /* Read the frame number /after/ reading completion status. */ 389 membar_consumer(); 390 frame = shared[ref].frame; 391 ASSERT(frame != 0); 392 393 return (frame); 394 } 395 396 gnttab_frame_t 397 gnttab_end_foreign_transfer(grant_ref_t ref) 398 { 399 gnttab_frame_t frame; 400 401 ASSERT(VALID_GRANT_REF(ref)); 402 403 frame = gnttab_end_foreign_transfer_ref(ref); 404 put_free_entry(ref); 405 return (frame); 406 } 407 408 void 409 gnttab_free_grant_reference(grant_ref_t ref) 410 { 411 ASSERT(VALID_GRANT_REF(ref)); 412 413 put_free_entry(ref); 414 } 415 416 void 417 gnttab_free_grant_references(grant_ref_t head) 418 { 419 grant_ref_t ref; 420 int count = 1; 421 422 if (head == GNTTAB_LIST_END) 423 return; 424 mutex_enter(&gnttab_list_lock); 425 ref = head; 426 while (GNTTAB_ENTRY(ref) != GNTTAB_LIST_END) { 427 ref = GNTTAB_ENTRY(ref); 428 count++; 429 } 430 GNTTAB_ENTRY(ref) = gnttab_free_head; 431 gnttab_free_head = head; 432 gnttab_free_count += count; 433 check_free_callbacks(); 434 mutex_exit(&gnttab_list_lock); 435 } 436 437 int 438 gnttab_alloc_grant_references(uint16_t count, grant_ref_t *head) 439 { 440 int h = get_free_entries(count); 441 442 if (h == -1) 443 return (-1); 444 445 *head = h; 446 447 return (0); 448 } 449 450 int 451 gnttab_empty_grant_references(const grant_ref_t *private_head) 452 { 453 return (*private_head == GNTTAB_LIST_END); 454 } 455 456 int 457 gnttab_claim_grant_reference(grant_ref_t *private_head) 458 { 459 grant_ref_t g = *private_head; 460 461 if (g == GNTTAB_LIST_END) 462 return (-1); 463 *private_head = GNTTAB_ENTRY(g); 464 return (g); 465 } 466 467 void 468 gnttab_release_grant_reference(grant_ref_t *private_head, grant_ref_t release) 469 { 470 ASSERT(VALID_GRANT_REF(release)); 471 472 GNTTAB_ENTRY(release) = *private_head; 473 *private_head = release; 474 } 475 476 void 477 gnttab_request_free_callback(struct gnttab_free_callback *callback, 478 void (*fn)(void *), void *arg, uint16_t count) 479 { 480 mutex_enter(&gnttab_list_lock); 481 if (callback->next) 482 goto out; 483 callback->fn = fn; 484 callback->arg = arg; 485 callback->count = count; 486 callback->next = gnttab_free_callback_list; 487 gnttab_free_callback_list = callback; 488 check_free_callbacks(); 489 out: 490 mutex_exit(&gnttab_list_lock); 491 } 492 493 void 494 gnttab_cancel_free_callback(struct gnttab_free_callback *callback) 495 { 496 struct gnttab_free_callback **pcb; 497 498 mutex_enter(&gnttab_list_lock); 499 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) { 500 if (*pcb == callback) { 501 *pcb = callback->next; 502 break; 503 } 504 } 505 mutex_exit(&gnttab_list_lock); 506 } 507 508 static gnttab_frame_t * 509 gnttab_setup(gnttab_setup_table_t *pset) 510 { 511 gnttab_frame_t *frames; 512 513 frames = kmem_alloc(pset->nr_frames * sizeof (gnttab_frame_t), 514 KM_SLEEP); 515 516 /*LINTED: constant in conditional context*/ 517 set_xen_guest_handle(pset->frame_list, frames); 518 519 #ifndef XPV_HVM_DRIVER 520 /* 521 * Take pset->nr_frames pages of grant table space from 522 * the hypervisor and map it 523 */ 524 if ((HYPERVISOR_grant_table_op(GNTTABOP_setup_table, pset, 1) != 0) || 525 (pset->status != 0)) { 526 cmn_err(CE_PANIC, "Grant Table setup failed"); 527 } 528 #endif 529 530 return (frames); 531 } 532 533 #ifdef XPV_HVM_DRIVER 534 static void 535 gnttab_map(void) 536 { 537 struct xen_add_to_physmap xatp; 538 caddr_t va; 539 pfn_t pfn; 540 int i; 541 542 va = (caddr_t)shared; 543 for (i = 0; i < max_nr_grant_frames(); i++) { 544 if ((pfn = hat_getpfnum(kas.a_hat, va)) == PFN_INVALID) 545 cmn_err(CE_PANIC, "gnttab_map: Invalid pfn"); 546 547 xatp.domid = DOMID_SELF; 548 xatp.idx = i; 549 xatp.space = XENMAPSPACE_grant_table; 550 xatp.gpfn = pfn; 551 hat_unload(kas.a_hat, va, MMU_PAGESIZE, HAT_UNLOAD); 552 /* 553 * This call replaces the existing machine page backing 554 * the given gpfn with the page from the allocated grant 555 * table at index idx. The existing machine page is 556 * returned to the free list. 557 */ 558 if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp) != 0) 559 panic("Couldn't map grant table"); 560 hat_devload(kas.a_hat, va, MMU_PAGESIZE, pfn, 561 PROT_READ | PROT_WRITE | HAT_STORECACHING_OK, 562 HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST); 563 va += MMU_PAGESIZE; 564 } 565 } 566 #endif /* XPV_HVM_DRIVER */ 567 568 void 569 gnttab_init(void) 570 { 571 gnttab_setup_table_t set; 572 int i; 573 uint_t nr_init_grefs, max_nr_glist_frames, nr_glist_frames; 574 gnttab_frame_t *frames; 575 576 /* 577 * gnttab_init() should only be invoked once. 578 */ 579 mutex_enter(&gnttab_list_lock); 580 ASSERT(nr_grant_frames == 0); 581 nr_grant_frames = 1; 582 mutex_exit(&gnttab_list_lock); 583 584 max_nr_glist_frames = (max_nr_grant_frames() * 585 GREFS_PER_GRANT_FRAME / RPP); 586 587 set.dom = DOMID_SELF; 588 set.nr_frames = max_nr_grant_frames(); 589 frames = gnttab_setup(&set); 590 591 #ifdef XPV_HVM_DRIVER 592 shared = (grant_entry_t *)xen_alloc_pages(set.nr_frames); 593 594 gnttab_map(); 595 #else /* XPV_HVM_DRIVER */ 596 shared = vmem_xalloc(heap_arena, set.nr_frames * MMU_PAGESIZE, 597 MMU_PAGESIZE, 0, 0, 0, 0, VM_SLEEP); 598 for (i = 0; i < set.nr_frames; i++) { 599 hat_devload(kas.a_hat, (caddr_t)GT_PGADDR(i), PAGESIZE, 600 xen_assign_pfn(frames[i]), 601 PROT_READ | PROT_WRITE | HAT_STORECACHING_OK, 602 HAT_LOAD_LOCK); 603 } 604 #endif 605 606 gnttab_list = kmem_alloc(max_nr_glist_frames * sizeof (grant_ref_t *), 607 KM_SLEEP); 608 609 nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) 610 / RPP; 611 for (i = 0; i < nr_glist_frames; i++) { 612 gnttab_list[i] = kmem_alloc(PAGESIZE, KM_SLEEP); 613 } 614 615 kmem_free(frames, set.nr_frames * sizeof (gnttab_frame_t)); 616 617 nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME; 618 619 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++) 620 GNTTAB_ENTRY(i) = i + 1; 621 622 GNTTAB_ENTRY(nr_init_grefs - 1) = GNTTAB_LIST_END; 623 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES; 624 gnttab_free_head = NR_RESERVED_ENTRIES; 625 } 626 627 void 628 gnttab_resume(void) 629 { 630 gnttab_setup_table_t set; 631 int i; 632 gnttab_frame_t *frames; 633 uint_t available_frames = max_nr_grant_frames(); 634 635 if (available_frames < nr_grant_frames) { 636 cmn_err(CE_PANIC, "Hypervisor does not have enough grant " 637 "frames: required(%u), available(%u)", nr_grant_frames, 638 available_frames); 639 } 640 641 #ifdef XPV_HVM_DRIVER 642 gnttab_map(); 643 #endif /* XPV_HVM_DRIVER */ 644 645 set.dom = DOMID_SELF; 646 set.nr_frames = available_frames; 647 frames = gnttab_setup(&set); 648 649 for (i = 0; i < available_frames; i++) { 650 (void) HYPERVISOR_update_va_mapping(GT_PGADDR(i), 651 FRAME_TO_MA(frames[i]) | PT_VALID | PT_WRITABLE, 652 UVMF_INVLPG | UVMF_ALL); 653 } 654 kmem_free(frames, set.nr_frames * sizeof (gnttab_frame_t)); 655 } 656 657 void 658 gnttab_suspend(void) 659 { 660 int i; 661 662 /* 663 * clear grant table mappings before suspending 664 */ 665 for (i = 0; i < max_nr_grant_frames(); i++) { 666 (void) HYPERVISOR_update_va_mapping(GT_PGADDR(i), 667 0, UVMF_INVLPG); 668 } 669 } 670 671 /* 672 * Local variables: 673 * c-file-style: "solaris" 674 * indent-tabs-mode: t 675 * c-indent-level: 8 676 * c-basic-offset: 8 677 * tab-width: 8 678 * End: 679 */ 680