1 /****************************************************************************** 2 * gnttab.c 3 * 4 * Two sets of functionality: 5 * 1. Granting foreign access to our memory reservation. 6 * 2. Accessing others' memory reservations via grant references. 7 * (i.e., mechanisms for both sender and recipient of grant references) 8 * 9 * Copyright (c) 2005, Christopher Clark 10 * Copyright (c) 2004, K A Fraser 11 */ 12 13 #include <sys/cdefs.h> 14 __FBSDID("$FreeBSD$"); 15 16 #include <sys/param.h> 17 #include <sys/systm.h> 18 #include <sys/bus.h> 19 #include <sys/conf.h> 20 #include <sys/module.h> 21 #include <sys/kernel.h> 22 #include <sys/lock.h> 23 #include <sys/malloc.h> 24 #include <sys/mman.h> 25 #include <sys/limits.h> 26 #include <sys/rman.h> 27 #include <machine/resource.h> 28 #include <machine/cpu.h> 29 30 #include <xen/xen-os.h> 31 #include <xen/hypervisor.h> 32 #include <machine/xen/synch_bitops.h> 33 34 #include <xen/hypervisor.h> 35 #include <xen/gnttab.h> 36 37 #include <vm/vm.h> 38 #include <vm/vm_kern.h> 39 #include <vm/vm_extern.h> 40 #include <vm/pmap.h> 41 42 /* External tools reserve first few grant table entries. */ 43 #define NR_RESERVED_ENTRIES 8 44 #define GREFS_PER_GRANT_FRAME (PAGE_SIZE / sizeof(grant_entry_t)) 45 46 static grant_ref_t **gnttab_list; 47 static unsigned int nr_grant_frames; 48 static unsigned int boot_max_nr_grant_frames; 49 static int gnttab_free_count; 50 static grant_ref_t gnttab_free_head; 51 static struct mtx gnttab_list_lock; 52 53 /* 54 * Resource representing allocated physical address space 55 * for the grant table metainfo 56 */ 57 static struct resource *gnttab_pseudo_phys_res; 58 59 /* Resource id for allocated physical address space. */ 60 static int gnttab_pseudo_phys_res_id; 61 62 static grant_entry_t *shared; 63 64 static struct gnttab_free_callback *gnttab_free_callback_list = NULL; 65 66 static int gnttab_expand(unsigned int req_entries); 67 68 #define RPP (PAGE_SIZE / sizeof(grant_ref_t)) 69 #define gnttab_entry(entry) (gnttab_list[(entry) / RPP][(entry) % RPP]) 70 71 static int 72 get_free_entries(int count, int *entries) 73 { 74 int ref, error; 75 grant_ref_t head; 76 77 mtx_lock(&gnttab_list_lock); 78 if ((gnttab_free_count < count) && 79 ((error = gnttab_expand(count - gnttab_free_count)) != 0)) { 80 mtx_unlock(&gnttab_list_lock); 81 return (error); 82 } 83 ref = head = gnttab_free_head; 84 gnttab_free_count -= count; 85 while (count-- > 1) 86 head = gnttab_entry(head); 87 gnttab_free_head = gnttab_entry(head); 88 gnttab_entry(head) = GNTTAB_LIST_END; 89 mtx_unlock(&gnttab_list_lock); 90 91 *entries = ref; 92 return (0); 93 } 94 95 static void 96 do_free_callbacks(void) 97 { 98 struct gnttab_free_callback *callback, *next; 99 100 callback = gnttab_free_callback_list; 101 gnttab_free_callback_list = NULL; 102 103 while (callback != NULL) { 104 next = callback->next; 105 if (gnttab_free_count >= callback->count) { 106 callback->next = NULL; 107 callback->fn(callback->arg); 108 } else { 109 callback->next = gnttab_free_callback_list; 110 gnttab_free_callback_list = callback; 111 } 112 callback = next; 113 } 114 } 115 116 static inline void 117 check_free_callbacks(void) 118 { 119 if (__predict_false(gnttab_free_callback_list != NULL)) 120 do_free_callbacks(); 121 } 122 123 static void 124 put_free_entry(grant_ref_t ref) 125 { 126 127 mtx_lock(&gnttab_list_lock); 128 gnttab_entry(ref) = gnttab_free_head; 129 gnttab_free_head = ref; 130 gnttab_free_count++; 131 check_free_callbacks(); 132 mtx_unlock(&gnttab_list_lock); 133 } 134 135 /* 136 * Public grant-issuing interface functions 137 */ 138 139 int 140 gnttab_grant_foreign_access(domid_t domid, unsigned long frame, int readonly, 141 grant_ref_t *result) 142 { 143 int error, ref; 144 145 error = get_free_entries(1, &ref); 146 147 if (__predict_false(error)) 148 return (error); 149 150 shared[ref].frame = frame; 151 shared[ref].domid = domid; 152 wmb(); 153 shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0); 154 155 if (result) 156 *result = ref; 157 158 return (0); 159 } 160 161 void 162 gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid, 163 unsigned long frame, int readonly) 164 { 165 166 shared[ref].frame = frame; 167 shared[ref].domid = domid; 168 wmb(); 169 shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0); 170 } 171 172 int 173 gnttab_query_foreign_access(grant_ref_t ref) 174 { 175 uint16_t nflags; 176 177 nflags = shared[ref].flags; 178 179 return (nflags & (GTF_reading|GTF_writing)); 180 } 181 182 int 183 gnttab_end_foreign_access_ref(grant_ref_t ref) 184 { 185 uint16_t flags, nflags; 186 187 nflags = shared[ref].flags; 188 do { 189 if ( (flags = nflags) & (GTF_reading|GTF_writing) ) { 190 printf("%s: WARNING: g.e. still in use!\n", __func__); 191 return (0); 192 } 193 } while ((nflags = synch_cmpxchg(&shared[ref].flags, flags, 0)) != 194 flags); 195 196 return (1); 197 } 198 199 void 200 gnttab_end_foreign_access(grant_ref_t ref, void *page) 201 { 202 if (gnttab_end_foreign_access_ref(ref)) { 203 put_free_entry(ref); 204 if (page != NULL) { 205 free(page, M_DEVBUF); 206 } 207 } 208 else { 209 /* XXX This needs to be fixed so that the ref and page are 210 placed on a list to be freed up later. */ 211 printf("%s: WARNING: leaking g.e. and page still in use!\n", 212 __func__); 213 } 214 } 215 216 void 217 gnttab_end_foreign_access_references(u_int count, grant_ref_t *refs) 218 { 219 grant_ref_t *last_ref; 220 grant_ref_t head; 221 grant_ref_t tail; 222 223 head = GNTTAB_LIST_END; 224 tail = *refs; 225 last_ref = refs + count; 226 while (refs != last_ref) { 227 228 if (gnttab_end_foreign_access_ref(*refs)) { 229 gnttab_entry(*refs) = head; 230 head = *refs; 231 } else { 232 /* 233 * XXX This needs to be fixed so that the ref 234 * is placed on a list to be freed up later. 235 */ 236 printf("%s: WARNING: leaking g.e. still in use!\n", 237 __func__); 238 count--; 239 } 240 refs++; 241 } 242 243 if (count != 0) { 244 mtx_lock(&gnttab_list_lock); 245 gnttab_free_count += count; 246 gnttab_entry(tail) = gnttab_free_head; 247 gnttab_free_head = head; 248 mtx_unlock(&gnttab_list_lock); 249 } 250 } 251 252 int 253 gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn, 254 grant_ref_t *result) 255 { 256 int error, ref; 257 258 error = get_free_entries(1, &ref); 259 if (__predict_false(error)) 260 return (error); 261 262 gnttab_grant_foreign_transfer_ref(ref, domid, pfn); 263 264 *result = ref; 265 return (0); 266 } 267 268 void 269 gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid, 270 unsigned long pfn) 271 { 272 shared[ref].frame = pfn; 273 shared[ref].domid = domid; 274 wmb(); 275 shared[ref].flags = GTF_accept_transfer; 276 } 277 278 unsigned long 279 gnttab_end_foreign_transfer_ref(grant_ref_t ref) 280 { 281 unsigned long frame; 282 uint16_t flags; 283 284 /* 285 * If a transfer is not even yet started, try to reclaim the grant 286 * reference and return failure (== 0). 287 */ 288 while (!((flags = shared[ref].flags) & GTF_transfer_committed)) { 289 if ( synch_cmpxchg(&shared[ref].flags, flags, 0) == flags ) 290 return (0); 291 cpu_spinwait(); 292 } 293 294 /* If a transfer is in progress then wait until it is completed. */ 295 while (!(flags & GTF_transfer_completed)) { 296 flags = shared[ref].flags; 297 cpu_spinwait(); 298 } 299 300 /* Read the frame number /after/ reading completion status. */ 301 rmb(); 302 frame = shared[ref].frame; 303 KASSERT(frame != 0, ("grant table inconsistent")); 304 305 return (frame); 306 } 307 308 unsigned long 309 gnttab_end_foreign_transfer(grant_ref_t ref) 310 { 311 unsigned long frame = gnttab_end_foreign_transfer_ref(ref); 312 313 put_free_entry(ref); 314 return (frame); 315 } 316 317 void 318 gnttab_free_grant_reference(grant_ref_t ref) 319 { 320 321 put_free_entry(ref); 322 } 323 324 void 325 gnttab_free_grant_references(grant_ref_t head) 326 { 327 grant_ref_t ref; 328 int count = 1; 329 330 if (head == GNTTAB_LIST_END) 331 return; 332 333 ref = head; 334 while (gnttab_entry(ref) != GNTTAB_LIST_END) { 335 ref = gnttab_entry(ref); 336 count++; 337 } 338 mtx_lock(&gnttab_list_lock); 339 gnttab_entry(ref) = gnttab_free_head; 340 gnttab_free_head = head; 341 gnttab_free_count += count; 342 check_free_callbacks(); 343 mtx_unlock(&gnttab_list_lock); 344 } 345 346 int 347 gnttab_alloc_grant_references(uint16_t count, grant_ref_t *head) 348 { 349 int ref, error; 350 351 error = get_free_entries(count, &ref); 352 if (__predict_false(error)) 353 return (error); 354 355 *head = ref; 356 return (0); 357 } 358 359 int 360 gnttab_empty_grant_references(const grant_ref_t *private_head) 361 { 362 363 return (*private_head == GNTTAB_LIST_END); 364 } 365 366 int 367 gnttab_claim_grant_reference(grant_ref_t *private_head) 368 { 369 grant_ref_t g = *private_head; 370 371 if (__predict_false(g == GNTTAB_LIST_END)) 372 return (g); 373 *private_head = gnttab_entry(g); 374 return (g); 375 } 376 377 void 378 gnttab_release_grant_reference(grant_ref_t *private_head, grant_ref_t release) 379 { 380 381 gnttab_entry(release) = *private_head; 382 *private_head = release; 383 } 384 385 void 386 gnttab_request_free_callback(struct gnttab_free_callback *callback, 387 void (*fn)(void *), void *arg, uint16_t count) 388 { 389 390 mtx_lock(&gnttab_list_lock); 391 if (callback->next) 392 goto out; 393 callback->fn = fn; 394 callback->arg = arg; 395 callback->count = count; 396 callback->next = gnttab_free_callback_list; 397 gnttab_free_callback_list = callback; 398 check_free_callbacks(); 399 out: 400 mtx_unlock(&gnttab_list_lock); 401 402 } 403 404 void 405 gnttab_cancel_free_callback(struct gnttab_free_callback *callback) 406 { 407 struct gnttab_free_callback **pcb; 408 409 mtx_lock(&gnttab_list_lock); 410 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) { 411 if (*pcb == callback) { 412 *pcb = callback->next; 413 break; 414 } 415 } 416 mtx_unlock(&gnttab_list_lock); 417 } 418 419 420 static int 421 grow_gnttab_list(unsigned int more_frames) 422 { 423 unsigned int new_nr_grant_frames, extra_entries, i; 424 425 new_nr_grant_frames = nr_grant_frames + more_frames; 426 extra_entries = more_frames * GREFS_PER_GRANT_FRAME; 427 428 for (i = nr_grant_frames; i < new_nr_grant_frames; i++) 429 { 430 gnttab_list[i] = (grant_ref_t *) 431 malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT); 432 433 if (!gnttab_list[i]) 434 goto grow_nomem; 435 } 436 437 for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames; 438 i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++) 439 gnttab_entry(i) = i + 1; 440 441 gnttab_entry(i) = gnttab_free_head; 442 gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames; 443 gnttab_free_count += extra_entries; 444 445 nr_grant_frames = new_nr_grant_frames; 446 447 check_free_callbacks(); 448 449 return (0); 450 451 grow_nomem: 452 for ( ; i >= nr_grant_frames; i--) 453 free(gnttab_list[i], M_DEVBUF); 454 return (ENOMEM); 455 } 456 457 static unsigned int 458 __max_nr_grant_frames(void) 459 { 460 struct gnttab_query_size query; 461 int rc; 462 463 query.dom = DOMID_SELF; 464 465 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1); 466 if ((rc < 0) || (query.status != GNTST_okay)) 467 return (4); /* Legacy max supported number of frames */ 468 469 return (query.max_nr_frames); 470 } 471 472 static inline 473 unsigned int max_nr_grant_frames(void) 474 { 475 unsigned int xen_max = __max_nr_grant_frames(); 476 477 if (xen_max > boot_max_nr_grant_frames) 478 return (boot_max_nr_grant_frames); 479 return (xen_max); 480 } 481 482 #ifdef notyet 483 /* 484 * XXX needed for backend support 485 * 486 */ 487 static int 488 map_pte_fn(pte_t *pte, struct page *pmd_page, 489 unsigned long addr, void *data) 490 { 491 unsigned long **frames = (unsigned long **)data; 492 493 set_pte_at(&init_mm, addr, pte, pfn_pte_ma((*frames)[0], PAGE_KERNEL)); 494 (*frames)++; 495 return 0; 496 } 497 498 static int 499 unmap_pte_fn(pte_t *pte, struct page *pmd_page, 500 unsigned long addr, void *data) 501 { 502 503 set_pte_at(&init_mm, addr, pte, __pte(0)); 504 return 0; 505 } 506 #endif 507 508 static vm_paddr_t resume_frames; 509 510 static int 511 gnttab_map(unsigned int start_idx, unsigned int end_idx) 512 { 513 struct xen_add_to_physmap xatp; 514 unsigned int i = end_idx; 515 516 /* 517 * Loop backwards, so that the first hypercall has the largest index, 518 * ensuring that the table will grow only once. 519 */ 520 do { 521 xatp.domid = DOMID_SELF; 522 xatp.idx = i; 523 xatp.space = XENMAPSPACE_grant_table; 524 xatp.gpfn = (resume_frames >> PAGE_SHIFT) + i; 525 if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp)) 526 panic("HYPERVISOR_memory_op failed to map gnttab"); 527 } while (i-- > start_idx); 528 529 if (shared == NULL) { 530 vm_offset_t area; 531 532 area = kva_alloc(PAGE_SIZE * max_nr_grant_frames()); 533 KASSERT(area, ("can't allocate VM space for grant table")); 534 shared = (grant_entry_t *)area; 535 } 536 537 for (i = start_idx; i <= end_idx; i++) { 538 pmap_kenter((vm_offset_t) shared + i * PAGE_SIZE, 539 resume_frames + i * PAGE_SIZE); 540 } 541 542 return (0); 543 } 544 545 int 546 gnttab_resume(device_t dev) 547 { 548 unsigned int max_nr_gframes, nr_gframes; 549 550 nr_gframes = nr_grant_frames; 551 max_nr_gframes = max_nr_grant_frames(); 552 if (max_nr_gframes < nr_gframes) 553 return (ENOSYS); 554 555 if (!resume_frames) { 556 KASSERT(dev != NULL, 557 ("No resume frames and no device provided")); 558 559 gnttab_pseudo_phys_res = xenmem_alloc(dev, 560 &gnttab_pseudo_phys_res_id, PAGE_SIZE * max_nr_gframes); 561 if (gnttab_pseudo_phys_res == NULL) 562 panic("Unable to reserve physical memory for gnttab"); 563 resume_frames = rman_get_start(gnttab_pseudo_phys_res); 564 } 565 566 return (gnttab_map(0, nr_gframes - 1)); 567 } 568 569 static int 570 gnttab_expand(unsigned int req_entries) 571 { 572 int error; 573 unsigned int cur, extra; 574 575 cur = nr_grant_frames; 576 extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) / 577 GREFS_PER_GRANT_FRAME); 578 if (cur + extra > max_nr_grant_frames()) 579 return (ENOSPC); 580 581 error = gnttab_map(cur, cur + extra - 1); 582 if (!error) 583 error = grow_gnttab_list(extra); 584 585 return (error); 586 } 587 588 MTX_SYSINIT(gnttab, &gnttab_list_lock, "GNTTAB LOCK", MTX_DEF); 589 590 /*------------------ Private Device Attachment Functions --------------------*/ 591 /** 592 * \brief Identify instances of this device type in the system. 593 * 594 * \param driver The driver performing this identify action. 595 * \param parent The NewBus parent device for any devices this method adds. 596 */ 597 static void 598 granttable_identify(driver_t *driver __unused, device_t parent) 599 { 600 601 KASSERT(xen_domain(), 602 ("Trying to attach grant-table device on non Xen domain")); 603 /* 604 * A single device instance for our driver is always present 605 * in a system operating under Xen. 606 */ 607 if (BUS_ADD_CHILD(parent, 0, driver->name, 0) == NULL) 608 panic("unable to attach Xen Grant-table device"); 609 } 610 611 /** 612 * \brief Probe for the existence of the Xen Grant-table device 613 * 614 * \param dev NewBus device_t for this instance. 615 * 616 * \return Always returns 0 indicating success. 617 */ 618 static int 619 granttable_probe(device_t dev) 620 { 621 622 device_set_desc(dev, "Xen Grant-table Device"); 623 return (BUS_PROBE_NOWILDCARD); 624 } 625 626 /** 627 * \brief Attach the Xen Grant-table device. 628 * 629 * \param dev NewBus device_t for this instance. 630 * 631 * \return On success, 0. Otherwise an errno value indicating the 632 * type of failure. 633 */ 634 static int 635 granttable_attach(device_t dev) 636 { 637 int i; 638 unsigned int max_nr_glist_frames; 639 unsigned int nr_init_grefs; 640 641 nr_grant_frames = 1; 642 boot_max_nr_grant_frames = __max_nr_grant_frames(); 643 644 /* Determine the maximum number of frames required for the 645 * grant reference free list on the current hypervisor. 646 */ 647 max_nr_glist_frames = (boot_max_nr_grant_frames * 648 GREFS_PER_GRANT_FRAME / 649 (PAGE_SIZE / sizeof(grant_ref_t))); 650 651 gnttab_list = malloc(max_nr_glist_frames * sizeof(grant_ref_t *), 652 M_DEVBUF, M_NOWAIT); 653 654 if (gnttab_list == NULL) 655 return (ENOMEM); 656 657 for (i = 0; i < nr_grant_frames; i++) { 658 gnttab_list[i] = (grant_ref_t *) 659 malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT); 660 if (gnttab_list[i] == NULL) 661 goto ini_nomem; 662 } 663 664 if (gnttab_resume(dev)) 665 return (ENODEV); 666 667 nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME; 668 669 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++) 670 gnttab_entry(i) = i + 1; 671 672 gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END; 673 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES; 674 gnttab_free_head = NR_RESERVED_ENTRIES; 675 676 if (bootverbose) 677 printf("Grant table initialized\n"); 678 679 return (0); 680 681 ini_nomem: 682 for (i--; i >= 0; i--) 683 free(gnttab_list[i], M_DEVBUF); 684 free(gnttab_list, M_DEVBUF); 685 return (ENOMEM); 686 } 687 688 /*-------------------- Private Device Attachment Data -----------------------*/ 689 static device_method_t granttable_methods[] = { 690 /* Device interface */ 691 DEVMETHOD(device_identify, granttable_identify), 692 DEVMETHOD(device_probe, granttable_probe), 693 DEVMETHOD(device_attach, granttable_attach), 694 695 DEVMETHOD_END 696 }; 697 698 DEFINE_CLASS_0(granttable, granttable_driver, granttable_methods, 0); 699 devclass_t granttable_devclass; 700 701 DRIVER_MODULE_ORDERED(granttable, xenpv, granttable_driver, granttable_devclass, 702 NULL, NULL, SI_ORDER_FIRST); 703