1 /****************************************************************************** 2 * gntalloc.c 3 * 4 * Device for creating grant references (in user-space) that may be shared 5 * with other domains. 6 * 7 * This program is distributed in the hope that it will be useful, 8 * but WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 * GNU General Public License for more details. 11 * 12 * You should have received a copy of the GNU General Public License 13 * along with this program; if not, write to the Free Software 14 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 15 */ 16 17 /* 18 * This driver exists to allow userspace programs in Linux to allocate kernel 19 * memory that will later be shared with another domain. Without this device, 20 * Linux userspace programs cannot create grant references. 21 * 22 * How this stuff works: 23 * X -> granting a page to Y 24 * Y -> mapping the grant from X 25 * 26 * 1. X uses the gntalloc device to allocate a page of kernel memory, P. 27 * 2. X creates an entry in the grant table that says domid(Y) can access P. 28 * This is done without a hypercall unless the grant table needs expansion. 29 * 3. X gives the grant reference identifier, GREF, to Y. 30 * 4. Y maps the page, either directly into kernel memory for use in a backend 31 * driver, or via a the gntdev device to map into the address space of an 32 * application running in Y. This is the first point at which Xen does any 33 * tracking of the page. 34 * 5. A program in X mmap()s a segment of the gntalloc device that corresponds 35 * to the shared page, and can now communicate with Y over the shared page. 36 * 37 * 38 * NOTE TO USERSPACE LIBRARIES: 39 * The grant allocation and mmap()ing are, naturally, two separate operations. 40 * You set up the sharing by calling the create ioctl() and then the mmap(). 41 * Teardown requires munmap() and either close() or ioctl(). 42 * 43 * WARNING: Since Xen does not allow a guest to forcibly end the use of a grant 44 * reference, this device can be used to consume kernel memory by leaving grant 45 * references mapped by another domain when an application exits. Therefore, 46 * there is a global limit on the number of pages that can be allocated. When 47 * all references to the page are unmapped, it will be freed during the next 48 * grant operation. 49 */ 50 51 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt 52 53 #include <linux/atomic.h> 54 #include <linux/module.h> 55 #include <linux/miscdevice.h> 56 #include <linux/kernel.h> 57 #include <linux/init.h> 58 #include <linux/slab.h> 59 #include <linux/fs.h> 60 #include <linux/device.h> 61 #include <linux/mm.h> 62 #include <linux/uaccess.h> 63 #include <linux/types.h> 64 #include <linux/list.h> 65 #include <linux/highmem.h> 66 67 #include <xen/xen.h> 68 #include <xen/page.h> 69 #include <xen/grant_table.h> 70 #include <xen/gntalloc.h> 71 #include <xen/events.h> 72 73 static int limit = 1024; 74 module_param(limit, int, 0644); 75 MODULE_PARM_DESC(limit, "Maximum number of grants that may be allocated by " 76 "the gntalloc device"); 77 78 static LIST_HEAD(gref_list); 79 static DEFINE_MUTEX(gref_mutex); 80 static int gref_size; 81 82 struct notify_info { 83 uint16_t pgoff:12; /* Bits 0-11: Offset of the byte to clear */ 84 uint16_t flags:2; /* Bits 12-13: Unmap notification flags */ 85 int event; /* Port (event channel) to notify */ 86 }; 87 88 /* Metadata on a grant reference. */ 89 struct gntalloc_gref { 90 struct list_head next_gref; /* list entry gref_list */ 91 struct list_head next_file; /* list entry file->list, if open */ 92 struct page *page; /* The shared page */ 93 uint64_t file_index; /* File offset for mmap() */ 94 unsigned int users; /* Use count - when zero, waiting on Xen */ 95 grant_ref_t gref_id; /* The grant reference number */ 96 struct notify_info notify; /* Unmap notification */ 97 }; 98 99 struct gntalloc_file_private_data { 100 struct list_head list; 101 uint64_t index; 102 }; 103 104 struct gntalloc_vma_private_data { 105 struct gntalloc_gref *gref; 106 int users; 107 int count; 108 }; 109 110 static void __del_gref(struct gntalloc_gref *gref); 111 112 static void do_cleanup(void) 113 { 114 struct gntalloc_gref *gref, *n; 115 list_for_each_entry_safe(gref, n, &gref_list, next_gref) { 116 if (!gref->users) 117 __del_gref(gref); 118 } 119 } 120 121 static int add_grefs(struct ioctl_gntalloc_alloc_gref *op, 122 uint32_t *gref_ids, struct gntalloc_file_private_data *priv) 123 { 124 int i, rc, readonly; 125 LIST_HEAD(queue_gref); 126 LIST_HEAD(queue_file); 127 struct gntalloc_gref *gref, *next; 128 129 readonly = !(op->flags & GNTALLOC_FLAG_WRITABLE); 130 for (i = 0; i < op->count; i++) { 131 gref = kzalloc(sizeof(*gref), GFP_KERNEL); 132 if (!gref) { 133 rc = -ENOMEM; 134 goto undo; 135 } 136 list_add_tail(&gref->next_gref, &queue_gref); 137 list_add_tail(&gref->next_file, &queue_file); 138 gref->users = 1; 139 gref->file_index = op->index + i * PAGE_SIZE; 140 gref->page = alloc_page(GFP_KERNEL|__GFP_ZERO); 141 if (!gref->page) { 142 rc = -ENOMEM; 143 goto undo; 144 } 145 146 /* Grant foreign access to the page. */ 147 rc = gnttab_grant_foreign_access(op->domid, 148 xen_page_to_gfn(gref->page), 149 readonly); 150 if (rc < 0) 151 goto undo; 152 gref_ids[i] = gref->gref_id = rc; 153 } 154 155 /* Add to gref lists. */ 156 mutex_lock(&gref_mutex); 157 list_splice_tail(&queue_gref, &gref_list); 158 list_splice_tail(&queue_file, &priv->list); 159 mutex_unlock(&gref_mutex); 160 161 return 0; 162 163 undo: 164 mutex_lock(&gref_mutex); 165 gref_size -= (op->count - i); 166 167 list_for_each_entry_safe(gref, next, &queue_file, next_file) { 168 list_del(&gref->next_file); 169 __del_gref(gref); 170 } 171 172 mutex_unlock(&gref_mutex); 173 return rc; 174 } 175 176 static void __del_gref(struct gntalloc_gref *gref) 177 { 178 if (gref->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) { 179 uint8_t *tmp = kmap_local_page(gref->page); 180 tmp[gref->notify.pgoff] = 0; 181 kunmap_local(tmp); 182 } 183 if (gref->notify.flags & UNMAP_NOTIFY_SEND_EVENT) { 184 notify_remote_via_evtchn(gref->notify.event); 185 evtchn_put(gref->notify.event); 186 } 187 188 gref->notify.flags = 0; 189 190 if (gref->gref_id) { 191 if (gref->page) 192 gnttab_end_foreign_access(gref->gref_id, gref->page); 193 else 194 gnttab_free_grant_reference(gref->gref_id); 195 } 196 197 gref_size--; 198 list_del(&gref->next_gref); 199 200 kfree(gref); 201 } 202 203 /* finds contiguous grant references in a file, returns the first */ 204 static struct gntalloc_gref *find_grefs(struct gntalloc_file_private_data *priv, 205 uint64_t index, uint32_t count) 206 { 207 struct gntalloc_gref *rv = NULL, *gref; 208 list_for_each_entry(gref, &priv->list, next_file) { 209 if (gref->file_index == index && !rv) 210 rv = gref; 211 if (rv) { 212 if (gref->file_index != index) 213 return NULL; 214 index += PAGE_SIZE; 215 count--; 216 if (count == 0) 217 return rv; 218 } 219 } 220 return NULL; 221 } 222 223 /* 224 * ------------------------------------- 225 * File operations. 226 * ------------------------------------- 227 */ 228 static int gntalloc_open(struct inode *inode, struct file *filp) 229 { 230 struct gntalloc_file_private_data *priv; 231 232 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 233 if (!priv) 234 goto out_nomem; 235 INIT_LIST_HEAD(&priv->list); 236 237 filp->private_data = priv; 238 239 pr_debug("%s: priv %p\n", __func__, priv); 240 241 return 0; 242 243 out_nomem: 244 return -ENOMEM; 245 } 246 247 static int gntalloc_release(struct inode *inode, struct file *filp) 248 { 249 struct gntalloc_file_private_data *priv = filp->private_data; 250 struct gntalloc_gref *gref; 251 252 pr_debug("%s: priv %p\n", __func__, priv); 253 254 mutex_lock(&gref_mutex); 255 while (!list_empty(&priv->list)) { 256 gref = list_entry(priv->list.next, 257 struct gntalloc_gref, next_file); 258 list_del(&gref->next_file); 259 gref->users--; 260 if (gref->users == 0) 261 __del_gref(gref); 262 } 263 kfree(priv); 264 mutex_unlock(&gref_mutex); 265 266 return 0; 267 } 268 269 static long gntalloc_ioctl_alloc(struct gntalloc_file_private_data *priv, 270 struct ioctl_gntalloc_alloc_gref __user *arg) 271 { 272 int rc = 0; 273 struct ioctl_gntalloc_alloc_gref op; 274 uint32_t *gref_ids; 275 276 pr_debug("%s: priv %p\n", __func__, priv); 277 278 if (copy_from_user(&op, arg, sizeof(op))) { 279 rc = -EFAULT; 280 goto out; 281 } 282 283 gref_ids = kcalloc(op.count, sizeof(gref_ids[0]), GFP_KERNEL); 284 if (!gref_ids) { 285 rc = -ENOMEM; 286 goto out; 287 } 288 289 mutex_lock(&gref_mutex); 290 /* Clean up pages that were at zero (local) users but were still mapped 291 * by remote domains. Since those pages count towards the limit that we 292 * are about to enforce, removing them here is a good idea. 293 */ 294 do_cleanup(); 295 if (gref_size + op.count > limit) { 296 mutex_unlock(&gref_mutex); 297 rc = -ENOSPC; 298 goto out_free; 299 } 300 gref_size += op.count; 301 op.index = priv->index; 302 priv->index += op.count * PAGE_SIZE; 303 mutex_unlock(&gref_mutex); 304 305 rc = add_grefs(&op, gref_ids, priv); 306 if (rc < 0) 307 goto out_free; 308 309 /* Once we finish add_grefs, it is unsafe to touch the new reference, 310 * since it is possible for a concurrent ioctl to remove it (by guessing 311 * its index). If the userspace application doesn't provide valid memory 312 * to write the IDs to, then it will need to close the file in order to 313 * release - which it will do by segfaulting when it tries to access the 314 * IDs to close them. 315 */ 316 if (copy_to_user(arg, &op, sizeof(op))) { 317 rc = -EFAULT; 318 goto out_free; 319 } 320 if (copy_to_user(arg->gref_ids, gref_ids, 321 sizeof(gref_ids[0]) * op.count)) { 322 rc = -EFAULT; 323 goto out_free; 324 } 325 326 out_free: 327 kfree(gref_ids); 328 out: 329 return rc; 330 } 331 332 static long gntalloc_ioctl_dealloc(struct gntalloc_file_private_data *priv, 333 void __user *arg) 334 { 335 int i, rc = 0; 336 struct ioctl_gntalloc_dealloc_gref op; 337 struct gntalloc_gref *gref, *n; 338 339 pr_debug("%s: priv %p\n", __func__, priv); 340 341 if (copy_from_user(&op, arg, sizeof(op))) { 342 rc = -EFAULT; 343 goto dealloc_grant_out; 344 } 345 346 mutex_lock(&gref_mutex); 347 gref = find_grefs(priv, op.index, op.count); 348 if (gref) { 349 /* Remove from the file list only, and decrease reference count. 350 * The later call to do_cleanup() will remove from gref_list and 351 * free the memory if the pages aren't mapped anywhere. 352 */ 353 for (i = 0; i < op.count; i++) { 354 n = list_entry(gref->next_file.next, 355 struct gntalloc_gref, next_file); 356 list_del(&gref->next_file); 357 gref->users--; 358 gref = n; 359 } 360 } else { 361 rc = -EINVAL; 362 } 363 364 do_cleanup(); 365 366 mutex_unlock(&gref_mutex); 367 dealloc_grant_out: 368 return rc; 369 } 370 371 static long gntalloc_ioctl_unmap_notify(struct gntalloc_file_private_data *priv, 372 void __user *arg) 373 { 374 struct ioctl_gntalloc_unmap_notify op; 375 struct gntalloc_gref *gref; 376 uint64_t index; 377 int pgoff; 378 int rc; 379 380 if (copy_from_user(&op, arg, sizeof(op))) 381 return -EFAULT; 382 383 index = op.index & ~(PAGE_SIZE - 1); 384 pgoff = op.index & (PAGE_SIZE - 1); 385 386 mutex_lock(&gref_mutex); 387 388 gref = find_grefs(priv, index, 1); 389 if (!gref) { 390 rc = -ENOENT; 391 goto unlock_out; 392 } 393 394 if (op.action & ~(UNMAP_NOTIFY_CLEAR_BYTE|UNMAP_NOTIFY_SEND_EVENT)) { 395 rc = -EINVAL; 396 goto unlock_out; 397 } 398 399 /* We need to grab a reference to the event channel we are going to use 400 * to send the notify before releasing the reference we may already have 401 * (if someone has called this ioctl twice). This is required so that 402 * it is possible to change the clear_byte part of the notification 403 * without disturbing the event channel part, which may now be the last 404 * reference to that event channel. 405 */ 406 if (op.action & UNMAP_NOTIFY_SEND_EVENT) { 407 if (evtchn_get(op.event_channel_port)) { 408 rc = -EINVAL; 409 goto unlock_out; 410 } 411 } 412 413 if (gref->notify.flags & UNMAP_NOTIFY_SEND_EVENT) 414 evtchn_put(gref->notify.event); 415 416 gref->notify.flags = op.action; 417 gref->notify.pgoff = pgoff; 418 gref->notify.event = op.event_channel_port; 419 rc = 0; 420 421 unlock_out: 422 mutex_unlock(&gref_mutex); 423 return rc; 424 } 425 426 static long gntalloc_ioctl(struct file *filp, unsigned int cmd, 427 unsigned long arg) 428 { 429 struct gntalloc_file_private_data *priv = filp->private_data; 430 431 switch (cmd) { 432 case IOCTL_GNTALLOC_ALLOC_GREF: 433 return gntalloc_ioctl_alloc(priv, (void __user *)arg); 434 435 case IOCTL_GNTALLOC_DEALLOC_GREF: 436 return gntalloc_ioctl_dealloc(priv, (void __user *)arg); 437 438 case IOCTL_GNTALLOC_SET_UNMAP_NOTIFY: 439 return gntalloc_ioctl_unmap_notify(priv, (void __user *)arg); 440 441 default: 442 return -ENOIOCTLCMD; 443 } 444 445 return 0; 446 } 447 448 static void gntalloc_vma_open(struct vm_area_struct *vma) 449 { 450 struct gntalloc_vma_private_data *priv = vma->vm_private_data; 451 452 if (!priv) 453 return; 454 455 mutex_lock(&gref_mutex); 456 priv->users++; 457 mutex_unlock(&gref_mutex); 458 } 459 460 static void gntalloc_vma_close(struct vm_area_struct *vma) 461 { 462 struct gntalloc_vma_private_data *priv = vma->vm_private_data; 463 struct gntalloc_gref *gref, *next; 464 int i; 465 466 if (!priv) 467 return; 468 469 mutex_lock(&gref_mutex); 470 priv->users--; 471 if (priv->users == 0) { 472 gref = priv->gref; 473 for (i = 0; i < priv->count; i++) { 474 gref->users--; 475 next = list_entry(gref->next_gref.next, 476 struct gntalloc_gref, next_gref); 477 if (gref->users == 0) 478 __del_gref(gref); 479 gref = next; 480 } 481 kfree(priv); 482 } 483 mutex_unlock(&gref_mutex); 484 } 485 486 static const struct vm_operations_struct gntalloc_vmops = { 487 .open = gntalloc_vma_open, 488 .close = gntalloc_vma_close, 489 }; 490 491 static int gntalloc_mmap(struct file *filp, struct vm_area_struct *vma) 492 { 493 struct gntalloc_file_private_data *priv = filp->private_data; 494 struct gntalloc_vma_private_data *vm_priv; 495 struct gntalloc_gref *gref; 496 int count = vma_pages(vma); 497 int rv, i; 498 499 if (!(vma->vm_flags & VM_SHARED)) { 500 pr_err("%s: Mapping must be shared\n", __func__); 501 return -EINVAL; 502 } 503 504 vm_priv = kmalloc(sizeof(*vm_priv), GFP_KERNEL); 505 if (!vm_priv) 506 return -ENOMEM; 507 508 mutex_lock(&gref_mutex); 509 510 pr_debug("%s: priv %p,%p, page %lu+%d\n", __func__, 511 priv, vm_priv, vma->vm_pgoff, count); 512 513 gref = find_grefs(priv, vma->vm_pgoff << PAGE_SHIFT, count); 514 if (gref == NULL) { 515 rv = -ENOENT; 516 pr_debug("%s: Could not find grant reference", 517 __func__); 518 kfree(vm_priv); 519 goto out_unlock; 520 } 521 522 vm_priv->gref = gref; 523 vm_priv->users = 1; 524 vm_priv->count = count; 525 526 vma->vm_private_data = vm_priv; 527 528 vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP; 529 530 vma->vm_ops = &gntalloc_vmops; 531 532 for (i = 0; i < count; i++) { 533 gref->users++; 534 rv = vm_insert_page(vma, vma->vm_start + i * PAGE_SIZE, 535 gref->page); 536 if (rv) 537 goto out_unlock; 538 539 gref = list_entry(gref->next_file.next, 540 struct gntalloc_gref, next_file); 541 } 542 rv = 0; 543 544 out_unlock: 545 mutex_unlock(&gref_mutex); 546 return rv; 547 } 548 549 static const struct file_operations gntalloc_fops = { 550 .owner = THIS_MODULE, 551 .open = gntalloc_open, 552 .release = gntalloc_release, 553 .unlocked_ioctl = gntalloc_ioctl, 554 .mmap = gntalloc_mmap 555 }; 556 557 /* 558 * ------------------------------------- 559 * Module creation/destruction. 560 * ------------------------------------- 561 */ 562 static struct miscdevice gntalloc_miscdev = { 563 .minor = MISC_DYNAMIC_MINOR, 564 .name = "xen/gntalloc", 565 .fops = &gntalloc_fops, 566 }; 567 568 static int __init gntalloc_init(void) 569 { 570 int err; 571 572 if (!xen_domain()) 573 return -ENODEV; 574 575 err = misc_register(&gntalloc_miscdev); 576 if (err != 0) { 577 pr_err("Could not register misc gntalloc device\n"); 578 return err; 579 } 580 581 pr_debug("Created grant allocation device at %d,%d\n", 582 MISC_MAJOR, gntalloc_miscdev.minor); 583 584 return 0; 585 } 586 587 static void __exit gntalloc_exit(void) 588 { 589 misc_deregister(&gntalloc_miscdev); 590 } 591 592 module_init(gntalloc_init); 593 module_exit(gntalloc_exit); 594 595 MODULE_LICENSE("GPL"); 596 MODULE_AUTHOR("Carter Weatherly <carter.weatherly@jhuapl.edu>, " 597 "Daniel De Graaf <dgdegra@tycho.nsa.gov>"); 598 MODULE_DESCRIPTION("User-space grant reference allocator driver"); 599