1 /* 2 * fs/userfaultfd.c 3 * 4 * Copyright (C) 2007 Davide Libenzi <davidel@xmailserver.org> 5 * Copyright (C) 2008-2009 Red Hat, Inc. 6 * Copyright (C) 2015 Red Hat, Inc. 7 * 8 * This work is licensed under the terms of the GNU GPL, version 2. See 9 * the COPYING file in the top-level directory. 10 * 11 * Some part derived from fs/eventfd.c (anon inode setup) and 12 * mm/ksm.c (mm hashing). 13 */ 14 15 #include <linux/hashtable.h> 16 #include <linux/sched.h> 17 #include <linux/mm.h> 18 #include <linux/poll.h> 19 #include <linux/slab.h> 20 #include <linux/seq_file.h> 21 #include <linux/file.h> 22 #include <linux/bug.h> 23 #include <linux/anon_inodes.h> 24 #include <linux/syscalls.h> 25 #include <linux/userfaultfd_k.h> 26 #include <linux/mempolicy.h> 27 #include <linux/ioctl.h> 28 #include <linux/security.h> 29 30 static struct kmem_cache *userfaultfd_ctx_cachep __read_mostly; 31 32 enum userfaultfd_state { 33 UFFD_STATE_WAIT_API, 34 UFFD_STATE_RUNNING, 35 }; 36 37 /* 38 * Start with fault_pending_wqh and fault_wqh so they're more likely 39 * to be in the same cacheline. 40 */ 41 struct userfaultfd_ctx { 42 /* waitqueue head for the pending (i.e. not read) userfaults */ 43 wait_queue_head_t fault_pending_wqh; 44 /* waitqueue head for the userfaults */ 45 wait_queue_head_t fault_wqh; 46 /* waitqueue head for the pseudo fd to wakeup poll/read */ 47 wait_queue_head_t fd_wqh; 48 /* a refile sequence protected by fault_pending_wqh lock */ 49 struct seqcount refile_seq; 50 /* pseudo fd refcounting */ 51 atomic_t refcount; 52 /* userfaultfd syscall flags */ 53 unsigned int flags; 54 /* state machine */ 55 enum userfaultfd_state state; 56 /* released */ 57 bool released; 58 /* mm with one ore more vmas attached to this userfaultfd_ctx */ 59 struct mm_struct *mm; 60 }; 61 62 struct userfaultfd_wait_queue { 63 struct uffd_msg msg; 64 wait_queue_t wq; 65 struct userfaultfd_ctx *ctx; 66 }; 67 68 struct userfaultfd_wake_range { 69 unsigned long start; 70 unsigned long len; 71 }; 72 73 static int userfaultfd_wake_function(wait_queue_t *wq, unsigned mode, 74 int wake_flags, void *key) 75 { 76 struct userfaultfd_wake_range *range = key; 77 int ret; 78 struct userfaultfd_wait_queue *uwq; 79 unsigned long start, len; 80 81 uwq = container_of(wq, struct userfaultfd_wait_queue, wq); 82 ret = 0; 83 /* len == 0 means wake all */ 84 start = range->start; 85 len = range->len; 86 if (len && (start > uwq->msg.arg.pagefault.address || 87 start + len <= uwq->msg.arg.pagefault.address)) 88 goto out; 89 ret = wake_up_state(wq->private, mode); 90 if (ret) 91 /* 92 * Wake only once, autoremove behavior. 93 * 94 * After the effect of list_del_init is visible to the 95 * other CPUs, the waitqueue may disappear from under 96 * us, see the !list_empty_careful() in 97 * handle_userfault(). try_to_wake_up() has an 98 * implicit smp_mb__before_spinlock, and the 99 * wq->private is read before calling the extern 100 * function "wake_up_state" (which in turns calls 101 * try_to_wake_up). While the spin_lock;spin_unlock; 102 * wouldn't be enough, the smp_mb__before_spinlock is 103 * enough to avoid an explicit smp_mb() here. 104 */ 105 list_del_init(&wq->task_list); 106 out: 107 return ret; 108 } 109 110 /** 111 * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd 112 * context. 113 * @ctx: [in] Pointer to the userfaultfd context. 114 * 115 * Returns: In case of success, returns not zero. 116 */ 117 static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx) 118 { 119 if (!atomic_inc_not_zero(&ctx->refcount)) 120 BUG(); 121 } 122 123 /** 124 * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd 125 * context. 126 * @ctx: [in] Pointer to userfaultfd context. 127 * 128 * The userfaultfd context reference must have been previously acquired either 129 * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget(). 130 */ 131 static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx) 132 { 133 if (atomic_dec_and_test(&ctx->refcount)) { 134 VM_BUG_ON(spin_is_locked(&ctx->fault_pending_wqh.lock)); 135 VM_BUG_ON(waitqueue_active(&ctx->fault_pending_wqh)); 136 VM_BUG_ON(spin_is_locked(&ctx->fault_wqh.lock)); 137 VM_BUG_ON(waitqueue_active(&ctx->fault_wqh)); 138 VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock)); 139 VM_BUG_ON(waitqueue_active(&ctx->fd_wqh)); 140 mmput(ctx->mm); 141 kmem_cache_free(userfaultfd_ctx_cachep, ctx); 142 } 143 } 144 145 static inline void msg_init(struct uffd_msg *msg) 146 { 147 BUILD_BUG_ON(sizeof(struct uffd_msg) != 32); 148 /* 149 * Must use memset to zero out the paddings or kernel data is 150 * leaked to userland. 151 */ 152 memset(msg, 0, sizeof(struct uffd_msg)); 153 } 154 155 static inline struct uffd_msg userfault_msg(unsigned long address, 156 unsigned int flags, 157 unsigned long reason) 158 { 159 struct uffd_msg msg; 160 msg_init(&msg); 161 msg.event = UFFD_EVENT_PAGEFAULT; 162 msg.arg.pagefault.address = address; 163 if (flags & FAULT_FLAG_WRITE) 164 /* 165 * If UFFD_FEATURE_PAGEFAULT_FLAG_WRITE was set in the 166 * uffdio_api.features and UFFD_PAGEFAULT_FLAG_WRITE 167 * was not set in a UFFD_EVENT_PAGEFAULT, it means it 168 * was a read fault, otherwise if set it means it's 169 * a write fault. 170 */ 171 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE; 172 if (reason & VM_UFFD_WP) 173 /* 174 * If UFFD_FEATURE_PAGEFAULT_FLAG_WP was set in the 175 * uffdio_api.features and UFFD_PAGEFAULT_FLAG_WP was 176 * not set in a UFFD_EVENT_PAGEFAULT, it means it was 177 * a missing fault, otherwise if set it means it's a 178 * write protect fault. 179 */ 180 msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP; 181 return msg; 182 } 183 184 /* 185 * Verify the pagetables are still not ok after having reigstered into 186 * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any 187 * userfault that has already been resolved, if userfaultfd_read and 188 * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different 189 * threads. 190 */ 191 static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, 192 unsigned long address, 193 unsigned long flags, 194 unsigned long reason) 195 { 196 struct mm_struct *mm = ctx->mm; 197 pgd_t *pgd; 198 pud_t *pud; 199 pmd_t *pmd, _pmd; 200 pte_t *pte; 201 bool ret = true; 202 203 VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem)); 204 205 pgd = pgd_offset(mm, address); 206 if (!pgd_present(*pgd)) 207 goto out; 208 pud = pud_offset(pgd, address); 209 if (!pud_present(*pud)) 210 goto out; 211 pmd = pmd_offset(pud, address); 212 /* 213 * READ_ONCE must function as a barrier with narrower scope 214 * and it must be equivalent to: 215 * _pmd = *pmd; barrier(); 216 * 217 * This is to deal with the instability (as in 218 * pmd_trans_unstable) of the pmd. 219 */ 220 _pmd = READ_ONCE(*pmd); 221 if (!pmd_present(_pmd)) 222 goto out; 223 224 ret = false; 225 if (pmd_trans_huge(_pmd)) 226 goto out; 227 228 /* 229 * the pmd is stable (as in !pmd_trans_unstable) so we can re-read it 230 * and use the standard pte_offset_map() instead of parsing _pmd. 231 */ 232 pte = pte_offset_map(pmd, address); 233 /* 234 * Lockless access: we're in a wait_event so it's ok if it 235 * changes under us. 236 */ 237 if (pte_none(*pte)) 238 ret = true; 239 pte_unmap(pte); 240 241 out: 242 return ret; 243 } 244 245 /* 246 * The locking rules involved in returning VM_FAULT_RETRY depending on 247 * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and 248 * FAULT_FLAG_KILLABLE are not straightforward. The "Caution" 249 * recommendation in __lock_page_or_retry is not an understatement. 250 * 251 * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_sem must be released 252 * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is 253 * not set. 254 * 255 * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not 256 * set, VM_FAULT_RETRY can still be returned if and only if there are 257 * fatal_signal_pending()s, and the mmap_sem must be released before 258 * returning it. 259 */ 260 int handle_userfault(struct vm_area_struct *vma, unsigned long address, 261 unsigned int flags, unsigned long reason) 262 { 263 struct mm_struct *mm = vma->vm_mm; 264 struct userfaultfd_ctx *ctx; 265 struct userfaultfd_wait_queue uwq; 266 int ret; 267 bool must_wait, return_to_userland; 268 269 BUG_ON(!rwsem_is_locked(&mm->mmap_sem)); 270 271 ret = VM_FAULT_SIGBUS; 272 ctx = vma->vm_userfaultfd_ctx.ctx; 273 if (!ctx) 274 goto out; 275 276 BUG_ON(ctx->mm != mm); 277 278 VM_BUG_ON(reason & ~(VM_UFFD_MISSING|VM_UFFD_WP)); 279 VM_BUG_ON(!(reason & VM_UFFD_MISSING) ^ !!(reason & VM_UFFD_WP)); 280 281 /* 282 * If it's already released don't get it. This avoids to loop 283 * in __get_user_pages if userfaultfd_release waits on the 284 * caller of handle_userfault to release the mmap_sem. 285 */ 286 if (unlikely(ACCESS_ONCE(ctx->released))) 287 goto out; 288 289 /* 290 * Check that we can return VM_FAULT_RETRY. 291 * 292 * NOTE: it should become possible to return VM_FAULT_RETRY 293 * even if FAULT_FLAG_TRIED is set without leading to gup() 294 * -EBUSY failures, if the userfaultfd is to be extended for 295 * VM_UFFD_WP tracking and we intend to arm the userfault 296 * without first stopping userland access to the memory. For 297 * VM_UFFD_MISSING userfaults this is enough for now. 298 */ 299 if (unlikely(!(flags & FAULT_FLAG_ALLOW_RETRY))) { 300 /* 301 * Validate the invariant that nowait must allow retry 302 * to be sure not to return SIGBUS erroneously on 303 * nowait invocations. 304 */ 305 BUG_ON(flags & FAULT_FLAG_RETRY_NOWAIT); 306 #ifdef CONFIG_DEBUG_VM 307 if (printk_ratelimit()) { 308 printk(KERN_WARNING 309 "FAULT_FLAG_ALLOW_RETRY missing %x\n", flags); 310 dump_stack(); 311 } 312 #endif 313 goto out; 314 } 315 316 /* 317 * Handle nowait, not much to do other than tell it to retry 318 * and wait. 319 */ 320 ret = VM_FAULT_RETRY; 321 if (flags & FAULT_FLAG_RETRY_NOWAIT) 322 goto out; 323 324 /* take the reference before dropping the mmap_sem */ 325 userfaultfd_ctx_get(ctx); 326 327 init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function); 328 uwq.wq.private = current; 329 uwq.msg = userfault_msg(address, flags, reason); 330 uwq.ctx = ctx; 331 332 return_to_userland = (flags & (FAULT_FLAG_USER|FAULT_FLAG_KILLABLE)) == 333 (FAULT_FLAG_USER|FAULT_FLAG_KILLABLE); 334 335 spin_lock(&ctx->fault_pending_wqh.lock); 336 /* 337 * After the __add_wait_queue the uwq is visible to userland 338 * through poll/read(). 339 */ 340 __add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq); 341 /* 342 * The smp_mb() after __set_current_state prevents the reads 343 * following the spin_unlock to happen before the list_add in 344 * __add_wait_queue. 345 */ 346 set_current_state(return_to_userland ? TASK_INTERRUPTIBLE : 347 TASK_KILLABLE); 348 spin_unlock(&ctx->fault_pending_wqh.lock); 349 350 must_wait = userfaultfd_must_wait(ctx, address, flags, reason); 351 up_read(&mm->mmap_sem); 352 353 if (likely(must_wait && !ACCESS_ONCE(ctx->released) && 354 (return_to_userland ? !signal_pending(current) : 355 !fatal_signal_pending(current)))) { 356 wake_up_poll(&ctx->fd_wqh, POLLIN); 357 schedule(); 358 ret |= VM_FAULT_MAJOR; 359 } 360 361 __set_current_state(TASK_RUNNING); 362 363 if (return_to_userland) { 364 if (signal_pending(current) && 365 !fatal_signal_pending(current)) { 366 /* 367 * If we got a SIGSTOP or SIGCONT and this is 368 * a normal userland page fault, just let 369 * userland return so the signal will be 370 * handled and gdb debugging works. The page 371 * fault code immediately after we return from 372 * this function is going to release the 373 * mmap_sem and it's not depending on it 374 * (unlike gup would if we were not to return 375 * VM_FAULT_RETRY). 376 * 377 * If a fatal signal is pending we still take 378 * the streamlined VM_FAULT_RETRY failure path 379 * and there's no need to retake the mmap_sem 380 * in such case. 381 */ 382 down_read(&mm->mmap_sem); 383 ret = 0; 384 } 385 } 386 387 /* 388 * Here we race with the list_del; list_add in 389 * userfaultfd_ctx_read(), however because we don't ever run 390 * list_del_init() to refile across the two lists, the prev 391 * and next pointers will never point to self. list_add also 392 * would never let any of the two pointers to point to 393 * self. So list_empty_careful won't risk to see both pointers 394 * pointing to self at any time during the list refile. The 395 * only case where list_del_init() is called is the full 396 * removal in the wake function and there we don't re-list_add 397 * and it's fine not to block on the spinlock. The uwq on this 398 * kernel stack can be released after the list_del_init. 399 */ 400 if (!list_empty_careful(&uwq.wq.task_list)) { 401 spin_lock(&ctx->fault_pending_wqh.lock); 402 /* 403 * No need of list_del_init(), the uwq on the stack 404 * will be freed shortly anyway. 405 */ 406 list_del(&uwq.wq.task_list); 407 spin_unlock(&ctx->fault_pending_wqh.lock); 408 } 409 410 /* 411 * ctx may go away after this if the userfault pseudo fd is 412 * already released. 413 */ 414 userfaultfd_ctx_put(ctx); 415 416 out: 417 return ret; 418 } 419 420 static int userfaultfd_release(struct inode *inode, struct file *file) 421 { 422 struct userfaultfd_ctx *ctx = file->private_data; 423 struct mm_struct *mm = ctx->mm; 424 struct vm_area_struct *vma, *prev; 425 /* len == 0 means wake all */ 426 struct userfaultfd_wake_range range = { .len = 0, }; 427 unsigned long new_flags; 428 429 ACCESS_ONCE(ctx->released) = true; 430 431 /* 432 * Flush page faults out of all CPUs. NOTE: all page faults 433 * must be retried without returning VM_FAULT_SIGBUS if 434 * userfaultfd_ctx_get() succeeds but vma->vma_userfault_ctx 435 * changes while handle_userfault released the mmap_sem. So 436 * it's critical that released is set to true (above), before 437 * taking the mmap_sem for writing. 438 */ 439 down_write(&mm->mmap_sem); 440 prev = NULL; 441 for (vma = mm->mmap; vma; vma = vma->vm_next) { 442 cond_resched(); 443 BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^ 444 !!(vma->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP))); 445 if (vma->vm_userfaultfd_ctx.ctx != ctx) { 446 prev = vma; 447 continue; 448 } 449 new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP); 450 prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end, 451 new_flags, vma->anon_vma, 452 vma->vm_file, vma->vm_pgoff, 453 vma_policy(vma), 454 NULL_VM_UFFD_CTX); 455 if (prev) 456 vma = prev; 457 else 458 prev = vma; 459 vma->vm_flags = new_flags; 460 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; 461 } 462 up_write(&mm->mmap_sem); 463 464 /* 465 * After no new page faults can wait on this fault_*wqh, flush 466 * the last page faults that may have been already waiting on 467 * the fault_*wqh. 468 */ 469 spin_lock(&ctx->fault_pending_wqh.lock); 470 __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range); 471 __wake_up_locked_key(&ctx->fault_wqh, TASK_NORMAL, &range); 472 spin_unlock(&ctx->fault_pending_wqh.lock); 473 474 wake_up_poll(&ctx->fd_wqh, POLLHUP); 475 userfaultfd_ctx_put(ctx); 476 return 0; 477 } 478 479 /* fault_pending_wqh.lock must be hold by the caller */ 480 static inline struct userfaultfd_wait_queue *find_userfault( 481 struct userfaultfd_ctx *ctx) 482 { 483 wait_queue_t *wq; 484 struct userfaultfd_wait_queue *uwq; 485 486 VM_BUG_ON(!spin_is_locked(&ctx->fault_pending_wqh.lock)); 487 488 uwq = NULL; 489 if (!waitqueue_active(&ctx->fault_pending_wqh)) 490 goto out; 491 /* walk in reverse to provide FIFO behavior to read userfaults */ 492 wq = list_last_entry(&ctx->fault_pending_wqh.task_list, 493 typeof(*wq), task_list); 494 uwq = container_of(wq, struct userfaultfd_wait_queue, wq); 495 out: 496 return uwq; 497 } 498 499 static unsigned int userfaultfd_poll(struct file *file, poll_table *wait) 500 { 501 struct userfaultfd_ctx *ctx = file->private_data; 502 unsigned int ret; 503 504 poll_wait(file, &ctx->fd_wqh, wait); 505 506 switch (ctx->state) { 507 case UFFD_STATE_WAIT_API: 508 return POLLERR; 509 case UFFD_STATE_RUNNING: 510 /* 511 * poll() never guarantees that read won't block. 512 * userfaults can be waken before they're read(). 513 */ 514 if (unlikely(!(file->f_flags & O_NONBLOCK))) 515 return POLLERR; 516 /* 517 * lockless access to see if there are pending faults 518 * __pollwait last action is the add_wait_queue but 519 * the spin_unlock would allow the waitqueue_active to 520 * pass above the actual list_add inside 521 * add_wait_queue critical section. So use a full 522 * memory barrier to serialize the list_add write of 523 * add_wait_queue() with the waitqueue_active read 524 * below. 525 */ 526 ret = 0; 527 smp_mb(); 528 if (waitqueue_active(&ctx->fault_pending_wqh)) 529 ret = POLLIN; 530 return ret; 531 default: 532 BUG(); 533 } 534 } 535 536 static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait, 537 struct uffd_msg *msg) 538 { 539 ssize_t ret; 540 DECLARE_WAITQUEUE(wait, current); 541 struct userfaultfd_wait_queue *uwq; 542 543 /* always take the fd_wqh lock before the fault_pending_wqh lock */ 544 spin_lock(&ctx->fd_wqh.lock); 545 __add_wait_queue(&ctx->fd_wqh, &wait); 546 for (;;) { 547 set_current_state(TASK_INTERRUPTIBLE); 548 spin_lock(&ctx->fault_pending_wqh.lock); 549 uwq = find_userfault(ctx); 550 if (uwq) { 551 /* 552 * Use a seqcount to repeat the lockless check 553 * in wake_userfault() to avoid missing 554 * wakeups because during the refile both 555 * waitqueue could become empty if this is the 556 * only userfault. 557 */ 558 write_seqcount_begin(&ctx->refile_seq); 559 560 /* 561 * The fault_pending_wqh.lock prevents the uwq 562 * to disappear from under us. 563 * 564 * Refile this userfault from 565 * fault_pending_wqh to fault_wqh, it's not 566 * pending anymore after we read it. 567 * 568 * Use list_del() by hand (as 569 * userfaultfd_wake_function also uses 570 * list_del_init() by hand) to be sure nobody 571 * changes __remove_wait_queue() to use 572 * list_del_init() in turn breaking the 573 * !list_empty_careful() check in 574 * handle_userfault(). The uwq->wq.task_list 575 * must never be empty at any time during the 576 * refile, or the waitqueue could disappear 577 * from under us. The "wait_queue_head_t" 578 * parameter of __remove_wait_queue() is unused 579 * anyway. 580 */ 581 list_del(&uwq->wq.task_list); 582 __add_wait_queue(&ctx->fault_wqh, &uwq->wq); 583 584 write_seqcount_end(&ctx->refile_seq); 585 586 /* careful to always initialize msg if ret == 0 */ 587 *msg = uwq->msg; 588 spin_unlock(&ctx->fault_pending_wqh.lock); 589 ret = 0; 590 break; 591 } 592 spin_unlock(&ctx->fault_pending_wqh.lock); 593 if (signal_pending(current)) { 594 ret = -ERESTARTSYS; 595 break; 596 } 597 if (no_wait) { 598 ret = -EAGAIN; 599 break; 600 } 601 spin_unlock(&ctx->fd_wqh.lock); 602 schedule(); 603 spin_lock(&ctx->fd_wqh.lock); 604 } 605 __remove_wait_queue(&ctx->fd_wqh, &wait); 606 __set_current_state(TASK_RUNNING); 607 spin_unlock(&ctx->fd_wqh.lock); 608 609 return ret; 610 } 611 612 static ssize_t userfaultfd_read(struct file *file, char __user *buf, 613 size_t count, loff_t *ppos) 614 { 615 struct userfaultfd_ctx *ctx = file->private_data; 616 ssize_t _ret, ret = 0; 617 struct uffd_msg msg; 618 int no_wait = file->f_flags & O_NONBLOCK; 619 620 if (ctx->state == UFFD_STATE_WAIT_API) 621 return -EINVAL; 622 623 for (;;) { 624 if (count < sizeof(msg)) 625 return ret ? ret : -EINVAL; 626 _ret = userfaultfd_ctx_read(ctx, no_wait, &msg); 627 if (_ret < 0) 628 return ret ? ret : _ret; 629 if (copy_to_user((__u64 __user *) buf, &msg, sizeof(msg))) 630 return ret ? ret : -EFAULT; 631 ret += sizeof(msg); 632 buf += sizeof(msg); 633 count -= sizeof(msg); 634 /* 635 * Allow to read more than one fault at time but only 636 * block if waiting for the very first one. 637 */ 638 no_wait = O_NONBLOCK; 639 } 640 } 641 642 static void __wake_userfault(struct userfaultfd_ctx *ctx, 643 struct userfaultfd_wake_range *range) 644 { 645 unsigned long start, end; 646 647 start = range->start; 648 end = range->start + range->len; 649 650 spin_lock(&ctx->fault_pending_wqh.lock); 651 /* wake all in the range and autoremove */ 652 if (waitqueue_active(&ctx->fault_pending_wqh)) 653 __wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, 654 range); 655 if (waitqueue_active(&ctx->fault_wqh)) 656 __wake_up_locked_key(&ctx->fault_wqh, TASK_NORMAL, range); 657 spin_unlock(&ctx->fault_pending_wqh.lock); 658 } 659 660 static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx, 661 struct userfaultfd_wake_range *range) 662 { 663 unsigned seq; 664 bool need_wakeup; 665 666 /* 667 * To be sure waitqueue_active() is not reordered by the CPU 668 * before the pagetable update, use an explicit SMP memory 669 * barrier here. PT lock release or up_read(mmap_sem) still 670 * have release semantics that can allow the 671 * waitqueue_active() to be reordered before the pte update. 672 */ 673 smp_mb(); 674 675 /* 676 * Use waitqueue_active because it's very frequent to 677 * change the address space atomically even if there are no 678 * userfaults yet. So we take the spinlock only when we're 679 * sure we've userfaults to wake. 680 */ 681 do { 682 seq = read_seqcount_begin(&ctx->refile_seq); 683 need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) || 684 waitqueue_active(&ctx->fault_wqh); 685 cond_resched(); 686 } while (read_seqcount_retry(&ctx->refile_seq, seq)); 687 if (need_wakeup) 688 __wake_userfault(ctx, range); 689 } 690 691 static __always_inline int validate_range(struct mm_struct *mm, 692 __u64 start, __u64 len) 693 { 694 __u64 task_size = mm->task_size; 695 696 if (start & ~PAGE_MASK) 697 return -EINVAL; 698 if (len & ~PAGE_MASK) 699 return -EINVAL; 700 if (!len) 701 return -EINVAL; 702 if (start < mmap_min_addr) 703 return -EINVAL; 704 if (start >= task_size) 705 return -EINVAL; 706 if (len > task_size - start) 707 return -EINVAL; 708 return 0; 709 } 710 711 static int userfaultfd_register(struct userfaultfd_ctx *ctx, 712 unsigned long arg) 713 { 714 struct mm_struct *mm = ctx->mm; 715 struct vm_area_struct *vma, *prev, *cur; 716 int ret; 717 struct uffdio_register uffdio_register; 718 struct uffdio_register __user *user_uffdio_register; 719 unsigned long vm_flags, new_flags; 720 bool found; 721 unsigned long start, end, vma_end; 722 723 user_uffdio_register = (struct uffdio_register __user *) arg; 724 725 ret = -EFAULT; 726 if (copy_from_user(&uffdio_register, user_uffdio_register, 727 sizeof(uffdio_register)-sizeof(__u64))) 728 goto out; 729 730 ret = -EINVAL; 731 if (!uffdio_register.mode) 732 goto out; 733 if (uffdio_register.mode & ~(UFFDIO_REGISTER_MODE_MISSING| 734 UFFDIO_REGISTER_MODE_WP)) 735 goto out; 736 vm_flags = 0; 737 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING) 738 vm_flags |= VM_UFFD_MISSING; 739 if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) { 740 vm_flags |= VM_UFFD_WP; 741 /* 742 * FIXME: remove the below error constraint by 743 * implementing the wprotect tracking mode. 744 */ 745 ret = -EINVAL; 746 goto out; 747 } 748 749 ret = validate_range(mm, uffdio_register.range.start, 750 uffdio_register.range.len); 751 if (ret) 752 goto out; 753 754 start = uffdio_register.range.start; 755 end = start + uffdio_register.range.len; 756 757 down_write(&mm->mmap_sem); 758 vma = find_vma_prev(mm, start, &prev); 759 760 ret = -ENOMEM; 761 if (!vma) 762 goto out_unlock; 763 764 /* check that there's at least one vma in the range */ 765 ret = -EINVAL; 766 if (vma->vm_start >= end) 767 goto out_unlock; 768 769 /* 770 * Search for not compatible vmas. 771 * 772 * FIXME: this shall be relaxed later so that it doesn't fail 773 * on tmpfs backed vmas (in addition to the current allowance 774 * on anonymous vmas). 775 */ 776 found = false; 777 for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { 778 cond_resched(); 779 780 BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ 781 !!(cur->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP))); 782 783 /* check not compatible vmas */ 784 ret = -EINVAL; 785 if (cur->vm_ops) 786 goto out_unlock; 787 788 /* 789 * Check that this vma isn't already owned by a 790 * different userfaultfd. We can't allow more than one 791 * userfaultfd to own a single vma simultaneously or we 792 * wouldn't know which one to deliver the userfaults to. 793 */ 794 ret = -EBUSY; 795 if (cur->vm_userfaultfd_ctx.ctx && 796 cur->vm_userfaultfd_ctx.ctx != ctx) 797 goto out_unlock; 798 799 found = true; 800 } 801 BUG_ON(!found); 802 803 if (vma->vm_start < start) 804 prev = vma; 805 806 ret = 0; 807 do { 808 cond_resched(); 809 810 BUG_ON(vma->vm_ops); 811 BUG_ON(vma->vm_userfaultfd_ctx.ctx && 812 vma->vm_userfaultfd_ctx.ctx != ctx); 813 814 /* 815 * Nothing to do: this vma is already registered into this 816 * userfaultfd and with the right tracking mode too. 817 */ 818 if (vma->vm_userfaultfd_ctx.ctx == ctx && 819 (vma->vm_flags & vm_flags) == vm_flags) 820 goto skip; 821 822 if (vma->vm_start > start) 823 start = vma->vm_start; 824 vma_end = min(end, vma->vm_end); 825 826 new_flags = (vma->vm_flags & ~vm_flags) | vm_flags; 827 prev = vma_merge(mm, prev, start, vma_end, new_flags, 828 vma->anon_vma, vma->vm_file, vma->vm_pgoff, 829 vma_policy(vma), 830 ((struct vm_userfaultfd_ctx){ ctx })); 831 if (prev) { 832 vma = prev; 833 goto next; 834 } 835 if (vma->vm_start < start) { 836 ret = split_vma(mm, vma, start, 1); 837 if (ret) 838 break; 839 } 840 if (vma->vm_end > end) { 841 ret = split_vma(mm, vma, end, 0); 842 if (ret) 843 break; 844 } 845 next: 846 /* 847 * In the vma_merge() successful mprotect-like case 8: 848 * the next vma was merged into the current one and 849 * the current one has not been updated yet. 850 */ 851 vma->vm_flags = new_flags; 852 vma->vm_userfaultfd_ctx.ctx = ctx; 853 854 skip: 855 prev = vma; 856 start = vma->vm_end; 857 vma = vma->vm_next; 858 } while (vma && vma->vm_start < end); 859 out_unlock: 860 up_write(&mm->mmap_sem); 861 if (!ret) { 862 /* 863 * Now that we scanned all vmas we can already tell 864 * userland which ioctls methods are guaranteed to 865 * succeed on this range. 866 */ 867 if (put_user(UFFD_API_RANGE_IOCTLS, 868 &user_uffdio_register->ioctls)) 869 ret = -EFAULT; 870 } 871 out: 872 return ret; 873 } 874 875 static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, 876 unsigned long arg) 877 { 878 struct mm_struct *mm = ctx->mm; 879 struct vm_area_struct *vma, *prev, *cur; 880 int ret; 881 struct uffdio_range uffdio_unregister; 882 unsigned long new_flags; 883 bool found; 884 unsigned long start, end, vma_end; 885 const void __user *buf = (void __user *)arg; 886 887 ret = -EFAULT; 888 if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister))) 889 goto out; 890 891 ret = validate_range(mm, uffdio_unregister.start, 892 uffdio_unregister.len); 893 if (ret) 894 goto out; 895 896 start = uffdio_unregister.start; 897 end = start + uffdio_unregister.len; 898 899 down_write(&mm->mmap_sem); 900 vma = find_vma_prev(mm, start, &prev); 901 902 ret = -ENOMEM; 903 if (!vma) 904 goto out_unlock; 905 906 /* check that there's at least one vma in the range */ 907 ret = -EINVAL; 908 if (vma->vm_start >= end) 909 goto out_unlock; 910 911 /* 912 * Search for not compatible vmas. 913 * 914 * FIXME: this shall be relaxed later so that it doesn't fail 915 * on tmpfs backed vmas (in addition to the current allowance 916 * on anonymous vmas). 917 */ 918 found = false; 919 ret = -EINVAL; 920 for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { 921 cond_resched(); 922 923 BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ 924 !!(cur->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP))); 925 926 /* 927 * Check not compatible vmas, not strictly required 928 * here as not compatible vmas cannot have an 929 * userfaultfd_ctx registered on them, but this 930 * provides for more strict behavior to notice 931 * unregistration errors. 932 */ 933 if (cur->vm_ops) 934 goto out_unlock; 935 936 found = true; 937 } 938 BUG_ON(!found); 939 940 if (vma->vm_start < start) 941 prev = vma; 942 943 ret = 0; 944 do { 945 cond_resched(); 946 947 BUG_ON(vma->vm_ops); 948 949 /* 950 * Nothing to do: this vma is already registered into this 951 * userfaultfd and with the right tracking mode too. 952 */ 953 if (!vma->vm_userfaultfd_ctx.ctx) 954 goto skip; 955 956 if (vma->vm_start > start) 957 start = vma->vm_start; 958 vma_end = min(end, vma->vm_end); 959 960 new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP); 961 prev = vma_merge(mm, prev, start, vma_end, new_flags, 962 vma->anon_vma, vma->vm_file, vma->vm_pgoff, 963 vma_policy(vma), 964 NULL_VM_UFFD_CTX); 965 if (prev) { 966 vma = prev; 967 goto next; 968 } 969 if (vma->vm_start < start) { 970 ret = split_vma(mm, vma, start, 1); 971 if (ret) 972 break; 973 } 974 if (vma->vm_end > end) { 975 ret = split_vma(mm, vma, end, 0); 976 if (ret) 977 break; 978 } 979 next: 980 /* 981 * In the vma_merge() successful mprotect-like case 8: 982 * the next vma was merged into the current one and 983 * the current one has not been updated yet. 984 */ 985 vma->vm_flags = new_flags; 986 vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; 987 988 skip: 989 prev = vma; 990 start = vma->vm_end; 991 vma = vma->vm_next; 992 } while (vma && vma->vm_start < end); 993 out_unlock: 994 up_write(&mm->mmap_sem); 995 out: 996 return ret; 997 } 998 999 /* 1000 * userfaultfd_wake may be used in combination with the 1001 * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches. 1002 */ 1003 static int userfaultfd_wake(struct userfaultfd_ctx *ctx, 1004 unsigned long arg) 1005 { 1006 int ret; 1007 struct uffdio_range uffdio_wake; 1008 struct userfaultfd_wake_range range; 1009 const void __user *buf = (void __user *)arg; 1010 1011 ret = -EFAULT; 1012 if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake))) 1013 goto out; 1014 1015 ret = validate_range(ctx->mm, uffdio_wake.start, uffdio_wake.len); 1016 if (ret) 1017 goto out; 1018 1019 range.start = uffdio_wake.start; 1020 range.len = uffdio_wake.len; 1021 1022 /* 1023 * len == 0 means wake all and we don't want to wake all here, 1024 * so check it again to be sure. 1025 */ 1026 VM_BUG_ON(!range.len); 1027 1028 wake_userfault(ctx, &range); 1029 ret = 0; 1030 1031 out: 1032 return ret; 1033 } 1034 1035 static int userfaultfd_copy(struct userfaultfd_ctx *ctx, 1036 unsigned long arg) 1037 { 1038 __s64 ret; 1039 struct uffdio_copy uffdio_copy; 1040 struct uffdio_copy __user *user_uffdio_copy; 1041 struct userfaultfd_wake_range range; 1042 1043 user_uffdio_copy = (struct uffdio_copy __user *) arg; 1044 1045 ret = -EFAULT; 1046 if (copy_from_user(&uffdio_copy, user_uffdio_copy, 1047 /* don't copy "copy" last field */ 1048 sizeof(uffdio_copy)-sizeof(__s64))) 1049 goto out; 1050 1051 ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len); 1052 if (ret) 1053 goto out; 1054 /* 1055 * double check for wraparound just in case. copy_from_user() 1056 * will later check uffdio_copy.src + uffdio_copy.len to fit 1057 * in the userland range. 1058 */ 1059 ret = -EINVAL; 1060 if (uffdio_copy.src + uffdio_copy.len <= uffdio_copy.src) 1061 goto out; 1062 if (uffdio_copy.mode & ~UFFDIO_COPY_MODE_DONTWAKE) 1063 goto out; 1064 1065 ret = mcopy_atomic(ctx->mm, uffdio_copy.dst, uffdio_copy.src, 1066 uffdio_copy.len); 1067 if (unlikely(put_user(ret, &user_uffdio_copy->copy))) 1068 return -EFAULT; 1069 if (ret < 0) 1070 goto out; 1071 BUG_ON(!ret); 1072 /* len == 0 would wake all */ 1073 range.len = ret; 1074 if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) { 1075 range.start = uffdio_copy.dst; 1076 wake_userfault(ctx, &range); 1077 } 1078 ret = range.len == uffdio_copy.len ? 0 : -EAGAIN; 1079 out: 1080 return ret; 1081 } 1082 1083 static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx, 1084 unsigned long arg) 1085 { 1086 __s64 ret; 1087 struct uffdio_zeropage uffdio_zeropage; 1088 struct uffdio_zeropage __user *user_uffdio_zeropage; 1089 struct userfaultfd_wake_range range; 1090 1091 user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg; 1092 1093 ret = -EFAULT; 1094 if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage, 1095 /* don't copy "zeropage" last field */ 1096 sizeof(uffdio_zeropage)-sizeof(__s64))) 1097 goto out; 1098 1099 ret = validate_range(ctx->mm, uffdio_zeropage.range.start, 1100 uffdio_zeropage.range.len); 1101 if (ret) 1102 goto out; 1103 ret = -EINVAL; 1104 if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE) 1105 goto out; 1106 1107 ret = mfill_zeropage(ctx->mm, uffdio_zeropage.range.start, 1108 uffdio_zeropage.range.len); 1109 if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage))) 1110 return -EFAULT; 1111 if (ret < 0) 1112 goto out; 1113 /* len == 0 would wake all */ 1114 BUG_ON(!ret); 1115 range.len = ret; 1116 if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) { 1117 range.start = uffdio_zeropage.range.start; 1118 wake_userfault(ctx, &range); 1119 } 1120 ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN; 1121 out: 1122 return ret; 1123 } 1124 1125 /* 1126 * userland asks for a certain API version and we return which bits 1127 * and ioctl commands are implemented in this kernel for such API 1128 * version or -EINVAL if unknown. 1129 */ 1130 static int userfaultfd_api(struct userfaultfd_ctx *ctx, 1131 unsigned long arg) 1132 { 1133 struct uffdio_api uffdio_api; 1134 void __user *buf = (void __user *)arg; 1135 int ret; 1136 1137 ret = -EINVAL; 1138 if (ctx->state != UFFD_STATE_WAIT_API) 1139 goto out; 1140 ret = -EFAULT; 1141 if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api))) 1142 goto out; 1143 if (uffdio_api.api != UFFD_API || uffdio_api.features) { 1144 memset(&uffdio_api, 0, sizeof(uffdio_api)); 1145 if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api))) 1146 goto out; 1147 ret = -EINVAL; 1148 goto out; 1149 } 1150 uffdio_api.features = UFFD_API_FEATURES; 1151 uffdio_api.ioctls = UFFD_API_IOCTLS; 1152 ret = -EFAULT; 1153 if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api))) 1154 goto out; 1155 ctx->state = UFFD_STATE_RUNNING; 1156 ret = 0; 1157 out: 1158 return ret; 1159 } 1160 1161 static long userfaultfd_ioctl(struct file *file, unsigned cmd, 1162 unsigned long arg) 1163 { 1164 int ret = -EINVAL; 1165 struct userfaultfd_ctx *ctx = file->private_data; 1166 1167 if (cmd != UFFDIO_API && ctx->state == UFFD_STATE_WAIT_API) 1168 return -EINVAL; 1169 1170 switch(cmd) { 1171 case UFFDIO_API: 1172 ret = userfaultfd_api(ctx, arg); 1173 break; 1174 case UFFDIO_REGISTER: 1175 ret = userfaultfd_register(ctx, arg); 1176 break; 1177 case UFFDIO_UNREGISTER: 1178 ret = userfaultfd_unregister(ctx, arg); 1179 break; 1180 case UFFDIO_WAKE: 1181 ret = userfaultfd_wake(ctx, arg); 1182 break; 1183 case UFFDIO_COPY: 1184 ret = userfaultfd_copy(ctx, arg); 1185 break; 1186 case UFFDIO_ZEROPAGE: 1187 ret = userfaultfd_zeropage(ctx, arg); 1188 break; 1189 } 1190 return ret; 1191 } 1192 1193 #ifdef CONFIG_PROC_FS 1194 static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f) 1195 { 1196 struct userfaultfd_ctx *ctx = f->private_data; 1197 wait_queue_t *wq; 1198 struct userfaultfd_wait_queue *uwq; 1199 unsigned long pending = 0, total = 0; 1200 1201 spin_lock(&ctx->fault_pending_wqh.lock); 1202 list_for_each_entry(wq, &ctx->fault_pending_wqh.task_list, task_list) { 1203 uwq = container_of(wq, struct userfaultfd_wait_queue, wq); 1204 pending++; 1205 total++; 1206 } 1207 list_for_each_entry(wq, &ctx->fault_wqh.task_list, task_list) { 1208 uwq = container_of(wq, struct userfaultfd_wait_queue, wq); 1209 total++; 1210 } 1211 spin_unlock(&ctx->fault_pending_wqh.lock); 1212 1213 /* 1214 * If more protocols will be added, there will be all shown 1215 * separated by a space. Like this: 1216 * protocols: aa:... bb:... 1217 */ 1218 seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n", 1219 pending, total, UFFD_API, UFFD_API_FEATURES, 1220 UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS); 1221 } 1222 #endif 1223 1224 static const struct file_operations userfaultfd_fops = { 1225 #ifdef CONFIG_PROC_FS 1226 .show_fdinfo = userfaultfd_show_fdinfo, 1227 #endif 1228 .release = userfaultfd_release, 1229 .poll = userfaultfd_poll, 1230 .read = userfaultfd_read, 1231 .unlocked_ioctl = userfaultfd_ioctl, 1232 .compat_ioctl = userfaultfd_ioctl, 1233 .llseek = noop_llseek, 1234 }; 1235 1236 static void init_once_userfaultfd_ctx(void *mem) 1237 { 1238 struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem; 1239 1240 init_waitqueue_head(&ctx->fault_pending_wqh); 1241 init_waitqueue_head(&ctx->fault_wqh); 1242 init_waitqueue_head(&ctx->fd_wqh); 1243 seqcount_init(&ctx->refile_seq); 1244 } 1245 1246 /** 1247 * userfaultfd_file_create - Creates an userfaultfd file pointer. 1248 * @flags: Flags for the userfaultfd file. 1249 * 1250 * This function creates an userfaultfd file pointer, w/out installing 1251 * it into the fd table. This is useful when the userfaultfd file is 1252 * used during the initialization of data structures that require 1253 * extra setup after the userfaultfd creation. So the userfaultfd 1254 * creation is split into the file pointer creation phase, and the 1255 * file descriptor installation phase. In this way races with 1256 * userspace closing the newly installed file descriptor can be 1257 * avoided. Returns an userfaultfd file pointer, or a proper error 1258 * pointer. 1259 */ 1260 static struct file *userfaultfd_file_create(int flags) 1261 { 1262 struct file *file; 1263 struct userfaultfd_ctx *ctx; 1264 1265 BUG_ON(!current->mm); 1266 1267 /* Check the UFFD_* constants for consistency. */ 1268 BUILD_BUG_ON(UFFD_CLOEXEC != O_CLOEXEC); 1269 BUILD_BUG_ON(UFFD_NONBLOCK != O_NONBLOCK); 1270 1271 file = ERR_PTR(-EINVAL); 1272 if (flags & ~UFFD_SHARED_FCNTL_FLAGS) 1273 goto out; 1274 1275 file = ERR_PTR(-ENOMEM); 1276 ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL); 1277 if (!ctx) 1278 goto out; 1279 1280 atomic_set(&ctx->refcount, 1); 1281 ctx->flags = flags; 1282 ctx->state = UFFD_STATE_WAIT_API; 1283 ctx->released = false; 1284 ctx->mm = current->mm; 1285 /* prevent the mm struct to be freed */ 1286 atomic_inc(&ctx->mm->mm_users); 1287 1288 file = anon_inode_getfile("[userfaultfd]", &userfaultfd_fops, ctx, 1289 O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS)); 1290 if (IS_ERR(file)) { 1291 mmput(ctx->mm); 1292 kmem_cache_free(userfaultfd_ctx_cachep, ctx); 1293 } 1294 out: 1295 return file; 1296 } 1297 1298 SYSCALL_DEFINE1(userfaultfd, int, flags) 1299 { 1300 int fd, error; 1301 struct file *file; 1302 1303 error = get_unused_fd_flags(flags & UFFD_SHARED_FCNTL_FLAGS); 1304 if (error < 0) 1305 return error; 1306 fd = error; 1307 1308 file = userfaultfd_file_create(flags); 1309 if (IS_ERR(file)) { 1310 error = PTR_ERR(file); 1311 goto err_put_unused_fd; 1312 } 1313 fd_install(fd, file); 1314 1315 return fd; 1316 1317 err_put_unused_fd: 1318 put_unused_fd(fd); 1319 1320 return error; 1321 } 1322 1323 static int __init userfaultfd_init(void) 1324 { 1325 userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache", 1326 sizeof(struct userfaultfd_ctx), 1327 0, 1328 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 1329 init_once_userfaultfd_ctx); 1330 return 0; 1331 } 1332 __initcall(userfaultfd_init); 1333