1 /* 2 * linux/mm/mlock.c 3 * 4 * (C) Copyright 1995 Linus Torvalds 5 * (C) Copyright 2002 Christoph Hellwig 6 */ 7 8 #include <linux/capability.h> 9 #include <linux/mman.h> 10 #include <linux/mm.h> 11 #include <linux/swap.h> 12 #include <linux/swapops.h> 13 #include <linux/pagemap.h> 14 #include <linux/mempolicy.h> 15 #include <linux/syscalls.h> 16 #include <linux/sched.h> 17 #include <linux/module.h> 18 #include <linux/rmap.h> 19 #include <linux/mmzone.h> 20 #include <linux/hugetlb.h> 21 22 #include "internal.h" 23 24 int can_do_mlock(void) 25 { 26 if (capable(CAP_IPC_LOCK)) 27 return 1; 28 if (current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur != 0) 29 return 1; 30 return 0; 31 } 32 EXPORT_SYMBOL(can_do_mlock); 33 34 #ifdef CONFIG_UNEVICTABLE_LRU 35 /* 36 * Mlocked pages are marked with PageMlocked() flag for efficient testing 37 * in vmscan and, possibly, the fault path; and to support semi-accurate 38 * statistics. 39 * 40 * An mlocked page [PageMlocked(page)] is unevictable. As such, it will 41 * be placed on the LRU "unevictable" list, rather than the [in]active lists. 42 * The unevictable list is an LRU sibling list to the [in]active lists. 43 * PageUnevictable is set to indicate the unevictable state. 44 * 45 * When lazy mlocking via vmscan, it is important to ensure that the 46 * vma's VM_LOCKED status is not concurrently being modified, otherwise we 47 * may have mlocked a page that is being munlocked. So lazy mlock must take 48 * the mmap_sem for read, and verify that the vma really is locked 49 * (see mm/rmap.c). 50 */ 51 52 /* 53 * LRU accounting for clear_page_mlock() 54 */ 55 void __clear_page_mlock(struct page *page) 56 { 57 VM_BUG_ON(!PageLocked(page)); 58 59 if (!page->mapping) { /* truncated ? */ 60 return; 61 } 62 63 dec_zone_page_state(page, NR_MLOCK); 64 count_vm_event(UNEVICTABLE_PGCLEARED); 65 if (!isolate_lru_page(page)) { 66 putback_lru_page(page); 67 } else { 68 /* 69 * We lost the race. the page already moved to evictable list. 70 */ 71 if (PageUnevictable(page)) 72 count_vm_event(UNEVICTABLE_PGSTRANDED); 73 } 74 } 75 76 /* 77 * Mark page as mlocked if not already. 78 * If page on LRU, isolate and putback to move to unevictable list. 79 */ 80 void mlock_vma_page(struct page *page) 81 { 82 BUG_ON(!PageLocked(page)); 83 84 if (!TestSetPageMlocked(page)) { 85 inc_zone_page_state(page, NR_MLOCK); 86 count_vm_event(UNEVICTABLE_PGMLOCKED); 87 if (!isolate_lru_page(page)) 88 putback_lru_page(page); 89 } 90 } 91 92 /* 93 * called from munlock()/munmap() path with page supposedly on the LRU. 94 * 95 * Note: unlike mlock_vma_page(), we can't just clear the PageMlocked 96 * [in try_to_munlock()] and then attempt to isolate the page. We must 97 * isolate the page to keep others from messing with its unevictable 98 * and mlocked state while trying to munlock. However, we pre-clear the 99 * mlocked state anyway as we might lose the isolation race and we might 100 * not get another chance to clear PageMlocked. If we successfully 101 * isolate the page and try_to_munlock() detects other VM_LOCKED vmas 102 * mapping the page, it will restore the PageMlocked state, unless the page 103 * is mapped in a non-linear vma. So, we go ahead and SetPageMlocked(), 104 * perhaps redundantly. 105 * If we lose the isolation race, and the page is mapped by other VM_LOCKED 106 * vmas, we'll detect this in vmscan--via try_to_munlock() or try_to_unmap() 107 * either of which will restore the PageMlocked state by calling 108 * mlock_vma_page() above, if it can grab the vma's mmap sem. 109 */ 110 static void munlock_vma_page(struct page *page) 111 { 112 BUG_ON(!PageLocked(page)); 113 114 if (TestClearPageMlocked(page)) { 115 dec_zone_page_state(page, NR_MLOCK); 116 if (!isolate_lru_page(page)) { 117 int ret = try_to_munlock(page); 118 /* 119 * did try_to_unlock() succeed or punt? 120 */ 121 if (ret == SWAP_SUCCESS || ret == SWAP_AGAIN) 122 count_vm_event(UNEVICTABLE_PGMUNLOCKED); 123 124 putback_lru_page(page); 125 } else { 126 /* 127 * We lost the race. let try_to_unmap() deal 128 * with it. At least we get the page state and 129 * mlock stats right. However, page is still on 130 * the noreclaim list. We'll fix that up when 131 * the page is eventually freed or we scan the 132 * noreclaim list. 133 */ 134 if (PageUnevictable(page)) 135 count_vm_event(UNEVICTABLE_PGSTRANDED); 136 else 137 count_vm_event(UNEVICTABLE_PGMUNLOCKED); 138 } 139 } 140 } 141 142 /** 143 * __mlock_vma_pages_range() - mlock/munlock a range of pages in the vma. 144 * @vma: target vma 145 * @start: start address 146 * @end: end address 147 * @mlock: 0 indicate munlock, otherwise mlock. 148 * 149 * If @mlock == 0, unlock an mlocked range; 150 * else mlock the range of pages. This takes care of making the pages present , 151 * too. 152 * 153 * return 0 on success, negative error code on error. 154 * 155 * vma->vm_mm->mmap_sem must be held for at least read. 156 */ 157 static long __mlock_vma_pages_range(struct vm_area_struct *vma, 158 unsigned long start, unsigned long end, 159 int mlock) 160 { 161 struct mm_struct *mm = vma->vm_mm; 162 unsigned long addr = start; 163 struct page *pages[16]; /* 16 gives a reasonable batch */ 164 int nr_pages = (end - start) / PAGE_SIZE; 165 int ret = 0; 166 int gup_flags = 0; 167 168 VM_BUG_ON(start & ~PAGE_MASK); 169 VM_BUG_ON(end & ~PAGE_MASK); 170 VM_BUG_ON(start < vma->vm_start); 171 VM_BUG_ON(end > vma->vm_end); 172 VM_BUG_ON((!rwsem_is_locked(&mm->mmap_sem)) && 173 (atomic_read(&mm->mm_users) != 0)); 174 175 /* 176 * mlock: don't page populate if vma has PROT_NONE permission. 177 * munlock: always do munlock although the vma has PROT_NONE 178 * permission, or SIGKILL is pending. 179 */ 180 if (!mlock) 181 gup_flags |= GUP_FLAGS_IGNORE_VMA_PERMISSIONS | 182 GUP_FLAGS_IGNORE_SIGKILL; 183 184 if (vma->vm_flags & VM_WRITE) 185 gup_flags |= GUP_FLAGS_WRITE; 186 187 while (nr_pages > 0) { 188 int i; 189 190 cond_resched(); 191 192 /* 193 * get_user_pages makes pages present if we are 194 * setting mlock. and this extra reference count will 195 * disable migration of this page. However, page may 196 * still be truncated out from under us. 197 */ 198 ret = __get_user_pages(current, mm, addr, 199 min_t(int, nr_pages, ARRAY_SIZE(pages)), 200 gup_flags, pages, NULL); 201 /* 202 * This can happen for, e.g., VM_NONLINEAR regions before 203 * a page has been allocated and mapped at a given offset, 204 * or for addresses that map beyond end of a file. 205 * We'll mlock the the pages if/when they get faulted in. 206 */ 207 if (ret < 0) 208 break; 209 if (ret == 0) { 210 /* 211 * We know the vma is there, so the only time 212 * we cannot get a single page should be an 213 * error (ret < 0) case. 214 */ 215 WARN_ON(1); 216 break; 217 } 218 219 lru_add_drain(); /* push cached pages to LRU */ 220 221 for (i = 0; i < ret; i++) { 222 struct page *page = pages[i]; 223 224 lock_page(page); 225 /* 226 * Because we lock page here and migration is blocked 227 * by the elevated reference, we need only check for 228 * page truncation (file-cache only). 229 */ 230 if (page->mapping) { 231 if (mlock) 232 mlock_vma_page(page); 233 else 234 munlock_vma_page(page); 235 } 236 unlock_page(page); 237 put_page(page); /* ref from get_user_pages() */ 238 239 /* 240 * here we assume that get_user_pages() has given us 241 * a list of virtually contiguous pages. 242 */ 243 addr += PAGE_SIZE; /* for next get_user_pages() */ 244 nr_pages--; 245 } 246 ret = 0; 247 } 248 249 return ret; /* count entire vma as locked_vm */ 250 } 251 252 /* 253 * convert get_user_pages() return value to posix mlock() error 254 */ 255 static int __mlock_posix_error_return(long retval) 256 { 257 if (retval == -EFAULT) 258 retval = -ENOMEM; 259 else if (retval == -ENOMEM) 260 retval = -EAGAIN; 261 return retval; 262 } 263 264 #else /* CONFIG_UNEVICTABLE_LRU */ 265 266 /* 267 * Just make pages present if VM_LOCKED. No-op if unlocking. 268 */ 269 static long __mlock_vma_pages_range(struct vm_area_struct *vma, 270 unsigned long start, unsigned long end, 271 int mlock) 272 { 273 if (mlock && (vma->vm_flags & VM_LOCKED)) 274 return make_pages_present(start, end); 275 return 0; 276 } 277 278 static inline int __mlock_posix_error_return(long retval) 279 { 280 return 0; 281 } 282 283 #endif /* CONFIG_UNEVICTABLE_LRU */ 284 285 /** 286 * mlock_vma_pages_range() - mlock pages in specified vma range. 287 * @vma - the vma containing the specfied address range 288 * @start - starting address in @vma to mlock 289 * @end - end address [+1] in @vma to mlock 290 * 291 * For mmap()/mremap()/expansion of mlocked vma. 292 * 293 * return 0 on success for "normal" vmas. 294 * 295 * return number of pages [> 0] to be removed from locked_vm on success 296 * of "special" vmas. 297 */ 298 long mlock_vma_pages_range(struct vm_area_struct *vma, 299 unsigned long start, unsigned long end) 300 { 301 int nr_pages = (end - start) / PAGE_SIZE; 302 BUG_ON(!(vma->vm_flags & VM_LOCKED)); 303 304 /* 305 * filter unlockable vmas 306 */ 307 if (vma->vm_flags & (VM_IO | VM_PFNMAP)) 308 goto no_mlock; 309 310 if (!((vma->vm_flags & (VM_DONTEXPAND | VM_RESERVED)) || 311 is_vm_hugetlb_page(vma) || 312 vma == get_gate_vma(current))) { 313 314 __mlock_vma_pages_range(vma, start, end, 1); 315 316 /* Hide errors from mmap() and other callers */ 317 return 0; 318 } 319 320 /* 321 * User mapped kernel pages or huge pages: 322 * make these pages present to populate the ptes, but 323 * fall thru' to reset VM_LOCKED--no need to unlock, and 324 * return nr_pages so these don't get counted against task's 325 * locked limit. huge pages are already counted against 326 * locked vm limit. 327 */ 328 make_pages_present(start, end); 329 330 no_mlock: 331 vma->vm_flags &= ~VM_LOCKED; /* and don't come back! */ 332 return nr_pages; /* error or pages NOT mlocked */ 333 } 334 335 336 /* 337 * munlock_vma_pages_range() - munlock all pages in the vma range.' 338 * @vma - vma containing range to be munlock()ed. 339 * @start - start address in @vma of the range 340 * @end - end of range in @vma. 341 * 342 * For mremap(), munmap() and exit(). 343 * 344 * Called with @vma VM_LOCKED. 345 * 346 * Returns with VM_LOCKED cleared. Callers must be prepared to 347 * deal with this. 348 * 349 * We don't save and restore VM_LOCKED here because pages are 350 * still on lru. In unmap path, pages might be scanned by reclaim 351 * and re-mlocked by try_to_{munlock|unmap} before we unmap and 352 * free them. This will result in freeing mlocked pages. 353 */ 354 void munlock_vma_pages_range(struct vm_area_struct *vma, 355 unsigned long start, unsigned long end) 356 { 357 vma->vm_flags &= ~VM_LOCKED; 358 __mlock_vma_pages_range(vma, start, end, 0); 359 } 360 361 /* 362 * mlock_fixup - handle mlock[all]/munlock[all] requests. 363 * 364 * Filters out "special" vmas -- VM_LOCKED never gets set for these, and 365 * munlock is a no-op. However, for some special vmas, we go ahead and 366 * populate the ptes via make_pages_present(). 367 * 368 * For vmas that pass the filters, merge/split as appropriate. 369 */ 370 static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev, 371 unsigned long start, unsigned long end, unsigned int newflags) 372 { 373 struct mm_struct *mm = vma->vm_mm; 374 pgoff_t pgoff; 375 int nr_pages; 376 int ret = 0; 377 int lock = newflags & VM_LOCKED; 378 379 if (newflags == vma->vm_flags || 380 (vma->vm_flags & (VM_IO | VM_PFNMAP))) 381 goto out; /* don't set VM_LOCKED, don't count */ 382 383 if ((vma->vm_flags & (VM_DONTEXPAND | VM_RESERVED)) || 384 is_vm_hugetlb_page(vma) || 385 vma == get_gate_vma(current)) { 386 if (lock) 387 make_pages_present(start, end); 388 goto out; /* don't set VM_LOCKED, don't count */ 389 } 390 391 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); 392 *prev = vma_merge(mm, *prev, start, end, newflags, vma->anon_vma, 393 vma->vm_file, pgoff, vma_policy(vma)); 394 if (*prev) { 395 vma = *prev; 396 goto success; 397 } 398 399 if (start != vma->vm_start) { 400 ret = split_vma(mm, vma, start, 1); 401 if (ret) 402 goto out; 403 } 404 405 if (end != vma->vm_end) { 406 ret = split_vma(mm, vma, end, 0); 407 if (ret) 408 goto out; 409 } 410 411 success: 412 /* 413 * Keep track of amount of locked VM. 414 */ 415 nr_pages = (end - start) >> PAGE_SHIFT; 416 if (!lock) 417 nr_pages = -nr_pages; 418 mm->locked_vm += nr_pages; 419 420 /* 421 * vm_flags is protected by the mmap_sem held in write mode. 422 * It's okay if try_to_unmap_one unmaps a page just after we 423 * set VM_LOCKED, __mlock_vma_pages_range will bring it back. 424 */ 425 vma->vm_flags = newflags; 426 427 if (lock) { 428 ret = __mlock_vma_pages_range(vma, start, end, 1); 429 430 if (ret > 0) { 431 mm->locked_vm -= ret; 432 ret = 0; 433 } else 434 ret = __mlock_posix_error_return(ret); /* translate if needed */ 435 } else { 436 __mlock_vma_pages_range(vma, start, end, 0); 437 } 438 439 out: 440 *prev = vma; 441 return ret; 442 } 443 444 static int do_mlock(unsigned long start, size_t len, int on) 445 { 446 unsigned long nstart, end, tmp; 447 struct vm_area_struct * vma, * prev; 448 int error; 449 450 len = PAGE_ALIGN(len); 451 end = start + len; 452 if (end < start) 453 return -EINVAL; 454 if (end == start) 455 return 0; 456 vma = find_vma_prev(current->mm, start, &prev); 457 if (!vma || vma->vm_start > start) 458 return -ENOMEM; 459 460 if (start > vma->vm_start) 461 prev = vma; 462 463 for (nstart = start ; ; ) { 464 unsigned int newflags; 465 466 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */ 467 468 newflags = vma->vm_flags | VM_LOCKED; 469 if (!on) 470 newflags &= ~VM_LOCKED; 471 472 tmp = vma->vm_end; 473 if (tmp > end) 474 tmp = end; 475 error = mlock_fixup(vma, &prev, nstart, tmp, newflags); 476 if (error) 477 break; 478 nstart = tmp; 479 if (nstart < prev->vm_end) 480 nstart = prev->vm_end; 481 if (nstart >= end) 482 break; 483 484 vma = prev->vm_next; 485 if (!vma || vma->vm_start != nstart) { 486 error = -ENOMEM; 487 break; 488 } 489 } 490 return error; 491 } 492 493 SYSCALL_DEFINE2(mlock, unsigned long, start, size_t, len) 494 { 495 unsigned long locked; 496 unsigned long lock_limit; 497 int error = -ENOMEM; 498 499 if (!can_do_mlock()) 500 return -EPERM; 501 502 lru_add_drain_all(); /* flush pagevec */ 503 504 down_write(¤t->mm->mmap_sem); 505 len = PAGE_ALIGN(len + (start & ~PAGE_MASK)); 506 start &= PAGE_MASK; 507 508 locked = len >> PAGE_SHIFT; 509 locked += current->mm->locked_vm; 510 511 lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur; 512 lock_limit >>= PAGE_SHIFT; 513 514 /* check against resource limits */ 515 if ((locked <= lock_limit) || capable(CAP_IPC_LOCK)) 516 error = do_mlock(start, len, 1); 517 up_write(¤t->mm->mmap_sem); 518 return error; 519 } 520 521 SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) 522 { 523 int ret; 524 525 down_write(¤t->mm->mmap_sem); 526 len = PAGE_ALIGN(len + (start & ~PAGE_MASK)); 527 start &= PAGE_MASK; 528 ret = do_mlock(start, len, 0); 529 up_write(¤t->mm->mmap_sem); 530 return ret; 531 } 532 533 static int do_mlockall(int flags) 534 { 535 struct vm_area_struct * vma, * prev = NULL; 536 unsigned int def_flags = 0; 537 538 if (flags & MCL_FUTURE) 539 def_flags = VM_LOCKED; 540 current->mm->def_flags = def_flags; 541 if (flags == MCL_FUTURE) 542 goto out; 543 544 for (vma = current->mm->mmap; vma ; vma = prev->vm_next) { 545 unsigned int newflags; 546 547 newflags = vma->vm_flags | VM_LOCKED; 548 if (!(flags & MCL_CURRENT)) 549 newflags &= ~VM_LOCKED; 550 551 /* Ignore errors */ 552 mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags); 553 } 554 out: 555 return 0; 556 } 557 558 SYSCALL_DEFINE1(mlockall, int, flags) 559 { 560 unsigned long lock_limit; 561 int ret = -EINVAL; 562 563 if (!flags || (flags & ~(MCL_CURRENT | MCL_FUTURE))) 564 goto out; 565 566 ret = -EPERM; 567 if (!can_do_mlock()) 568 goto out; 569 570 lru_add_drain_all(); /* flush pagevec */ 571 572 down_write(¤t->mm->mmap_sem); 573 574 lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur; 575 lock_limit >>= PAGE_SHIFT; 576 577 ret = -ENOMEM; 578 if (!(flags & MCL_CURRENT) || (current->mm->total_vm <= lock_limit) || 579 capable(CAP_IPC_LOCK)) 580 ret = do_mlockall(flags); 581 up_write(¤t->mm->mmap_sem); 582 out: 583 return ret; 584 } 585 586 SYSCALL_DEFINE0(munlockall) 587 { 588 int ret; 589 590 down_write(¤t->mm->mmap_sem); 591 ret = do_mlockall(0); 592 up_write(¤t->mm->mmap_sem); 593 return ret; 594 } 595 596 /* 597 * Objects with different lifetime than processes (SHM_LOCK and SHM_HUGETLB 598 * shm segments) get accounted against the user_struct instead. 599 */ 600 static DEFINE_SPINLOCK(shmlock_user_lock); 601 602 int user_shm_lock(size_t size, struct user_struct *user) 603 { 604 unsigned long lock_limit, locked; 605 int allowed = 0; 606 607 locked = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 608 lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur; 609 if (lock_limit == RLIM_INFINITY) 610 allowed = 1; 611 lock_limit >>= PAGE_SHIFT; 612 spin_lock(&shmlock_user_lock); 613 if (!allowed && 614 locked + user->locked_shm > lock_limit && !capable(CAP_IPC_LOCK)) 615 goto out; 616 get_uid(user); 617 user->locked_shm += locked; 618 allowed = 1; 619 out: 620 spin_unlock(&shmlock_user_lock); 621 return allowed; 622 } 623 624 void user_shm_unlock(size_t size, struct user_struct *user) 625 { 626 spin_lock(&shmlock_user_lock); 627 user->locked_shm -= (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 628 spin_unlock(&shmlock_user_lock); 629 free_uid(user); 630 } 631 632 void *alloc_locked_buffer(size_t size) 633 { 634 unsigned long rlim, vm, pgsz; 635 void *buffer = NULL; 636 637 pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT; 638 639 down_write(¤t->mm->mmap_sem); 640 641 rlim = current->signal->rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT; 642 vm = current->mm->total_vm + pgsz; 643 if (rlim < vm) 644 goto out; 645 646 rlim = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT; 647 vm = current->mm->locked_vm + pgsz; 648 if (rlim < vm) 649 goto out; 650 651 buffer = kzalloc(size, GFP_KERNEL); 652 if (!buffer) 653 goto out; 654 655 current->mm->total_vm += pgsz; 656 current->mm->locked_vm += pgsz; 657 658 out: 659 up_write(¤t->mm->mmap_sem); 660 return buffer; 661 } 662 663 void release_locked_buffer(void *buffer, size_t size) 664 { 665 unsigned long pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT; 666 667 down_write(¤t->mm->mmap_sem); 668 669 current->mm->total_vm -= pgsz; 670 current->mm->locked_vm -= pgsz; 671 672 up_write(¤t->mm->mmap_sem); 673 } 674 675 void free_locked_buffer(void *buffer, size_t size) 676 { 677 release_locked_buffer(buffer, size); 678 679 kfree(buffer); 680 } 681