1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * mm/mprotect.c 4 * 5 * (C) Copyright 1994 Linus Torvalds 6 * (C) Copyright 2002 Christoph Hellwig 7 * 8 * Address space accounting code <alan@lxorguk.ukuu.org.uk> 9 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved 10 */ 11 12 #include <linux/mm.h> 13 #include <linux/hugetlb.h> 14 #include <linux/shm.h> 15 #include <linux/mman.h> 16 #include <linux/fs.h> 17 #include <linux/highmem.h> 18 #include <linux/security.h> 19 #include <linux/mempolicy.h> 20 #include <linux/personality.h> 21 #include <linux/syscalls.h> 22 #include <linux/swap.h> 23 #include <linux/swapops.h> 24 #include <linux/mmu_notifier.h> 25 #include <linux/migrate.h> 26 #include <linux/perf_event.h> 27 #include <linux/pkeys.h> 28 #include <linux/ksm.h> 29 #include <linux/uaccess.h> 30 #include <linux/mm_inline.h> 31 #include <asm/pgtable.h> 32 #include <asm/cacheflush.h> 33 #include <asm/mmu_context.h> 34 #include <asm/tlbflush.h> 35 36 #include "internal.h" 37 38 static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, 39 unsigned long addr, unsigned long end, pgprot_t newprot, 40 int dirty_accountable, int prot_numa) 41 { 42 struct mm_struct *mm = vma->vm_mm; 43 pte_t *pte, oldpte; 44 spinlock_t *ptl; 45 unsigned long pages = 0; 46 int target_node = NUMA_NO_NODE; 47 48 /* 49 * Can be called with only the mmap_sem for reading by 50 * prot_numa so we must check the pmd isn't constantly 51 * changing from under us from pmd_none to pmd_trans_huge 52 * and/or the other way around. 53 */ 54 if (pmd_trans_unstable(pmd)) 55 return 0; 56 57 /* 58 * The pmd points to a regular pte so the pmd can't change 59 * from under us even if the mmap_sem is only hold for 60 * reading. 61 */ 62 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 63 64 /* Get target node for single threaded private VMAs */ 65 if (prot_numa && !(vma->vm_flags & VM_SHARED) && 66 atomic_read(&vma->vm_mm->mm_users) == 1) 67 target_node = numa_node_id(); 68 69 flush_tlb_batched_pending(vma->vm_mm); 70 arch_enter_lazy_mmu_mode(); 71 do { 72 oldpte = *pte; 73 if (pte_present(oldpte)) { 74 pte_t ptent; 75 bool preserve_write = prot_numa && pte_write(oldpte); 76 77 /* 78 * Avoid trapping faults against the zero or KSM 79 * pages. See similar comment in change_huge_pmd. 80 */ 81 if (prot_numa) { 82 struct page *page; 83 84 page = vm_normal_page(vma, addr, oldpte); 85 if (!page || PageKsm(page)) 86 continue; 87 88 /* Also skip shared copy-on-write pages */ 89 if (is_cow_mapping(vma->vm_flags) && 90 page_mapcount(page) != 1) 91 continue; 92 93 /* 94 * While migration can move some dirty pages, 95 * it cannot move them all from MIGRATE_ASYNC 96 * context. 97 */ 98 if (page_is_file_cache(page) && PageDirty(page)) 99 continue; 100 101 /* Avoid TLB flush if possible */ 102 if (pte_protnone(oldpte)) 103 continue; 104 105 /* 106 * Don't mess with PTEs if page is already on the node 107 * a single-threaded process is running on. 108 */ 109 if (target_node == page_to_nid(page)) 110 continue; 111 } 112 113 oldpte = ptep_modify_prot_start(vma, addr, pte); 114 ptent = pte_modify(oldpte, newprot); 115 if (preserve_write) 116 ptent = pte_mk_savedwrite(ptent); 117 118 /* Avoid taking write faults for known dirty pages */ 119 if (dirty_accountable && pte_dirty(ptent) && 120 (pte_soft_dirty(ptent) || 121 !(vma->vm_flags & VM_SOFTDIRTY))) { 122 ptent = pte_mkwrite(ptent); 123 } 124 ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent); 125 pages++; 126 } else if (IS_ENABLED(CONFIG_MIGRATION)) { 127 swp_entry_t entry = pte_to_swp_entry(oldpte); 128 129 if (is_write_migration_entry(entry)) { 130 pte_t newpte; 131 /* 132 * A protection check is difficult so 133 * just be safe and disable write 134 */ 135 make_migration_entry_read(&entry); 136 newpte = swp_entry_to_pte(entry); 137 if (pte_swp_soft_dirty(oldpte)) 138 newpte = pte_swp_mksoft_dirty(newpte); 139 set_pte_at(mm, addr, pte, newpte); 140 141 pages++; 142 } 143 144 if (is_write_device_private_entry(entry)) { 145 pte_t newpte; 146 147 /* 148 * We do not preserve soft-dirtiness. See 149 * copy_one_pte() for explanation. 150 */ 151 make_device_private_entry_read(&entry); 152 newpte = swp_entry_to_pte(entry); 153 set_pte_at(mm, addr, pte, newpte); 154 155 pages++; 156 } 157 } 158 } while (pte++, addr += PAGE_SIZE, addr != end); 159 arch_leave_lazy_mmu_mode(); 160 pte_unmap_unlock(pte - 1, ptl); 161 162 return pages; 163 } 164 165 static inline unsigned long change_pmd_range(struct vm_area_struct *vma, 166 pud_t *pud, unsigned long addr, unsigned long end, 167 pgprot_t newprot, int dirty_accountable, int prot_numa) 168 { 169 pmd_t *pmd; 170 unsigned long next; 171 unsigned long pages = 0; 172 unsigned long nr_huge_updates = 0; 173 struct mmu_notifier_range range; 174 175 range.start = 0; 176 177 pmd = pmd_offset(pud, addr); 178 do { 179 unsigned long this_pages; 180 181 next = pmd_addr_end(addr, end); 182 if (!is_swap_pmd(*pmd) && !pmd_trans_huge(*pmd) && !pmd_devmap(*pmd) 183 && pmd_none_or_clear_bad(pmd)) 184 goto next; 185 186 /* invoke the mmu notifier if the pmd is populated */ 187 if (!range.start) { 188 mmu_notifier_range_init(&range, vma->vm_mm, addr, end); 189 mmu_notifier_invalidate_range_start(&range); 190 } 191 192 if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) { 193 if (next - addr != HPAGE_PMD_SIZE) { 194 __split_huge_pmd(vma, pmd, addr, false, NULL); 195 } else { 196 int nr_ptes = change_huge_pmd(vma, pmd, addr, 197 newprot, prot_numa); 198 199 if (nr_ptes) { 200 if (nr_ptes == HPAGE_PMD_NR) { 201 pages += HPAGE_PMD_NR; 202 nr_huge_updates++; 203 } 204 205 /* huge pmd was handled */ 206 goto next; 207 } 208 } 209 /* fall through, the trans huge pmd just split */ 210 } 211 this_pages = change_pte_range(vma, pmd, addr, next, newprot, 212 dirty_accountable, prot_numa); 213 pages += this_pages; 214 next: 215 cond_resched(); 216 } while (pmd++, addr = next, addr != end); 217 218 if (range.start) 219 mmu_notifier_invalidate_range_end(&range); 220 221 if (nr_huge_updates) 222 count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates); 223 return pages; 224 } 225 226 static inline unsigned long change_pud_range(struct vm_area_struct *vma, 227 p4d_t *p4d, unsigned long addr, unsigned long end, 228 pgprot_t newprot, int dirty_accountable, int prot_numa) 229 { 230 pud_t *pud; 231 unsigned long next; 232 unsigned long pages = 0; 233 234 pud = pud_offset(p4d, addr); 235 do { 236 next = pud_addr_end(addr, end); 237 if (pud_none_or_clear_bad(pud)) 238 continue; 239 pages += change_pmd_range(vma, pud, addr, next, newprot, 240 dirty_accountable, prot_numa); 241 } while (pud++, addr = next, addr != end); 242 243 return pages; 244 } 245 246 static inline unsigned long change_p4d_range(struct vm_area_struct *vma, 247 pgd_t *pgd, unsigned long addr, unsigned long end, 248 pgprot_t newprot, int dirty_accountable, int prot_numa) 249 { 250 p4d_t *p4d; 251 unsigned long next; 252 unsigned long pages = 0; 253 254 p4d = p4d_offset(pgd, addr); 255 do { 256 next = p4d_addr_end(addr, end); 257 if (p4d_none_or_clear_bad(p4d)) 258 continue; 259 pages += change_pud_range(vma, p4d, addr, next, newprot, 260 dirty_accountable, prot_numa); 261 } while (p4d++, addr = next, addr != end); 262 263 return pages; 264 } 265 266 static unsigned long change_protection_range(struct vm_area_struct *vma, 267 unsigned long addr, unsigned long end, pgprot_t newprot, 268 int dirty_accountable, int prot_numa) 269 { 270 struct mm_struct *mm = vma->vm_mm; 271 pgd_t *pgd; 272 unsigned long next; 273 unsigned long start = addr; 274 unsigned long pages = 0; 275 276 BUG_ON(addr >= end); 277 pgd = pgd_offset(mm, addr); 278 flush_cache_range(vma, addr, end); 279 inc_tlb_flush_pending(mm); 280 do { 281 next = pgd_addr_end(addr, end); 282 if (pgd_none_or_clear_bad(pgd)) 283 continue; 284 pages += change_p4d_range(vma, pgd, addr, next, newprot, 285 dirty_accountable, prot_numa); 286 } while (pgd++, addr = next, addr != end); 287 288 /* Only flush the TLB if we actually modified any entries: */ 289 if (pages) 290 flush_tlb_range(vma, start, end); 291 dec_tlb_flush_pending(mm); 292 293 return pages; 294 } 295 296 unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, 297 unsigned long end, pgprot_t newprot, 298 int dirty_accountable, int prot_numa) 299 { 300 unsigned long pages; 301 302 if (is_vm_hugetlb_page(vma)) 303 pages = hugetlb_change_protection(vma, start, end, newprot); 304 else 305 pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa); 306 307 return pages; 308 } 309 310 static int prot_none_pte_entry(pte_t *pte, unsigned long addr, 311 unsigned long next, struct mm_walk *walk) 312 { 313 return pfn_modify_allowed(pte_pfn(*pte), *(pgprot_t *)(walk->private)) ? 314 0 : -EACCES; 315 } 316 317 static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask, 318 unsigned long addr, unsigned long next, 319 struct mm_walk *walk) 320 { 321 return pfn_modify_allowed(pte_pfn(*pte), *(pgprot_t *)(walk->private)) ? 322 0 : -EACCES; 323 } 324 325 static int prot_none_test(unsigned long addr, unsigned long next, 326 struct mm_walk *walk) 327 { 328 return 0; 329 } 330 331 static int prot_none_walk(struct vm_area_struct *vma, unsigned long start, 332 unsigned long end, unsigned long newflags) 333 { 334 pgprot_t new_pgprot = vm_get_page_prot(newflags); 335 struct mm_walk prot_none_walk = { 336 .pte_entry = prot_none_pte_entry, 337 .hugetlb_entry = prot_none_hugetlb_entry, 338 .test_walk = prot_none_test, 339 .mm = current->mm, 340 .private = &new_pgprot, 341 }; 342 343 return walk_page_range(start, end, &prot_none_walk); 344 } 345 346 int 347 mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, 348 unsigned long start, unsigned long end, unsigned long newflags) 349 { 350 struct mm_struct *mm = vma->vm_mm; 351 unsigned long oldflags = vma->vm_flags; 352 long nrpages = (end - start) >> PAGE_SHIFT; 353 unsigned long charged = 0; 354 pgoff_t pgoff; 355 int error; 356 int dirty_accountable = 0; 357 358 if (newflags == oldflags) { 359 *pprev = vma; 360 return 0; 361 } 362 363 /* 364 * Do PROT_NONE PFN permission checks here when we can still 365 * bail out without undoing a lot of state. This is a rather 366 * uncommon case, so doesn't need to be very optimized. 367 */ 368 if (arch_has_pfn_modify_check() && 369 (vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && 370 (newflags & (VM_READ|VM_WRITE|VM_EXEC)) == 0) { 371 error = prot_none_walk(vma, start, end, newflags); 372 if (error) 373 return error; 374 } 375 376 /* 377 * If we make a private mapping writable we increase our commit; 378 * but (without finer accounting) cannot reduce our commit if we 379 * make it unwritable again. hugetlb mapping were accounted for 380 * even if read-only so there is no need to account for them here 381 */ 382 if (newflags & VM_WRITE) { 383 /* Check space limits when area turns into data. */ 384 if (!may_expand_vm(mm, newflags, nrpages) && 385 may_expand_vm(mm, oldflags, nrpages)) 386 return -ENOMEM; 387 if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB| 388 VM_SHARED|VM_NORESERVE))) { 389 charged = nrpages; 390 if (security_vm_enough_memory_mm(mm, charged)) 391 return -ENOMEM; 392 newflags |= VM_ACCOUNT; 393 } 394 } 395 396 /* 397 * First try to merge with previous and/or next vma. 398 */ 399 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); 400 *pprev = vma_merge(mm, *pprev, start, end, newflags, 401 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), 402 vma->vm_userfaultfd_ctx); 403 if (*pprev) { 404 vma = *pprev; 405 VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY); 406 goto success; 407 } 408 409 *pprev = vma; 410 411 if (start != vma->vm_start) { 412 error = split_vma(mm, vma, start, 1); 413 if (error) 414 goto fail; 415 } 416 417 if (end != vma->vm_end) { 418 error = split_vma(mm, vma, end, 0); 419 if (error) 420 goto fail; 421 } 422 423 success: 424 /* 425 * vm_flags and vm_page_prot are protected by the mmap_sem 426 * held in write mode. 427 */ 428 vma->vm_flags = newflags; 429 dirty_accountable = vma_wants_writenotify(vma, vma->vm_page_prot); 430 vma_set_page_prot(vma); 431 432 change_protection(vma, start, end, vma->vm_page_prot, 433 dirty_accountable, 0); 434 435 /* 436 * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major 437 * fault on access. 438 */ 439 if ((oldflags & (VM_WRITE | VM_SHARED | VM_LOCKED)) == VM_LOCKED && 440 (newflags & VM_WRITE)) { 441 populate_vma_page_range(vma, start, end, NULL); 442 } 443 444 vm_stat_account(mm, oldflags, -nrpages); 445 vm_stat_account(mm, newflags, nrpages); 446 perf_event_mmap(vma); 447 return 0; 448 449 fail: 450 vm_unacct_memory(charged); 451 return error; 452 } 453 454 /* 455 * pkey==-1 when doing a legacy mprotect() 456 */ 457 static int do_mprotect_pkey(unsigned long start, size_t len, 458 unsigned long prot, int pkey) 459 { 460 unsigned long nstart, end, tmp, reqprot; 461 struct vm_area_struct *vma, *prev; 462 int error = -EINVAL; 463 const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP); 464 const bool rier = (current->personality & READ_IMPLIES_EXEC) && 465 (prot & PROT_READ); 466 467 prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP); 468 if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ 469 return -EINVAL; 470 471 if (start & ~PAGE_MASK) 472 return -EINVAL; 473 if (!len) 474 return 0; 475 len = PAGE_ALIGN(len); 476 end = start + len; 477 if (end <= start) 478 return -ENOMEM; 479 if (!arch_validate_prot(prot, start)) 480 return -EINVAL; 481 482 reqprot = prot; 483 484 if (down_write_killable(¤t->mm->mmap_sem)) 485 return -EINTR; 486 487 /* 488 * If userspace did not allocate the pkey, do not let 489 * them use it here. 490 */ 491 error = -EINVAL; 492 if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey)) 493 goto out; 494 495 vma = find_vma(current->mm, start); 496 error = -ENOMEM; 497 if (!vma) 498 goto out; 499 prev = vma->vm_prev; 500 if (unlikely(grows & PROT_GROWSDOWN)) { 501 if (vma->vm_start >= end) 502 goto out; 503 start = vma->vm_start; 504 error = -EINVAL; 505 if (!(vma->vm_flags & VM_GROWSDOWN)) 506 goto out; 507 } else { 508 if (vma->vm_start > start) 509 goto out; 510 if (unlikely(grows & PROT_GROWSUP)) { 511 end = vma->vm_end; 512 error = -EINVAL; 513 if (!(vma->vm_flags & VM_GROWSUP)) 514 goto out; 515 } 516 } 517 if (start > vma->vm_start) 518 prev = vma; 519 520 for (nstart = start ; ; ) { 521 unsigned long mask_off_old_flags; 522 unsigned long newflags; 523 int new_vma_pkey; 524 525 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */ 526 527 /* Does the application expect PROT_READ to imply PROT_EXEC */ 528 if (rier && (vma->vm_flags & VM_MAYEXEC)) 529 prot |= PROT_EXEC; 530 531 /* 532 * Each mprotect() call explicitly passes r/w/x permissions. 533 * If a permission is not passed to mprotect(), it must be 534 * cleared from the VMA. 535 */ 536 mask_off_old_flags = VM_READ | VM_WRITE | VM_EXEC | 537 VM_FLAGS_CLEAR; 538 539 new_vma_pkey = arch_override_mprotect_pkey(vma, prot, pkey); 540 newflags = calc_vm_prot_bits(prot, new_vma_pkey); 541 newflags |= (vma->vm_flags & ~mask_off_old_flags); 542 543 /* newflags >> 4 shift VM_MAY% in place of VM_% */ 544 if ((newflags & ~(newflags >> 4)) & (VM_READ | VM_WRITE | VM_EXEC)) { 545 error = -EACCES; 546 goto out; 547 } 548 549 error = security_file_mprotect(vma, reqprot, prot); 550 if (error) 551 goto out; 552 553 tmp = vma->vm_end; 554 if (tmp > end) 555 tmp = end; 556 error = mprotect_fixup(vma, &prev, nstart, tmp, newflags); 557 if (error) 558 goto out; 559 nstart = tmp; 560 561 if (nstart < prev->vm_end) 562 nstart = prev->vm_end; 563 if (nstart >= end) 564 goto out; 565 566 vma = prev->vm_next; 567 if (!vma || vma->vm_start != nstart) { 568 error = -ENOMEM; 569 goto out; 570 } 571 prot = reqprot; 572 } 573 out: 574 up_write(¤t->mm->mmap_sem); 575 return error; 576 } 577 578 SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len, 579 unsigned long, prot) 580 { 581 return do_mprotect_pkey(start, len, prot, -1); 582 } 583 584 #ifdef CONFIG_ARCH_HAS_PKEYS 585 586 SYSCALL_DEFINE4(pkey_mprotect, unsigned long, start, size_t, len, 587 unsigned long, prot, int, pkey) 588 { 589 return do_mprotect_pkey(start, len, prot, pkey); 590 } 591 592 SYSCALL_DEFINE2(pkey_alloc, unsigned long, flags, unsigned long, init_val) 593 { 594 int pkey; 595 int ret; 596 597 /* No flags supported yet. */ 598 if (flags) 599 return -EINVAL; 600 /* check for unsupported init values */ 601 if (init_val & ~PKEY_ACCESS_MASK) 602 return -EINVAL; 603 604 down_write(¤t->mm->mmap_sem); 605 pkey = mm_pkey_alloc(current->mm); 606 607 ret = -ENOSPC; 608 if (pkey == -1) 609 goto out; 610 611 ret = arch_set_user_pkey_access(current, pkey, init_val); 612 if (ret) { 613 mm_pkey_free(current->mm, pkey); 614 goto out; 615 } 616 ret = pkey; 617 out: 618 up_write(¤t->mm->mmap_sem); 619 return ret; 620 } 621 622 SYSCALL_DEFINE1(pkey_free, int, pkey) 623 { 624 int ret; 625 626 down_write(¤t->mm->mmap_sem); 627 ret = mm_pkey_free(current->mm, pkey); 628 up_write(¤t->mm->mmap_sem); 629 630 /* 631 * We could provie warnings or errors if any VMA still 632 * has the pkey set here. 633 */ 634 return ret; 635 } 636 637 #endif /* CONFIG_ARCH_HAS_PKEYS */ 638