1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/err.h> 5 #include <linux/spinlock.h> 6 7 #include <linux/mm.h> 8 #include <linux/memremap.h> 9 #include <linux/pagemap.h> 10 #include <linux/rmap.h> 11 #include <linux/swap.h> 12 #include <linux/swapops.h> 13 #include <linux/secretmem.h> 14 15 #include <linux/sched/signal.h> 16 #include <linux/rwsem.h> 17 #include <linux/hugetlb.h> 18 #include <linux/migrate.h> 19 #include <linux/mm_inline.h> 20 #include <linux/sched/mm.h> 21 #include <linux/shmem_fs.h> 22 23 #include <asm/mmu_context.h> 24 #include <asm/tlbflush.h> 25 26 #include "internal.h" 27 28 struct follow_page_context { 29 struct dev_pagemap *pgmap; 30 unsigned int page_mask; 31 }; 32 33 static inline void sanity_check_pinned_pages(struct page **pages, 34 unsigned long npages) 35 { 36 if (!IS_ENABLED(CONFIG_DEBUG_VM)) 37 return; 38 39 /* 40 * We only pin anonymous pages if they are exclusive. Once pinned, we 41 * can no longer turn them possibly shared and PageAnonExclusive() will 42 * stick around until the page is freed. 43 * 44 * We'd like to verify that our pinned anonymous pages are still mapped 45 * exclusively. The issue with anon THP is that we don't know how 46 * they are/were mapped when pinning them. However, for anon 47 * THP we can assume that either the given page (PTE-mapped THP) or 48 * the head page (PMD-mapped THP) should be PageAnonExclusive(). If 49 * neither is the case, there is certainly something wrong. 50 */ 51 for (; npages; npages--, pages++) { 52 struct page *page = *pages; 53 struct folio *folio = page_folio(page); 54 55 if (is_zero_page(page) || 56 !folio_test_anon(folio)) 57 continue; 58 if (!folio_test_large(folio) || folio_test_hugetlb(folio)) 59 VM_BUG_ON_PAGE(!PageAnonExclusive(&folio->page), page); 60 else 61 /* Either a PTE-mapped or a PMD-mapped THP. */ 62 VM_BUG_ON_PAGE(!PageAnonExclusive(&folio->page) && 63 !PageAnonExclusive(page), page); 64 } 65 } 66 67 /* 68 * Return the folio with ref appropriately incremented, 69 * or NULL if that failed. 70 */ 71 static inline struct folio *try_get_folio(struct page *page, int refs) 72 { 73 struct folio *folio; 74 75 retry: 76 folio = page_folio(page); 77 if (WARN_ON_ONCE(folio_ref_count(folio) < 0)) 78 return NULL; 79 if (unlikely(!folio_ref_try_add_rcu(folio, refs))) 80 return NULL; 81 82 /* 83 * At this point we have a stable reference to the folio; but it 84 * could be that between calling page_folio() and the refcount 85 * increment, the folio was split, in which case we'd end up 86 * holding a reference on a folio that has nothing to do with the page 87 * we were given anymore. 88 * So now that the folio is stable, recheck that the page still 89 * belongs to this folio. 90 */ 91 if (unlikely(page_folio(page) != folio)) { 92 if (!put_devmap_managed_page_refs(&folio->page, refs)) 93 folio_put_refs(folio, refs); 94 goto retry; 95 } 96 97 return folio; 98 } 99 100 /** 101 * try_grab_folio() - Attempt to get or pin a folio. 102 * @page: pointer to page to be grabbed 103 * @refs: the value to (effectively) add to the folio's refcount 104 * @flags: gup flags: these are the FOLL_* flag values. 105 * 106 * "grab" names in this file mean, "look at flags to decide whether to use 107 * FOLL_PIN or FOLL_GET behavior, when incrementing the folio's refcount. 108 * 109 * Either FOLL_PIN or FOLL_GET (or neither) must be set, but not both at the 110 * same time. (That's true throughout the get_user_pages*() and 111 * pin_user_pages*() APIs.) Cases: 112 * 113 * FOLL_GET: folio's refcount will be incremented by @refs. 114 * 115 * FOLL_PIN on large folios: folio's refcount will be incremented by 116 * @refs, and its pincount will be incremented by @refs. 117 * 118 * FOLL_PIN on single-page folios: folio's refcount will be incremented by 119 * @refs * GUP_PIN_COUNTING_BIAS. 120 * 121 * Return: The folio containing @page (with refcount appropriately 122 * incremented) for success, or NULL upon failure. If neither FOLL_GET 123 * nor FOLL_PIN was set, that's considered failure, and furthermore, 124 * a likely bug in the caller, so a warning is also emitted. 125 */ 126 struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags) 127 { 128 struct folio *folio; 129 130 if (WARN_ON_ONCE((flags & (FOLL_GET | FOLL_PIN)) == 0)) 131 return NULL; 132 133 if (unlikely(!(flags & FOLL_PCI_P2PDMA) && is_pci_p2pdma_page(page))) 134 return NULL; 135 136 if (flags & FOLL_GET) 137 return try_get_folio(page, refs); 138 139 /* FOLL_PIN is set */ 140 141 /* 142 * Don't take a pin on the zero page - it's not going anywhere 143 * and it is used in a *lot* of places. 144 */ 145 if (is_zero_page(page)) 146 return page_folio(page); 147 148 folio = try_get_folio(page, refs); 149 if (!folio) 150 return NULL; 151 152 /* 153 * Can't do FOLL_LONGTERM + FOLL_PIN gup fast path if not in a 154 * right zone, so fail and let the caller fall back to the slow 155 * path. 156 */ 157 if (unlikely((flags & FOLL_LONGTERM) && 158 !folio_is_longterm_pinnable(folio))) { 159 if (!put_devmap_managed_page_refs(&folio->page, refs)) 160 folio_put_refs(folio, refs); 161 return NULL; 162 } 163 164 /* 165 * When pinning a large folio, use an exact count to track it. 166 * 167 * However, be sure to *also* increment the normal folio 168 * refcount field at least once, so that the folio really 169 * is pinned. That's why the refcount from the earlier 170 * try_get_folio() is left intact. 171 */ 172 if (folio_test_large(folio)) 173 atomic_add(refs, &folio->_pincount); 174 else 175 folio_ref_add(folio, 176 refs * (GUP_PIN_COUNTING_BIAS - 1)); 177 /* 178 * Adjust the pincount before re-checking the PTE for changes. 179 * This is essentially a smp_mb() and is paired with a memory 180 * barrier in folio_try_share_anon_rmap_*(). 181 */ 182 smp_mb__after_atomic(); 183 184 node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, refs); 185 186 return folio; 187 } 188 189 static void gup_put_folio(struct folio *folio, int refs, unsigned int flags) 190 { 191 if (flags & FOLL_PIN) { 192 if (is_zero_folio(folio)) 193 return; 194 node_stat_mod_folio(folio, NR_FOLL_PIN_RELEASED, refs); 195 if (folio_test_large(folio)) 196 atomic_sub(refs, &folio->_pincount); 197 else 198 refs *= GUP_PIN_COUNTING_BIAS; 199 } 200 201 if (!put_devmap_managed_page_refs(&folio->page, refs)) 202 folio_put_refs(folio, refs); 203 } 204 205 /** 206 * try_grab_page() - elevate a page's refcount by a flag-dependent amount 207 * @page: pointer to page to be grabbed 208 * @flags: gup flags: these are the FOLL_* flag values. 209 * 210 * This might not do anything at all, depending on the flags argument. 211 * 212 * "grab" names in this file mean, "look at flags to decide whether to use 213 * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount. 214 * 215 * Either FOLL_PIN or FOLL_GET (or neither) may be set, but not both at the same 216 * time. Cases: please see the try_grab_folio() documentation, with 217 * "refs=1". 218 * 219 * Return: 0 for success, or if no action was required (if neither FOLL_PIN 220 * nor FOLL_GET was set, nothing is done). A negative error code for failure: 221 * 222 * -ENOMEM FOLL_GET or FOLL_PIN was set, but the page could not 223 * be grabbed. 224 */ 225 int __must_check try_grab_page(struct page *page, unsigned int flags) 226 { 227 struct folio *folio = page_folio(page); 228 229 if (WARN_ON_ONCE(folio_ref_count(folio) <= 0)) 230 return -ENOMEM; 231 232 if (unlikely(!(flags & FOLL_PCI_P2PDMA) && is_pci_p2pdma_page(page))) 233 return -EREMOTEIO; 234 235 if (flags & FOLL_GET) 236 folio_ref_inc(folio); 237 else if (flags & FOLL_PIN) { 238 /* 239 * Don't take a pin on the zero page - it's not going anywhere 240 * and it is used in a *lot* of places. 241 */ 242 if (is_zero_page(page)) 243 return 0; 244 245 /* 246 * Similar to try_grab_folio(): be sure to *also* 247 * increment the normal page refcount field at least once, 248 * so that the page really is pinned. 249 */ 250 if (folio_test_large(folio)) { 251 folio_ref_add(folio, 1); 252 atomic_add(1, &folio->_pincount); 253 } else { 254 folio_ref_add(folio, GUP_PIN_COUNTING_BIAS); 255 } 256 257 node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, 1); 258 } 259 260 return 0; 261 } 262 263 /** 264 * unpin_user_page() - release a dma-pinned page 265 * @page: pointer to page to be released 266 * 267 * Pages that were pinned via pin_user_pages*() must be released via either 268 * unpin_user_page(), or one of the unpin_user_pages*() routines. This is so 269 * that such pages can be separately tracked and uniquely handled. In 270 * particular, interactions with RDMA and filesystems need special handling. 271 */ 272 void unpin_user_page(struct page *page) 273 { 274 sanity_check_pinned_pages(&page, 1); 275 gup_put_folio(page_folio(page), 1, FOLL_PIN); 276 } 277 EXPORT_SYMBOL(unpin_user_page); 278 279 /** 280 * folio_add_pin - Try to get an additional pin on a pinned folio 281 * @folio: The folio to be pinned 282 * 283 * Get an additional pin on a folio we already have a pin on. Makes no change 284 * if the folio is a zero_page. 285 */ 286 void folio_add_pin(struct folio *folio) 287 { 288 if (is_zero_folio(folio)) 289 return; 290 291 /* 292 * Similar to try_grab_folio(): be sure to *also* increment the normal 293 * page refcount field at least once, so that the page really is 294 * pinned. 295 */ 296 if (folio_test_large(folio)) { 297 WARN_ON_ONCE(atomic_read(&folio->_pincount) < 1); 298 folio_ref_inc(folio); 299 atomic_inc(&folio->_pincount); 300 } else { 301 WARN_ON_ONCE(folio_ref_count(folio) < GUP_PIN_COUNTING_BIAS); 302 folio_ref_add(folio, GUP_PIN_COUNTING_BIAS); 303 } 304 } 305 306 static inline struct folio *gup_folio_range_next(struct page *start, 307 unsigned long npages, unsigned long i, unsigned int *ntails) 308 { 309 struct page *next = nth_page(start, i); 310 struct folio *folio = page_folio(next); 311 unsigned int nr = 1; 312 313 if (folio_test_large(folio)) 314 nr = min_t(unsigned int, npages - i, 315 folio_nr_pages(folio) - folio_page_idx(folio, next)); 316 317 *ntails = nr; 318 return folio; 319 } 320 321 static inline struct folio *gup_folio_next(struct page **list, 322 unsigned long npages, unsigned long i, unsigned int *ntails) 323 { 324 struct folio *folio = page_folio(list[i]); 325 unsigned int nr; 326 327 for (nr = i + 1; nr < npages; nr++) { 328 if (page_folio(list[nr]) != folio) 329 break; 330 } 331 332 *ntails = nr - i; 333 return folio; 334 } 335 336 /** 337 * unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages 338 * @pages: array of pages to be maybe marked dirty, and definitely released. 339 * @npages: number of pages in the @pages array. 340 * @make_dirty: whether to mark the pages dirty 341 * 342 * "gup-pinned page" refers to a page that has had one of the get_user_pages() 343 * variants called on that page. 344 * 345 * For each page in the @pages array, make that page (or its head page, if a 346 * compound page) dirty, if @make_dirty is true, and if the page was previously 347 * listed as clean. In any case, releases all pages using unpin_user_page(), 348 * possibly via unpin_user_pages(), for the non-dirty case. 349 * 350 * Please see the unpin_user_page() documentation for details. 351 * 352 * set_page_dirty_lock() is used internally. If instead, set_page_dirty() is 353 * required, then the caller should a) verify that this is really correct, 354 * because _lock() is usually required, and b) hand code it: 355 * set_page_dirty_lock(), unpin_user_page(). 356 * 357 */ 358 void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages, 359 bool make_dirty) 360 { 361 unsigned long i; 362 struct folio *folio; 363 unsigned int nr; 364 365 if (!make_dirty) { 366 unpin_user_pages(pages, npages); 367 return; 368 } 369 370 sanity_check_pinned_pages(pages, npages); 371 for (i = 0; i < npages; i += nr) { 372 folio = gup_folio_next(pages, npages, i, &nr); 373 /* 374 * Checking PageDirty at this point may race with 375 * clear_page_dirty_for_io(), but that's OK. Two key 376 * cases: 377 * 378 * 1) This code sees the page as already dirty, so it 379 * skips the call to set_page_dirty(). That could happen 380 * because clear_page_dirty_for_io() called 381 * page_mkclean(), followed by set_page_dirty(). 382 * However, now the page is going to get written back, 383 * which meets the original intention of setting it 384 * dirty, so all is well: clear_page_dirty_for_io() goes 385 * on to call TestClearPageDirty(), and write the page 386 * back. 387 * 388 * 2) This code sees the page as clean, so it calls 389 * set_page_dirty(). The page stays dirty, despite being 390 * written back, so it gets written back again in the 391 * next writeback cycle. This is harmless. 392 */ 393 if (!folio_test_dirty(folio)) { 394 folio_lock(folio); 395 folio_mark_dirty(folio); 396 folio_unlock(folio); 397 } 398 gup_put_folio(folio, nr, FOLL_PIN); 399 } 400 } 401 EXPORT_SYMBOL(unpin_user_pages_dirty_lock); 402 403 /** 404 * unpin_user_page_range_dirty_lock() - release and optionally dirty 405 * gup-pinned page range 406 * 407 * @page: the starting page of a range maybe marked dirty, and definitely released. 408 * @npages: number of consecutive pages to release. 409 * @make_dirty: whether to mark the pages dirty 410 * 411 * "gup-pinned page range" refers to a range of pages that has had one of the 412 * pin_user_pages() variants called on that page. 413 * 414 * For the page ranges defined by [page .. page+npages], make that range (or 415 * its head pages, if a compound page) dirty, if @make_dirty is true, and if the 416 * page range was previously listed as clean. 417 * 418 * set_page_dirty_lock() is used internally. If instead, set_page_dirty() is 419 * required, then the caller should a) verify that this is really correct, 420 * because _lock() is usually required, and b) hand code it: 421 * set_page_dirty_lock(), unpin_user_page(). 422 * 423 */ 424 void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages, 425 bool make_dirty) 426 { 427 unsigned long i; 428 struct folio *folio; 429 unsigned int nr; 430 431 for (i = 0; i < npages; i += nr) { 432 folio = gup_folio_range_next(page, npages, i, &nr); 433 if (make_dirty && !folio_test_dirty(folio)) { 434 folio_lock(folio); 435 folio_mark_dirty(folio); 436 folio_unlock(folio); 437 } 438 gup_put_folio(folio, nr, FOLL_PIN); 439 } 440 } 441 EXPORT_SYMBOL(unpin_user_page_range_dirty_lock); 442 443 static void unpin_user_pages_lockless(struct page **pages, unsigned long npages) 444 { 445 unsigned long i; 446 struct folio *folio; 447 unsigned int nr; 448 449 /* 450 * Don't perform any sanity checks because we might have raced with 451 * fork() and some anonymous pages might now actually be shared -- 452 * which is why we're unpinning after all. 453 */ 454 for (i = 0; i < npages; i += nr) { 455 folio = gup_folio_next(pages, npages, i, &nr); 456 gup_put_folio(folio, nr, FOLL_PIN); 457 } 458 } 459 460 /** 461 * unpin_user_pages() - release an array of gup-pinned pages. 462 * @pages: array of pages to be marked dirty and released. 463 * @npages: number of pages in the @pages array. 464 * 465 * For each page in the @pages array, release the page using unpin_user_page(). 466 * 467 * Please see the unpin_user_page() documentation for details. 468 */ 469 void unpin_user_pages(struct page **pages, unsigned long npages) 470 { 471 unsigned long i; 472 struct folio *folio; 473 unsigned int nr; 474 475 /* 476 * If this WARN_ON() fires, then the system *might* be leaking pages (by 477 * leaving them pinned), but probably not. More likely, gup/pup returned 478 * a hard -ERRNO error to the caller, who erroneously passed it here. 479 */ 480 if (WARN_ON(IS_ERR_VALUE(npages))) 481 return; 482 483 sanity_check_pinned_pages(pages, npages); 484 for (i = 0; i < npages; i += nr) { 485 folio = gup_folio_next(pages, npages, i, &nr); 486 gup_put_folio(folio, nr, FOLL_PIN); 487 } 488 } 489 EXPORT_SYMBOL(unpin_user_pages); 490 491 /* 492 * Set the MMF_HAS_PINNED if not set yet; after set it'll be there for the mm's 493 * lifecycle. Avoid setting the bit unless necessary, or it might cause write 494 * cache bouncing on large SMP machines for concurrent pinned gups. 495 */ 496 static inline void mm_set_has_pinned_flag(unsigned long *mm_flags) 497 { 498 if (!test_bit(MMF_HAS_PINNED, mm_flags)) 499 set_bit(MMF_HAS_PINNED, mm_flags); 500 } 501 502 #ifdef CONFIG_MMU 503 static struct page *no_page_table(struct vm_area_struct *vma, 504 unsigned int flags) 505 { 506 /* 507 * When core dumping an enormous anonymous area that nobody 508 * has touched so far, we don't want to allocate unnecessary pages or 509 * page tables. Return error instead of NULL to skip handle_mm_fault, 510 * then get_dump_page() will return NULL to leave a hole in the dump. 511 * But we can only make this optimization where a hole would surely 512 * be zero-filled if handle_mm_fault() actually did handle it. 513 */ 514 if ((flags & FOLL_DUMP) && 515 (vma_is_anonymous(vma) || !vma->vm_ops->fault)) 516 return ERR_PTR(-EFAULT); 517 return NULL; 518 } 519 520 static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address, 521 pte_t *pte, unsigned int flags) 522 { 523 if (flags & FOLL_TOUCH) { 524 pte_t orig_entry = ptep_get(pte); 525 pte_t entry = orig_entry; 526 527 if (flags & FOLL_WRITE) 528 entry = pte_mkdirty(entry); 529 entry = pte_mkyoung(entry); 530 531 if (!pte_same(orig_entry, entry)) { 532 set_pte_at(vma->vm_mm, address, pte, entry); 533 update_mmu_cache(vma, address, pte); 534 } 535 } 536 537 /* Proper page table entry exists, but no corresponding struct page */ 538 return -EEXIST; 539 } 540 541 /* FOLL_FORCE can write to even unwritable PTEs in COW mappings. */ 542 static inline bool can_follow_write_pte(pte_t pte, struct page *page, 543 struct vm_area_struct *vma, 544 unsigned int flags) 545 { 546 /* If the pte is writable, we can write to the page. */ 547 if (pte_write(pte)) 548 return true; 549 550 /* Maybe FOLL_FORCE is set to override it? */ 551 if (!(flags & FOLL_FORCE)) 552 return false; 553 554 /* But FOLL_FORCE has no effect on shared mappings */ 555 if (vma->vm_flags & (VM_MAYSHARE | VM_SHARED)) 556 return false; 557 558 /* ... or read-only private ones */ 559 if (!(vma->vm_flags & VM_MAYWRITE)) 560 return false; 561 562 /* ... or already writable ones that just need to take a write fault */ 563 if (vma->vm_flags & VM_WRITE) 564 return false; 565 566 /* 567 * See can_change_pte_writable(): we broke COW and could map the page 568 * writable if we have an exclusive anonymous page ... 569 */ 570 if (!page || !PageAnon(page) || !PageAnonExclusive(page)) 571 return false; 572 573 /* ... and a write-fault isn't required for other reasons. */ 574 if (vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte)) 575 return false; 576 return !userfaultfd_pte_wp(vma, pte); 577 } 578 579 static struct page *follow_page_pte(struct vm_area_struct *vma, 580 unsigned long address, pmd_t *pmd, unsigned int flags, 581 struct dev_pagemap **pgmap) 582 { 583 struct mm_struct *mm = vma->vm_mm; 584 struct page *page; 585 spinlock_t *ptl; 586 pte_t *ptep, pte; 587 int ret; 588 589 /* FOLL_GET and FOLL_PIN are mutually exclusive. */ 590 if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) == 591 (FOLL_PIN | FOLL_GET))) 592 return ERR_PTR(-EINVAL); 593 594 ptep = pte_offset_map_lock(mm, pmd, address, &ptl); 595 if (!ptep) 596 return no_page_table(vma, flags); 597 pte = ptep_get(ptep); 598 if (!pte_present(pte)) 599 goto no_page; 600 if (pte_protnone(pte) && !gup_can_follow_protnone(vma, flags)) 601 goto no_page; 602 603 page = vm_normal_page(vma, address, pte); 604 605 /* 606 * We only care about anon pages in can_follow_write_pte() and don't 607 * have to worry about pte_devmap() because they are never anon. 608 */ 609 if ((flags & FOLL_WRITE) && 610 !can_follow_write_pte(pte, page, vma, flags)) { 611 page = NULL; 612 goto out; 613 } 614 615 if (!page && pte_devmap(pte) && (flags & (FOLL_GET | FOLL_PIN))) { 616 /* 617 * Only return device mapping pages in the FOLL_GET or FOLL_PIN 618 * case since they are only valid while holding the pgmap 619 * reference. 620 */ 621 *pgmap = get_dev_pagemap(pte_pfn(pte), *pgmap); 622 if (*pgmap) 623 page = pte_page(pte); 624 else 625 goto no_page; 626 } else if (unlikely(!page)) { 627 if (flags & FOLL_DUMP) { 628 /* Avoid special (like zero) pages in core dumps */ 629 page = ERR_PTR(-EFAULT); 630 goto out; 631 } 632 633 if (is_zero_pfn(pte_pfn(pte))) { 634 page = pte_page(pte); 635 } else { 636 ret = follow_pfn_pte(vma, address, ptep, flags); 637 page = ERR_PTR(ret); 638 goto out; 639 } 640 } 641 642 if (!pte_write(pte) && gup_must_unshare(vma, flags, page)) { 643 page = ERR_PTR(-EMLINK); 644 goto out; 645 } 646 647 VM_BUG_ON_PAGE((flags & FOLL_PIN) && PageAnon(page) && 648 !PageAnonExclusive(page), page); 649 650 /* try_grab_page() does nothing unless FOLL_GET or FOLL_PIN is set. */ 651 ret = try_grab_page(page, flags); 652 if (unlikely(ret)) { 653 page = ERR_PTR(ret); 654 goto out; 655 } 656 657 /* 658 * We need to make the page accessible if and only if we are going 659 * to access its content (the FOLL_PIN case). Please see 660 * Documentation/core-api/pin_user_pages.rst for details. 661 */ 662 if (flags & FOLL_PIN) { 663 ret = arch_make_page_accessible(page); 664 if (ret) { 665 unpin_user_page(page); 666 page = ERR_PTR(ret); 667 goto out; 668 } 669 } 670 if (flags & FOLL_TOUCH) { 671 if ((flags & FOLL_WRITE) && 672 !pte_dirty(pte) && !PageDirty(page)) 673 set_page_dirty(page); 674 /* 675 * pte_mkyoung() would be more correct here, but atomic care 676 * is needed to avoid losing the dirty bit: it is easier to use 677 * mark_page_accessed(). 678 */ 679 mark_page_accessed(page); 680 } 681 out: 682 pte_unmap_unlock(ptep, ptl); 683 return page; 684 no_page: 685 pte_unmap_unlock(ptep, ptl); 686 if (!pte_none(pte)) 687 return NULL; 688 return no_page_table(vma, flags); 689 } 690 691 static struct page *follow_pmd_mask(struct vm_area_struct *vma, 692 unsigned long address, pud_t *pudp, 693 unsigned int flags, 694 struct follow_page_context *ctx) 695 { 696 pmd_t *pmd, pmdval; 697 spinlock_t *ptl; 698 struct page *page; 699 struct mm_struct *mm = vma->vm_mm; 700 701 pmd = pmd_offset(pudp, address); 702 pmdval = pmdp_get_lockless(pmd); 703 if (pmd_none(pmdval)) 704 return no_page_table(vma, flags); 705 if (!pmd_present(pmdval)) 706 return no_page_table(vma, flags); 707 if (pmd_devmap(pmdval)) { 708 ptl = pmd_lock(mm, pmd); 709 page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap); 710 spin_unlock(ptl); 711 if (page) 712 return page; 713 return no_page_table(vma, flags); 714 } 715 if (likely(!pmd_trans_huge(pmdval))) 716 return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); 717 718 if (pmd_protnone(pmdval) && !gup_can_follow_protnone(vma, flags)) 719 return no_page_table(vma, flags); 720 721 ptl = pmd_lock(mm, pmd); 722 if (unlikely(!pmd_present(*pmd))) { 723 spin_unlock(ptl); 724 return no_page_table(vma, flags); 725 } 726 if (unlikely(!pmd_trans_huge(*pmd))) { 727 spin_unlock(ptl); 728 return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); 729 } 730 if (flags & FOLL_SPLIT_PMD) { 731 spin_unlock(ptl); 732 split_huge_pmd(vma, pmd, address); 733 /* If pmd was left empty, stuff a page table in there quickly */ 734 return pte_alloc(mm, pmd) ? ERR_PTR(-ENOMEM) : 735 follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); 736 } 737 page = follow_trans_huge_pmd(vma, address, pmd, flags); 738 spin_unlock(ptl); 739 ctx->page_mask = HPAGE_PMD_NR - 1; 740 return page; 741 } 742 743 static struct page *follow_pud_mask(struct vm_area_struct *vma, 744 unsigned long address, p4d_t *p4dp, 745 unsigned int flags, 746 struct follow_page_context *ctx) 747 { 748 pud_t *pud; 749 spinlock_t *ptl; 750 struct page *page; 751 struct mm_struct *mm = vma->vm_mm; 752 753 pud = pud_offset(p4dp, address); 754 if (pud_none(*pud)) 755 return no_page_table(vma, flags); 756 if (pud_devmap(*pud)) { 757 ptl = pud_lock(mm, pud); 758 page = follow_devmap_pud(vma, address, pud, flags, &ctx->pgmap); 759 spin_unlock(ptl); 760 if (page) 761 return page; 762 return no_page_table(vma, flags); 763 } 764 if (unlikely(pud_bad(*pud))) 765 return no_page_table(vma, flags); 766 767 return follow_pmd_mask(vma, address, pud, flags, ctx); 768 } 769 770 static struct page *follow_p4d_mask(struct vm_area_struct *vma, 771 unsigned long address, pgd_t *pgdp, 772 unsigned int flags, 773 struct follow_page_context *ctx) 774 { 775 p4d_t *p4d; 776 777 p4d = p4d_offset(pgdp, address); 778 if (p4d_none(*p4d)) 779 return no_page_table(vma, flags); 780 BUILD_BUG_ON(p4d_huge(*p4d)); 781 if (unlikely(p4d_bad(*p4d))) 782 return no_page_table(vma, flags); 783 784 return follow_pud_mask(vma, address, p4d, flags, ctx); 785 } 786 787 /** 788 * follow_page_mask - look up a page descriptor from a user-virtual address 789 * @vma: vm_area_struct mapping @address 790 * @address: virtual address to look up 791 * @flags: flags modifying lookup behaviour 792 * @ctx: contains dev_pagemap for %ZONE_DEVICE memory pinning and a 793 * pointer to output page_mask 794 * 795 * @flags can have FOLL_ flags set, defined in <linux/mm.h> 796 * 797 * When getting pages from ZONE_DEVICE memory, the @ctx->pgmap caches 798 * the device's dev_pagemap metadata to avoid repeating expensive lookups. 799 * 800 * When getting an anonymous page and the caller has to trigger unsharing 801 * of a shared anonymous page first, -EMLINK is returned. The caller should 802 * trigger a fault with FAULT_FLAG_UNSHARE set. Note that unsharing is only 803 * relevant with FOLL_PIN and !FOLL_WRITE. 804 * 805 * On output, the @ctx->page_mask is set according to the size of the page. 806 * 807 * Return: the mapped (struct page *), %NULL if no mapping exists, or 808 * an error pointer if there is a mapping to something not represented 809 * by a page descriptor (see also vm_normal_page()). 810 */ 811 static struct page *follow_page_mask(struct vm_area_struct *vma, 812 unsigned long address, unsigned int flags, 813 struct follow_page_context *ctx) 814 { 815 pgd_t *pgd; 816 struct mm_struct *mm = vma->vm_mm; 817 818 ctx->page_mask = 0; 819 820 /* 821 * Call hugetlb_follow_page_mask for hugetlb vmas as it will use 822 * special hugetlb page table walking code. This eliminates the 823 * need to check for hugetlb entries in the general walking code. 824 */ 825 if (is_vm_hugetlb_page(vma)) 826 return hugetlb_follow_page_mask(vma, address, flags, 827 &ctx->page_mask); 828 829 pgd = pgd_offset(mm, address); 830 831 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) 832 return no_page_table(vma, flags); 833 834 return follow_p4d_mask(vma, address, pgd, flags, ctx); 835 } 836 837 struct page *follow_page(struct vm_area_struct *vma, unsigned long address, 838 unsigned int foll_flags) 839 { 840 struct follow_page_context ctx = { NULL }; 841 struct page *page; 842 843 if (vma_is_secretmem(vma)) 844 return NULL; 845 846 if (WARN_ON_ONCE(foll_flags & FOLL_PIN)) 847 return NULL; 848 849 /* 850 * We never set FOLL_HONOR_NUMA_FAULT because callers don't expect 851 * to fail on PROT_NONE-mapped pages. 852 */ 853 page = follow_page_mask(vma, address, foll_flags, &ctx); 854 if (ctx.pgmap) 855 put_dev_pagemap(ctx.pgmap); 856 return page; 857 } 858 859 static int get_gate_page(struct mm_struct *mm, unsigned long address, 860 unsigned int gup_flags, struct vm_area_struct **vma, 861 struct page **page) 862 { 863 pgd_t *pgd; 864 p4d_t *p4d; 865 pud_t *pud; 866 pmd_t *pmd; 867 pte_t *pte; 868 pte_t entry; 869 int ret = -EFAULT; 870 871 /* user gate pages are read-only */ 872 if (gup_flags & FOLL_WRITE) 873 return -EFAULT; 874 if (address > TASK_SIZE) 875 pgd = pgd_offset_k(address); 876 else 877 pgd = pgd_offset_gate(mm, address); 878 if (pgd_none(*pgd)) 879 return -EFAULT; 880 p4d = p4d_offset(pgd, address); 881 if (p4d_none(*p4d)) 882 return -EFAULT; 883 pud = pud_offset(p4d, address); 884 if (pud_none(*pud)) 885 return -EFAULT; 886 pmd = pmd_offset(pud, address); 887 if (!pmd_present(*pmd)) 888 return -EFAULT; 889 pte = pte_offset_map(pmd, address); 890 if (!pte) 891 return -EFAULT; 892 entry = ptep_get(pte); 893 if (pte_none(entry)) 894 goto unmap; 895 *vma = get_gate_vma(mm); 896 if (!page) 897 goto out; 898 *page = vm_normal_page(*vma, address, entry); 899 if (!*page) { 900 if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(entry))) 901 goto unmap; 902 *page = pte_page(entry); 903 } 904 ret = try_grab_page(*page, gup_flags); 905 if (unlikely(ret)) 906 goto unmap; 907 out: 908 ret = 0; 909 unmap: 910 pte_unmap(pte); 911 return ret; 912 } 913 914 /* 915 * mmap_lock must be held on entry. If @flags has FOLL_UNLOCKABLE but not 916 * FOLL_NOWAIT, the mmap_lock may be released. If it is, *@locked will be set 917 * to 0 and -EBUSY returned. 918 */ 919 static int faultin_page(struct vm_area_struct *vma, 920 unsigned long address, unsigned int *flags, bool unshare, 921 int *locked) 922 { 923 unsigned int fault_flags = 0; 924 vm_fault_t ret; 925 926 if (*flags & FOLL_NOFAULT) 927 return -EFAULT; 928 if (*flags & FOLL_WRITE) 929 fault_flags |= FAULT_FLAG_WRITE; 930 if (*flags & FOLL_REMOTE) 931 fault_flags |= FAULT_FLAG_REMOTE; 932 if (*flags & FOLL_UNLOCKABLE) { 933 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 934 /* 935 * FAULT_FLAG_INTERRUPTIBLE is opt-in. GUP callers must set 936 * FOLL_INTERRUPTIBLE to enable FAULT_FLAG_INTERRUPTIBLE. 937 * That's because some callers may not be prepared to 938 * handle early exits caused by non-fatal signals. 939 */ 940 if (*flags & FOLL_INTERRUPTIBLE) 941 fault_flags |= FAULT_FLAG_INTERRUPTIBLE; 942 } 943 if (*flags & FOLL_NOWAIT) 944 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT; 945 if (*flags & FOLL_TRIED) { 946 /* 947 * Note: FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_TRIED 948 * can co-exist 949 */ 950 fault_flags |= FAULT_FLAG_TRIED; 951 } 952 if (unshare) { 953 fault_flags |= FAULT_FLAG_UNSHARE; 954 /* FAULT_FLAG_WRITE and FAULT_FLAG_UNSHARE are incompatible */ 955 VM_BUG_ON(fault_flags & FAULT_FLAG_WRITE); 956 } 957 958 ret = handle_mm_fault(vma, address, fault_flags, NULL); 959 960 if (ret & VM_FAULT_COMPLETED) { 961 /* 962 * With FAULT_FLAG_RETRY_NOWAIT we'll never release the 963 * mmap lock in the page fault handler. Sanity check this. 964 */ 965 WARN_ON_ONCE(fault_flags & FAULT_FLAG_RETRY_NOWAIT); 966 *locked = 0; 967 968 /* 969 * We should do the same as VM_FAULT_RETRY, but let's not 970 * return -EBUSY since that's not reflecting the reality of 971 * what has happened - we've just fully completed a page 972 * fault, with the mmap lock released. Use -EAGAIN to show 973 * that we want to take the mmap lock _again_. 974 */ 975 return -EAGAIN; 976 } 977 978 if (ret & VM_FAULT_ERROR) { 979 int err = vm_fault_to_errno(ret, *flags); 980 981 if (err) 982 return err; 983 BUG(); 984 } 985 986 if (ret & VM_FAULT_RETRY) { 987 if (!(fault_flags & FAULT_FLAG_RETRY_NOWAIT)) 988 *locked = 0; 989 return -EBUSY; 990 } 991 992 return 0; 993 } 994 995 /* 996 * Writing to file-backed mappings which require folio dirty tracking using GUP 997 * is a fundamentally broken operation, as kernel write access to GUP mappings 998 * do not adhere to the semantics expected by a file system. 999 * 1000 * Consider the following scenario:- 1001 * 1002 * 1. A folio is written to via GUP which write-faults the memory, notifying 1003 * the file system and dirtying the folio. 1004 * 2. Later, writeback is triggered, resulting in the folio being cleaned and 1005 * the PTE being marked read-only. 1006 * 3. The GUP caller writes to the folio, as it is mapped read/write via the 1007 * direct mapping. 1008 * 4. The GUP caller, now done with the page, unpins it and sets it dirty 1009 * (though it does not have to). 1010 * 1011 * This results in both data being written to a folio without writenotify, and 1012 * the folio being dirtied unexpectedly (if the caller decides to do so). 1013 */ 1014 static bool writable_file_mapping_allowed(struct vm_area_struct *vma, 1015 unsigned long gup_flags) 1016 { 1017 /* 1018 * If we aren't pinning then no problematic write can occur. A long term 1019 * pin is the most egregious case so this is the case we disallow. 1020 */ 1021 if ((gup_flags & (FOLL_PIN | FOLL_LONGTERM)) != 1022 (FOLL_PIN | FOLL_LONGTERM)) 1023 return true; 1024 1025 /* 1026 * If the VMA does not require dirty tracking then no problematic write 1027 * can occur either. 1028 */ 1029 return !vma_needs_dirty_tracking(vma); 1030 } 1031 1032 static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) 1033 { 1034 vm_flags_t vm_flags = vma->vm_flags; 1035 int write = (gup_flags & FOLL_WRITE); 1036 int foreign = (gup_flags & FOLL_REMOTE); 1037 bool vma_anon = vma_is_anonymous(vma); 1038 1039 if (vm_flags & (VM_IO | VM_PFNMAP)) 1040 return -EFAULT; 1041 1042 if ((gup_flags & FOLL_ANON) && !vma_anon) 1043 return -EFAULT; 1044 1045 if ((gup_flags & FOLL_LONGTERM) && vma_is_fsdax(vma)) 1046 return -EOPNOTSUPP; 1047 1048 if (vma_is_secretmem(vma)) 1049 return -EFAULT; 1050 1051 if (write) { 1052 if (!vma_anon && 1053 !writable_file_mapping_allowed(vma, gup_flags)) 1054 return -EFAULT; 1055 1056 if (!(vm_flags & VM_WRITE) || (vm_flags & VM_SHADOW_STACK)) { 1057 if (!(gup_flags & FOLL_FORCE)) 1058 return -EFAULT; 1059 /* hugetlb does not support FOLL_FORCE|FOLL_WRITE. */ 1060 if (is_vm_hugetlb_page(vma)) 1061 return -EFAULT; 1062 /* 1063 * We used to let the write,force case do COW in a 1064 * VM_MAYWRITE VM_SHARED !VM_WRITE vma, so ptrace could 1065 * set a breakpoint in a read-only mapping of an 1066 * executable, without corrupting the file (yet only 1067 * when that file had been opened for writing!). 1068 * Anon pages in shared mappings are surprising: now 1069 * just reject it. 1070 */ 1071 if (!is_cow_mapping(vm_flags)) 1072 return -EFAULT; 1073 } 1074 } else if (!(vm_flags & VM_READ)) { 1075 if (!(gup_flags & FOLL_FORCE)) 1076 return -EFAULT; 1077 /* 1078 * Is there actually any vma we can reach here which does not 1079 * have VM_MAYREAD set? 1080 */ 1081 if (!(vm_flags & VM_MAYREAD)) 1082 return -EFAULT; 1083 } 1084 /* 1085 * gups are always data accesses, not instruction 1086 * fetches, so execute=false here 1087 */ 1088 if (!arch_vma_access_permitted(vma, write, false, foreign)) 1089 return -EFAULT; 1090 return 0; 1091 } 1092 1093 /* 1094 * This is "vma_lookup()", but with a warning if we would have 1095 * historically expanded the stack in the GUP code. 1096 */ 1097 static struct vm_area_struct *gup_vma_lookup(struct mm_struct *mm, 1098 unsigned long addr) 1099 { 1100 #ifdef CONFIG_STACK_GROWSUP 1101 return vma_lookup(mm, addr); 1102 #else 1103 static volatile unsigned long next_warn; 1104 struct vm_area_struct *vma; 1105 unsigned long now, next; 1106 1107 vma = find_vma(mm, addr); 1108 if (!vma || (addr >= vma->vm_start)) 1109 return vma; 1110 1111 /* Only warn for half-way relevant accesses */ 1112 if (!(vma->vm_flags & VM_GROWSDOWN)) 1113 return NULL; 1114 if (vma->vm_start - addr > 65536) 1115 return NULL; 1116 1117 /* Let's not warn more than once an hour.. */ 1118 now = jiffies; next = next_warn; 1119 if (next && time_before(now, next)) 1120 return NULL; 1121 next_warn = now + 60*60*HZ; 1122 1123 /* Let people know things may have changed. */ 1124 pr_warn("GUP no longer grows the stack in %s (%d): %lx-%lx (%lx)\n", 1125 current->comm, task_pid_nr(current), 1126 vma->vm_start, vma->vm_end, addr); 1127 dump_stack(); 1128 return NULL; 1129 #endif 1130 } 1131 1132 /** 1133 * __get_user_pages() - pin user pages in memory 1134 * @mm: mm_struct of target mm 1135 * @start: starting user address 1136 * @nr_pages: number of pages from start to pin 1137 * @gup_flags: flags modifying pin behaviour 1138 * @pages: array that receives pointers to the pages pinned. 1139 * Should be at least nr_pages long. Or NULL, if caller 1140 * only intends to ensure the pages are faulted in. 1141 * @locked: whether we're still with the mmap_lock held 1142 * 1143 * Returns either number of pages pinned (which may be less than the 1144 * number requested), or an error. Details about the return value: 1145 * 1146 * -- If nr_pages is 0, returns 0. 1147 * -- If nr_pages is >0, but no pages were pinned, returns -errno. 1148 * -- If nr_pages is >0, and some pages were pinned, returns the number of 1149 * pages pinned. Again, this may be less than nr_pages. 1150 * -- 0 return value is possible when the fault would need to be retried. 1151 * 1152 * The caller is responsible for releasing returned @pages, via put_page(). 1153 * 1154 * Must be called with mmap_lock held. It may be released. See below. 1155 * 1156 * __get_user_pages walks a process's page tables and takes a reference to 1157 * each struct page that each user address corresponds to at a given 1158 * instant. That is, it takes the page that would be accessed if a user 1159 * thread accesses the given user virtual address at that instant. 1160 * 1161 * This does not guarantee that the page exists in the user mappings when 1162 * __get_user_pages returns, and there may even be a completely different 1163 * page there in some cases (eg. if mmapped pagecache has been invalidated 1164 * and subsequently re-faulted). However it does guarantee that the page 1165 * won't be freed completely. And mostly callers simply care that the page 1166 * contains data that was valid *at some point in time*. Typically, an IO 1167 * or similar operation cannot guarantee anything stronger anyway because 1168 * locks can't be held over the syscall boundary. 1169 * 1170 * If @gup_flags & FOLL_WRITE == 0, the page must not be written to. If 1171 * the page is written to, set_page_dirty (or set_page_dirty_lock, as 1172 * appropriate) must be called after the page is finished with, and 1173 * before put_page is called. 1174 * 1175 * If FOLL_UNLOCKABLE is set without FOLL_NOWAIT then the mmap_lock may 1176 * be released. If this happens *@locked will be set to 0 on return. 1177 * 1178 * A caller using such a combination of @gup_flags must therefore hold the 1179 * mmap_lock for reading only, and recognize when it's been released. Otherwise, 1180 * it must be held for either reading or writing and will not be released. 1181 * 1182 * In most cases, get_user_pages or get_user_pages_fast should be used 1183 * instead of __get_user_pages. __get_user_pages should be used only if 1184 * you need some special @gup_flags. 1185 */ 1186 static long __get_user_pages(struct mm_struct *mm, 1187 unsigned long start, unsigned long nr_pages, 1188 unsigned int gup_flags, struct page **pages, 1189 int *locked) 1190 { 1191 long ret = 0, i = 0; 1192 struct vm_area_struct *vma = NULL; 1193 struct follow_page_context ctx = { NULL }; 1194 1195 if (!nr_pages) 1196 return 0; 1197 1198 start = untagged_addr_remote(mm, start); 1199 1200 VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN))); 1201 1202 do { 1203 struct page *page; 1204 unsigned int foll_flags = gup_flags; 1205 unsigned int page_increm; 1206 1207 /* first iteration or cross vma bound */ 1208 if (!vma || start >= vma->vm_end) { 1209 vma = gup_vma_lookup(mm, start); 1210 if (!vma && in_gate_area(mm, start)) { 1211 ret = get_gate_page(mm, start & PAGE_MASK, 1212 gup_flags, &vma, 1213 pages ? &page : NULL); 1214 if (ret) 1215 goto out; 1216 ctx.page_mask = 0; 1217 goto next_page; 1218 } 1219 1220 if (!vma) { 1221 ret = -EFAULT; 1222 goto out; 1223 } 1224 ret = check_vma_flags(vma, gup_flags); 1225 if (ret) 1226 goto out; 1227 } 1228 retry: 1229 /* 1230 * If we have a pending SIGKILL, don't keep faulting pages and 1231 * potentially allocating memory. 1232 */ 1233 if (fatal_signal_pending(current)) { 1234 ret = -EINTR; 1235 goto out; 1236 } 1237 cond_resched(); 1238 1239 page = follow_page_mask(vma, start, foll_flags, &ctx); 1240 if (!page || PTR_ERR(page) == -EMLINK) { 1241 ret = faultin_page(vma, start, &foll_flags, 1242 PTR_ERR(page) == -EMLINK, locked); 1243 switch (ret) { 1244 case 0: 1245 goto retry; 1246 case -EBUSY: 1247 case -EAGAIN: 1248 ret = 0; 1249 fallthrough; 1250 case -EFAULT: 1251 case -ENOMEM: 1252 case -EHWPOISON: 1253 goto out; 1254 } 1255 BUG(); 1256 } else if (PTR_ERR(page) == -EEXIST) { 1257 /* 1258 * Proper page table entry exists, but no corresponding 1259 * struct page. If the caller expects **pages to be 1260 * filled in, bail out now, because that can't be done 1261 * for this page. 1262 */ 1263 if (pages) { 1264 ret = PTR_ERR(page); 1265 goto out; 1266 } 1267 } else if (IS_ERR(page)) { 1268 ret = PTR_ERR(page); 1269 goto out; 1270 } 1271 next_page: 1272 page_increm = 1 + (~(start >> PAGE_SHIFT) & ctx.page_mask); 1273 if (page_increm > nr_pages) 1274 page_increm = nr_pages; 1275 1276 if (pages) { 1277 struct page *subpage; 1278 unsigned int j; 1279 1280 /* 1281 * This must be a large folio (and doesn't need to 1282 * be the whole folio; it can be part of it), do 1283 * the refcount work for all the subpages too. 1284 * 1285 * NOTE: here the page may not be the head page 1286 * e.g. when start addr is not thp-size aligned. 1287 * try_grab_folio() should have taken care of tail 1288 * pages. 1289 */ 1290 if (page_increm > 1) { 1291 struct folio *folio; 1292 1293 /* 1294 * Since we already hold refcount on the 1295 * large folio, this should never fail. 1296 */ 1297 folio = try_grab_folio(page, page_increm - 1, 1298 foll_flags); 1299 if (WARN_ON_ONCE(!folio)) { 1300 /* 1301 * Release the 1st page ref if the 1302 * folio is problematic, fail hard. 1303 */ 1304 gup_put_folio(page_folio(page), 1, 1305 foll_flags); 1306 ret = -EFAULT; 1307 goto out; 1308 } 1309 } 1310 1311 for (j = 0; j < page_increm; j++) { 1312 subpage = nth_page(page, j); 1313 pages[i + j] = subpage; 1314 flush_anon_page(vma, subpage, start + j * PAGE_SIZE); 1315 flush_dcache_page(subpage); 1316 } 1317 } 1318 1319 i += page_increm; 1320 start += page_increm * PAGE_SIZE; 1321 nr_pages -= page_increm; 1322 } while (nr_pages); 1323 out: 1324 if (ctx.pgmap) 1325 put_dev_pagemap(ctx.pgmap); 1326 return i ? i : ret; 1327 } 1328 1329 static bool vma_permits_fault(struct vm_area_struct *vma, 1330 unsigned int fault_flags) 1331 { 1332 bool write = !!(fault_flags & FAULT_FLAG_WRITE); 1333 bool foreign = !!(fault_flags & FAULT_FLAG_REMOTE); 1334 vm_flags_t vm_flags = write ? VM_WRITE : VM_READ; 1335 1336 if (!(vm_flags & vma->vm_flags)) 1337 return false; 1338 1339 /* 1340 * The architecture might have a hardware protection 1341 * mechanism other than read/write that can deny access. 1342 * 1343 * gup always represents data access, not instruction 1344 * fetches, so execute=false here: 1345 */ 1346 if (!arch_vma_access_permitted(vma, write, false, foreign)) 1347 return false; 1348 1349 return true; 1350 } 1351 1352 /** 1353 * fixup_user_fault() - manually resolve a user page fault 1354 * @mm: mm_struct of target mm 1355 * @address: user address 1356 * @fault_flags:flags to pass down to handle_mm_fault() 1357 * @unlocked: did we unlock the mmap_lock while retrying, maybe NULL if caller 1358 * does not allow retry. If NULL, the caller must guarantee 1359 * that fault_flags does not contain FAULT_FLAG_ALLOW_RETRY. 1360 * 1361 * This is meant to be called in the specific scenario where for locking reasons 1362 * we try to access user memory in atomic context (within a pagefault_disable() 1363 * section), this returns -EFAULT, and we want to resolve the user fault before 1364 * trying again. 1365 * 1366 * Typically this is meant to be used by the futex code. 1367 * 1368 * The main difference with get_user_pages() is that this function will 1369 * unconditionally call handle_mm_fault() which will in turn perform all the 1370 * necessary SW fixup of the dirty and young bits in the PTE, while 1371 * get_user_pages() only guarantees to update these in the struct page. 1372 * 1373 * This is important for some architectures where those bits also gate the 1374 * access permission to the page because they are maintained in software. On 1375 * such architectures, gup() will not be enough to make a subsequent access 1376 * succeed. 1377 * 1378 * This function will not return with an unlocked mmap_lock. So it has not the 1379 * same semantics wrt the @mm->mmap_lock as does filemap_fault(). 1380 */ 1381 int fixup_user_fault(struct mm_struct *mm, 1382 unsigned long address, unsigned int fault_flags, 1383 bool *unlocked) 1384 { 1385 struct vm_area_struct *vma; 1386 vm_fault_t ret; 1387 1388 address = untagged_addr_remote(mm, address); 1389 1390 if (unlocked) 1391 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; 1392 1393 retry: 1394 vma = gup_vma_lookup(mm, address); 1395 if (!vma) 1396 return -EFAULT; 1397 1398 if (!vma_permits_fault(vma, fault_flags)) 1399 return -EFAULT; 1400 1401 if ((fault_flags & FAULT_FLAG_KILLABLE) && 1402 fatal_signal_pending(current)) 1403 return -EINTR; 1404 1405 ret = handle_mm_fault(vma, address, fault_flags, NULL); 1406 1407 if (ret & VM_FAULT_COMPLETED) { 1408 /* 1409 * NOTE: it's a pity that we need to retake the lock here 1410 * to pair with the unlock() in the callers. Ideally we 1411 * could tell the callers so they do not need to unlock. 1412 */ 1413 mmap_read_lock(mm); 1414 *unlocked = true; 1415 return 0; 1416 } 1417 1418 if (ret & VM_FAULT_ERROR) { 1419 int err = vm_fault_to_errno(ret, 0); 1420 1421 if (err) 1422 return err; 1423 BUG(); 1424 } 1425 1426 if (ret & VM_FAULT_RETRY) { 1427 mmap_read_lock(mm); 1428 *unlocked = true; 1429 fault_flags |= FAULT_FLAG_TRIED; 1430 goto retry; 1431 } 1432 1433 return 0; 1434 } 1435 EXPORT_SYMBOL_GPL(fixup_user_fault); 1436 1437 /* 1438 * GUP always responds to fatal signals. When FOLL_INTERRUPTIBLE is 1439 * specified, it'll also respond to generic signals. The caller of GUP 1440 * that has FOLL_INTERRUPTIBLE should take care of the GUP interruption. 1441 */ 1442 static bool gup_signal_pending(unsigned int flags) 1443 { 1444 if (fatal_signal_pending(current)) 1445 return true; 1446 1447 if (!(flags & FOLL_INTERRUPTIBLE)) 1448 return false; 1449 1450 return signal_pending(current); 1451 } 1452 1453 /* 1454 * Locking: (*locked == 1) means that the mmap_lock has already been acquired by 1455 * the caller. This function may drop the mmap_lock. If it does so, then it will 1456 * set (*locked = 0). 1457 * 1458 * (*locked == 0) means that the caller expects this function to acquire and 1459 * drop the mmap_lock. Therefore, the value of *locked will still be zero when 1460 * the function returns, even though it may have changed temporarily during 1461 * function execution. 1462 * 1463 * Please note that this function, unlike __get_user_pages(), will not return 0 1464 * for nr_pages > 0, unless FOLL_NOWAIT is used. 1465 */ 1466 static __always_inline long __get_user_pages_locked(struct mm_struct *mm, 1467 unsigned long start, 1468 unsigned long nr_pages, 1469 struct page **pages, 1470 int *locked, 1471 unsigned int flags) 1472 { 1473 long ret, pages_done; 1474 bool must_unlock = false; 1475 1476 if (!nr_pages) 1477 return 0; 1478 1479 /* 1480 * The internal caller expects GUP to manage the lock internally and the 1481 * lock must be released when this returns. 1482 */ 1483 if (!*locked) { 1484 if (mmap_read_lock_killable(mm)) 1485 return -EAGAIN; 1486 must_unlock = true; 1487 *locked = 1; 1488 } 1489 else 1490 mmap_assert_locked(mm); 1491 1492 if (flags & FOLL_PIN) 1493 mm_set_has_pinned_flag(&mm->flags); 1494 1495 /* 1496 * FOLL_PIN and FOLL_GET are mutually exclusive. Traditional behavior 1497 * is to set FOLL_GET if the caller wants pages[] filled in (but has 1498 * carelessly failed to specify FOLL_GET), so keep doing that, but only 1499 * for FOLL_GET, not for the newer FOLL_PIN. 1500 * 1501 * FOLL_PIN always expects pages to be non-null, but no need to assert 1502 * that here, as any failures will be obvious enough. 1503 */ 1504 if (pages && !(flags & FOLL_PIN)) 1505 flags |= FOLL_GET; 1506 1507 pages_done = 0; 1508 for (;;) { 1509 ret = __get_user_pages(mm, start, nr_pages, flags, pages, 1510 locked); 1511 if (!(flags & FOLL_UNLOCKABLE)) { 1512 /* VM_FAULT_RETRY couldn't trigger, bypass */ 1513 pages_done = ret; 1514 break; 1515 } 1516 1517 /* VM_FAULT_RETRY or VM_FAULT_COMPLETED cannot return errors */ 1518 if (!*locked) { 1519 BUG_ON(ret < 0); 1520 BUG_ON(ret >= nr_pages); 1521 } 1522 1523 if (ret > 0) { 1524 nr_pages -= ret; 1525 pages_done += ret; 1526 if (!nr_pages) 1527 break; 1528 } 1529 if (*locked) { 1530 /* 1531 * VM_FAULT_RETRY didn't trigger or it was a 1532 * FOLL_NOWAIT. 1533 */ 1534 if (!pages_done) 1535 pages_done = ret; 1536 break; 1537 } 1538 /* 1539 * VM_FAULT_RETRY triggered, so seek to the faulting offset. 1540 * For the prefault case (!pages) we only update counts. 1541 */ 1542 if (likely(pages)) 1543 pages += ret; 1544 start += ret << PAGE_SHIFT; 1545 1546 /* The lock was temporarily dropped, so we must unlock later */ 1547 must_unlock = true; 1548 1549 retry: 1550 /* 1551 * Repeat on the address that fired VM_FAULT_RETRY 1552 * with both FAULT_FLAG_ALLOW_RETRY and 1553 * FAULT_FLAG_TRIED. Note that GUP can be interrupted 1554 * by fatal signals of even common signals, depending on 1555 * the caller's request. So we need to check it before we 1556 * start trying again otherwise it can loop forever. 1557 */ 1558 if (gup_signal_pending(flags)) { 1559 if (!pages_done) 1560 pages_done = -EINTR; 1561 break; 1562 } 1563 1564 ret = mmap_read_lock_killable(mm); 1565 if (ret) { 1566 BUG_ON(ret > 0); 1567 if (!pages_done) 1568 pages_done = ret; 1569 break; 1570 } 1571 1572 *locked = 1; 1573 ret = __get_user_pages(mm, start, 1, flags | FOLL_TRIED, 1574 pages, locked); 1575 if (!*locked) { 1576 /* Continue to retry until we succeeded */ 1577 BUG_ON(ret != 0); 1578 goto retry; 1579 } 1580 if (ret != 1) { 1581 BUG_ON(ret > 1); 1582 if (!pages_done) 1583 pages_done = ret; 1584 break; 1585 } 1586 nr_pages--; 1587 pages_done++; 1588 if (!nr_pages) 1589 break; 1590 if (likely(pages)) 1591 pages++; 1592 start += PAGE_SIZE; 1593 } 1594 if (must_unlock && *locked) { 1595 /* 1596 * We either temporarily dropped the lock, or the caller 1597 * requested that we both acquire and drop the lock. Either way, 1598 * we must now unlock, and notify the caller of that state. 1599 */ 1600 mmap_read_unlock(mm); 1601 *locked = 0; 1602 } 1603 1604 /* 1605 * Failing to pin anything implies something has gone wrong (except when 1606 * FOLL_NOWAIT is specified). 1607 */ 1608 if (WARN_ON_ONCE(pages_done == 0 && !(flags & FOLL_NOWAIT))) 1609 return -EFAULT; 1610 1611 return pages_done; 1612 } 1613 1614 /** 1615 * populate_vma_page_range() - populate a range of pages in the vma. 1616 * @vma: target vma 1617 * @start: start address 1618 * @end: end address 1619 * @locked: whether the mmap_lock is still held 1620 * 1621 * This takes care of mlocking the pages too if VM_LOCKED is set. 1622 * 1623 * Return either number of pages pinned in the vma, or a negative error 1624 * code on error. 1625 * 1626 * vma->vm_mm->mmap_lock must be held. 1627 * 1628 * If @locked is NULL, it may be held for read or write and will 1629 * be unperturbed. 1630 * 1631 * If @locked is non-NULL, it must held for read only and may be 1632 * released. If it's released, *@locked will be set to 0. 1633 */ 1634 long populate_vma_page_range(struct vm_area_struct *vma, 1635 unsigned long start, unsigned long end, int *locked) 1636 { 1637 struct mm_struct *mm = vma->vm_mm; 1638 unsigned long nr_pages = (end - start) / PAGE_SIZE; 1639 int local_locked = 1; 1640 int gup_flags; 1641 long ret; 1642 1643 VM_BUG_ON(!PAGE_ALIGNED(start)); 1644 VM_BUG_ON(!PAGE_ALIGNED(end)); 1645 VM_BUG_ON_VMA(start < vma->vm_start, vma); 1646 VM_BUG_ON_VMA(end > vma->vm_end, vma); 1647 mmap_assert_locked(mm); 1648 1649 /* 1650 * Rightly or wrongly, the VM_LOCKONFAULT case has never used 1651 * faultin_page() to break COW, so it has no work to do here. 1652 */ 1653 if (vma->vm_flags & VM_LOCKONFAULT) 1654 return nr_pages; 1655 1656 /* ... similarly, we've never faulted in PROT_NONE pages */ 1657 if (!vma_is_accessible(vma)) 1658 return -EFAULT; 1659 1660 gup_flags = FOLL_TOUCH; 1661 /* 1662 * We want to touch writable mappings with a write fault in order 1663 * to break COW, except for shared mappings because these don't COW 1664 * and we would not want to dirty them for nothing. 1665 * 1666 * Otherwise, do a read fault, and use FOLL_FORCE in case it's not 1667 * readable (ie write-only or executable). 1668 */ 1669 if ((vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE) 1670 gup_flags |= FOLL_WRITE; 1671 else 1672 gup_flags |= FOLL_FORCE; 1673 1674 if (locked) 1675 gup_flags |= FOLL_UNLOCKABLE; 1676 1677 /* 1678 * We made sure addr is within a VMA, so the following will 1679 * not result in a stack expansion that recurses back here. 1680 */ 1681 ret = __get_user_pages(mm, start, nr_pages, gup_flags, 1682 NULL, locked ? locked : &local_locked); 1683 lru_add_drain(); 1684 return ret; 1685 } 1686 1687 /* 1688 * faultin_vma_page_range() - populate (prefault) page tables inside the 1689 * given VMA range readable/writable 1690 * 1691 * This takes care of mlocking the pages, too, if VM_LOCKED is set. 1692 * 1693 * @vma: target vma 1694 * @start: start address 1695 * @end: end address 1696 * @write: whether to prefault readable or writable 1697 * @locked: whether the mmap_lock is still held 1698 * 1699 * Returns either number of processed pages in the vma, or a negative error 1700 * code on error (see __get_user_pages()). 1701 * 1702 * vma->vm_mm->mmap_lock must be held. The range must be page-aligned and 1703 * covered by the VMA. If it's released, *@locked will be set to 0. 1704 */ 1705 long faultin_vma_page_range(struct vm_area_struct *vma, unsigned long start, 1706 unsigned long end, bool write, int *locked) 1707 { 1708 struct mm_struct *mm = vma->vm_mm; 1709 unsigned long nr_pages = (end - start) / PAGE_SIZE; 1710 int gup_flags; 1711 long ret; 1712 1713 VM_BUG_ON(!PAGE_ALIGNED(start)); 1714 VM_BUG_ON(!PAGE_ALIGNED(end)); 1715 VM_BUG_ON_VMA(start < vma->vm_start, vma); 1716 VM_BUG_ON_VMA(end > vma->vm_end, vma); 1717 mmap_assert_locked(mm); 1718 1719 /* 1720 * FOLL_TOUCH: Mark page accessed and thereby young; will also mark 1721 * the page dirty with FOLL_WRITE -- which doesn't make a 1722 * difference with !FOLL_FORCE, because the page is writable 1723 * in the page table. 1724 * FOLL_HWPOISON: Return -EHWPOISON instead of -EFAULT when we hit 1725 * a poisoned page. 1726 * !FOLL_FORCE: Require proper access permissions. 1727 */ 1728 gup_flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_UNLOCKABLE; 1729 if (write) 1730 gup_flags |= FOLL_WRITE; 1731 1732 /* 1733 * We want to report -EINVAL instead of -EFAULT for any permission 1734 * problems or incompatible mappings. 1735 */ 1736 if (check_vma_flags(vma, gup_flags)) 1737 return -EINVAL; 1738 1739 ret = __get_user_pages(mm, start, nr_pages, gup_flags, 1740 NULL, locked); 1741 lru_add_drain(); 1742 return ret; 1743 } 1744 1745 /* 1746 * __mm_populate - populate and/or mlock pages within a range of address space. 1747 * 1748 * This is used to implement mlock() and the MAP_POPULATE / MAP_LOCKED mmap 1749 * flags. VMAs must be already marked with the desired vm_flags, and 1750 * mmap_lock must not be held. 1751 */ 1752 int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) 1753 { 1754 struct mm_struct *mm = current->mm; 1755 unsigned long end, nstart, nend; 1756 struct vm_area_struct *vma = NULL; 1757 int locked = 0; 1758 long ret = 0; 1759 1760 end = start + len; 1761 1762 for (nstart = start; nstart < end; nstart = nend) { 1763 /* 1764 * We want to fault in pages for [nstart; end) address range. 1765 * Find first corresponding VMA. 1766 */ 1767 if (!locked) { 1768 locked = 1; 1769 mmap_read_lock(mm); 1770 vma = find_vma_intersection(mm, nstart, end); 1771 } else if (nstart >= vma->vm_end) 1772 vma = find_vma_intersection(mm, vma->vm_end, end); 1773 1774 if (!vma) 1775 break; 1776 /* 1777 * Set [nstart; nend) to intersection of desired address 1778 * range with the first VMA. Also, skip undesirable VMA types. 1779 */ 1780 nend = min(end, vma->vm_end); 1781 if (vma->vm_flags & (VM_IO | VM_PFNMAP)) 1782 continue; 1783 if (nstart < vma->vm_start) 1784 nstart = vma->vm_start; 1785 /* 1786 * Now fault in a range of pages. populate_vma_page_range() 1787 * double checks the vma flags, so that it won't mlock pages 1788 * if the vma was already munlocked. 1789 */ 1790 ret = populate_vma_page_range(vma, nstart, nend, &locked); 1791 if (ret < 0) { 1792 if (ignore_errors) { 1793 ret = 0; 1794 continue; /* continue at next VMA */ 1795 } 1796 break; 1797 } 1798 nend = nstart + ret * PAGE_SIZE; 1799 ret = 0; 1800 } 1801 if (locked) 1802 mmap_read_unlock(mm); 1803 return ret; /* 0 or negative error code */ 1804 } 1805 #else /* CONFIG_MMU */ 1806 static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start, 1807 unsigned long nr_pages, struct page **pages, 1808 int *locked, unsigned int foll_flags) 1809 { 1810 struct vm_area_struct *vma; 1811 bool must_unlock = false; 1812 unsigned long vm_flags; 1813 long i; 1814 1815 if (!nr_pages) 1816 return 0; 1817 1818 /* 1819 * The internal caller expects GUP to manage the lock internally and the 1820 * lock must be released when this returns. 1821 */ 1822 if (!*locked) { 1823 if (mmap_read_lock_killable(mm)) 1824 return -EAGAIN; 1825 must_unlock = true; 1826 *locked = 1; 1827 } 1828 1829 /* calculate required read or write permissions. 1830 * If FOLL_FORCE is set, we only require the "MAY" flags. 1831 */ 1832 vm_flags = (foll_flags & FOLL_WRITE) ? 1833 (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD); 1834 vm_flags &= (foll_flags & FOLL_FORCE) ? 1835 (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE); 1836 1837 for (i = 0; i < nr_pages; i++) { 1838 vma = find_vma(mm, start); 1839 if (!vma) 1840 break; 1841 1842 /* protect what we can, including chardevs */ 1843 if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) || 1844 !(vm_flags & vma->vm_flags)) 1845 break; 1846 1847 if (pages) { 1848 pages[i] = virt_to_page((void *)start); 1849 if (pages[i]) 1850 get_page(pages[i]); 1851 } 1852 1853 start = (start + PAGE_SIZE) & PAGE_MASK; 1854 } 1855 1856 if (must_unlock && *locked) { 1857 mmap_read_unlock(mm); 1858 *locked = 0; 1859 } 1860 1861 return i ? : -EFAULT; 1862 } 1863 #endif /* !CONFIG_MMU */ 1864 1865 /** 1866 * fault_in_writeable - fault in userspace address range for writing 1867 * @uaddr: start of address range 1868 * @size: size of address range 1869 * 1870 * Returns the number of bytes not faulted in (like copy_to_user() and 1871 * copy_from_user()). 1872 */ 1873 size_t fault_in_writeable(char __user *uaddr, size_t size) 1874 { 1875 char __user *start = uaddr, *end; 1876 1877 if (unlikely(size == 0)) 1878 return 0; 1879 if (!user_write_access_begin(uaddr, size)) 1880 return size; 1881 if (!PAGE_ALIGNED(uaddr)) { 1882 unsafe_put_user(0, uaddr, out); 1883 uaddr = (char __user *)PAGE_ALIGN((unsigned long)uaddr); 1884 } 1885 end = (char __user *)PAGE_ALIGN((unsigned long)start + size); 1886 if (unlikely(end < start)) 1887 end = NULL; 1888 while (uaddr != end) { 1889 unsafe_put_user(0, uaddr, out); 1890 uaddr += PAGE_SIZE; 1891 } 1892 1893 out: 1894 user_write_access_end(); 1895 if (size > uaddr - start) 1896 return size - (uaddr - start); 1897 return 0; 1898 } 1899 EXPORT_SYMBOL(fault_in_writeable); 1900 1901 /** 1902 * fault_in_subpage_writeable - fault in an address range for writing 1903 * @uaddr: start of address range 1904 * @size: size of address range 1905 * 1906 * Fault in a user address range for writing while checking for permissions at 1907 * sub-page granularity (e.g. arm64 MTE). This function should be used when 1908 * the caller cannot guarantee forward progress of a copy_to_user() loop. 1909 * 1910 * Returns the number of bytes not faulted in (like copy_to_user() and 1911 * copy_from_user()). 1912 */ 1913 size_t fault_in_subpage_writeable(char __user *uaddr, size_t size) 1914 { 1915 size_t faulted_in; 1916 1917 /* 1918 * Attempt faulting in at page granularity first for page table 1919 * permission checking. The arch-specific probe_subpage_writeable() 1920 * functions may not check for this. 1921 */ 1922 faulted_in = size - fault_in_writeable(uaddr, size); 1923 if (faulted_in) 1924 faulted_in -= probe_subpage_writeable(uaddr, faulted_in); 1925 1926 return size - faulted_in; 1927 } 1928 EXPORT_SYMBOL(fault_in_subpage_writeable); 1929 1930 /* 1931 * fault_in_safe_writeable - fault in an address range for writing 1932 * @uaddr: start of address range 1933 * @size: length of address range 1934 * 1935 * Faults in an address range for writing. This is primarily useful when we 1936 * already know that some or all of the pages in the address range aren't in 1937 * memory. 1938 * 1939 * Unlike fault_in_writeable(), this function is non-destructive. 1940 * 1941 * Note that we don't pin or otherwise hold the pages referenced that we fault 1942 * in. There's no guarantee that they'll stay in memory for any duration of 1943 * time. 1944 * 1945 * Returns the number of bytes not faulted in, like copy_to_user() and 1946 * copy_from_user(). 1947 */ 1948 size_t fault_in_safe_writeable(const char __user *uaddr, size_t size) 1949 { 1950 unsigned long start = (unsigned long)uaddr, end; 1951 struct mm_struct *mm = current->mm; 1952 bool unlocked = false; 1953 1954 if (unlikely(size == 0)) 1955 return 0; 1956 end = PAGE_ALIGN(start + size); 1957 if (end < start) 1958 end = 0; 1959 1960 mmap_read_lock(mm); 1961 do { 1962 if (fixup_user_fault(mm, start, FAULT_FLAG_WRITE, &unlocked)) 1963 break; 1964 start = (start + PAGE_SIZE) & PAGE_MASK; 1965 } while (start != end); 1966 mmap_read_unlock(mm); 1967 1968 if (size > (unsigned long)uaddr - start) 1969 return size - ((unsigned long)uaddr - start); 1970 return 0; 1971 } 1972 EXPORT_SYMBOL(fault_in_safe_writeable); 1973 1974 /** 1975 * fault_in_readable - fault in userspace address range for reading 1976 * @uaddr: start of user address range 1977 * @size: size of user address range 1978 * 1979 * Returns the number of bytes not faulted in (like copy_to_user() and 1980 * copy_from_user()). 1981 */ 1982 size_t fault_in_readable(const char __user *uaddr, size_t size) 1983 { 1984 const char __user *start = uaddr, *end; 1985 volatile char c; 1986 1987 if (unlikely(size == 0)) 1988 return 0; 1989 if (!user_read_access_begin(uaddr, size)) 1990 return size; 1991 if (!PAGE_ALIGNED(uaddr)) { 1992 unsafe_get_user(c, uaddr, out); 1993 uaddr = (const char __user *)PAGE_ALIGN((unsigned long)uaddr); 1994 } 1995 end = (const char __user *)PAGE_ALIGN((unsigned long)start + size); 1996 if (unlikely(end < start)) 1997 end = NULL; 1998 while (uaddr != end) { 1999 unsafe_get_user(c, uaddr, out); 2000 uaddr += PAGE_SIZE; 2001 } 2002 2003 out: 2004 user_read_access_end(); 2005 (void)c; 2006 if (size > uaddr - start) 2007 return size - (uaddr - start); 2008 return 0; 2009 } 2010 EXPORT_SYMBOL(fault_in_readable); 2011 2012 /** 2013 * get_dump_page() - pin user page in memory while writing it to core dump 2014 * @addr: user address 2015 * 2016 * Returns struct page pointer of user page pinned for dump, 2017 * to be freed afterwards by put_page(). 2018 * 2019 * Returns NULL on any kind of failure - a hole must then be inserted into 2020 * the corefile, to preserve alignment with its headers; and also returns 2021 * NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found - 2022 * allowing a hole to be left in the corefile to save disk space. 2023 * 2024 * Called without mmap_lock (takes and releases the mmap_lock by itself). 2025 */ 2026 #ifdef CONFIG_ELF_CORE 2027 struct page *get_dump_page(unsigned long addr) 2028 { 2029 struct page *page; 2030 int locked = 0; 2031 int ret; 2032 2033 ret = __get_user_pages_locked(current->mm, addr, 1, &page, &locked, 2034 FOLL_FORCE | FOLL_DUMP | FOLL_GET); 2035 return (ret == 1) ? page : NULL; 2036 } 2037 #endif /* CONFIG_ELF_CORE */ 2038 2039 #ifdef CONFIG_MIGRATION 2040 /* 2041 * Returns the number of collected pages. Return value is always >= 0. 2042 */ 2043 static unsigned long collect_longterm_unpinnable_pages( 2044 struct list_head *movable_page_list, 2045 unsigned long nr_pages, 2046 struct page **pages) 2047 { 2048 unsigned long i, collected = 0; 2049 struct folio *prev_folio = NULL; 2050 bool drain_allow = true; 2051 2052 for (i = 0; i < nr_pages; i++) { 2053 struct folio *folio = page_folio(pages[i]); 2054 2055 if (folio == prev_folio) 2056 continue; 2057 prev_folio = folio; 2058 2059 if (folio_is_longterm_pinnable(folio)) 2060 continue; 2061 2062 collected++; 2063 2064 if (folio_is_device_coherent(folio)) 2065 continue; 2066 2067 if (folio_test_hugetlb(folio)) { 2068 isolate_hugetlb(folio, movable_page_list); 2069 continue; 2070 } 2071 2072 if (!folio_test_lru(folio) && drain_allow) { 2073 lru_add_drain_all(); 2074 drain_allow = false; 2075 } 2076 2077 if (!folio_isolate_lru(folio)) 2078 continue; 2079 2080 list_add_tail(&folio->lru, movable_page_list); 2081 node_stat_mod_folio(folio, 2082 NR_ISOLATED_ANON + folio_is_file_lru(folio), 2083 folio_nr_pages(folio)); 2084 } 2085 2086 return collected; 2087 } 2088 2089 /* 2090 * Unpins all pages and migrates device coherent pages and movable_page_list. 2091 * Returns -EAGAIN if all pages were successfully migrated or -errno for failure 2092 * (or partial success). 2093 */ 2094 static int migrate_longterm_unpinnable_pages( 2095 struct list_head *movable_page_list, 2096 unsigned long nr_pages, 2097 struct page **pages) 2098 { 2099 int ret; 2100 unsigned long i; 2101 2102 for (i = 0; i < nr_pages; i++) { 2103 struct folio *folio = page_folio(pages[i]); 2104 2105 if (folio_is_device_coherent(folio)) { 2106 /* 2107 * Migration will fail if the page is pinned, so convert 2108 * the pin on the source page to a normal reference. 2109 */ 2110 pages[i] = NULL; 2111 folio_get(folio); 2112 gup_put_folio(folio, 1, FOLL_PIN); 2113 2114 if (migrate_device_coherent_page(&folio->page)) { 2115 ret = -EBUSY; 2116 goto err; 2117 } 2118 2119 continue; 2120 } 2121 2122 /* 2123 * We can't migrate pages with unexpected references, so drop 2124 * the reference obtained by __get_user_pages_locked(). 2125 * Migrating pages have been added to movable_page_list after 2126 * calling folio_isolate_lru() which takes a reference so the 2127 * page won't be freed if it's migrating. 2128 */ 2129 unpin_user_page(pages[i]); 2130 pages[i] = NULL; 2131 } 2132 2133 if (!list_empty(movable_page_list)) { 2134 struct migration_target_control mtc = { 2135 .nid = NUMA_NO_NODE, 2136 .gfp_mask = GFP_USER | __GFP_NOWARN, 2137 }; 2138 2139 if (migrate_pages(movable_page_list, alloc_migration_target, 2140 NULL, (unsigned long)&mtc, MIGRATE_SYNC, 2141 MR_LONGTERM_PIN, NULL)) { 2142 ret = -ENOMEM; 2143 goto err; 2144 } 2145 } 2146 2147 putback_movable_pages(movable_page_list); 2148 2149 return -EAGAIN; 2150 2151 err: 2152 for (i = 0; i < nr_pages; i++) 2153 if (pages[i]) 2154 unpin_user_page(pages[i]); 2155 putback_movable_pages(movable_page_list); 2156 2157 return ret; 2158 } 2159 2160 /* 2161 * Check whether all pages are *allowed* to be pinned. Rather confusingly, all 2162 * pages in the range are required to be pinned via FOLL_PIN, before calling 2163 * this routine. 2164 * 2165 * If any pages in the range are not allowed to be pinned, then this routine 2166 * will migrate those pages away, unpin all the pages in the range and return 2167 * -EAGAIN. The caller should re-pin the entire range with FOLL_PIN and then 2168 * call this routine again. 2169 * 2170 * If an error other than -EAGAIN occurs, this indicates a migration failure. 2171 * The caller should give up, and propagate the error back up the call stack. 2172 * 2173 * If everything is OK and all pages in the range are allowed to be pinned, then 2174 * this routine leaves all pages pinned and returns zero for success. 2175 */ 2176 static long check_and_migrate_movable_pages(unsigned long nr_pages, 2177 struct page **pages) 2178 { 2179 unsigned long collected; 2180 LIST_HEAD(movable_page_list); 2181 2182 collected = collect_longterm_unpinnable_pages(&movable_page_list, 2183 nr_pages, pages); 2184 if (!collected) 2185 return 0; 2186 2187 return migrate_longterm_unpinnable_pages(&movable_page_list, nr_pages, 2188 pages); 2189 } 2190 #else 2191 static long check_and_migrate_movable_pages(unsigned long nr_pages, 2192 struct page **pages) 2193 { 2194 return 0; 2195 } 2196 #endif /* CONFIG_MIGRATION */ 2197 2198 /* 2199 * __gup_longterm_locked() is a wrapper for __get_user_pages_locked which 2200 * allows us to process the FOLL_LONGTERM flag. 2201 */ 2202 static long __gup_longterm_locked(struct mm_struct *mm, 2203 unsigned long start, 2204 unsigned long nr_pages, 2205 struct page **pages, 2206 int *locked, 2207 unsigned int gup_flags) 2208 { 2209 unsigned int flags; 2210 long rc, nr_pinned_pages; 2211 2212 if (!(gup_flags & FOLL_LONGTERM)) 2213 return __get_user_pages_locked(mm, start, nr_pages, pages, 2214 locked, gup_flags); 2215 2216 flags = memalloc_pin_save(); 2217 do { 2218 nr_pinned_pages = __get_user_pages_locked(mm, start, nr_pages, 2219 pages, locked, 2220 gup_flags); 2221 if (nr_pinned_pages <= 0) { 2222 rc = nr_pinned_pages; 2223 break; 2224 } 2225 2226 /* FOLL_LONGTERM implies FOLL_PIN */ 2227 rc = check_and_migrate_movable_pages(nr_pinned_pages, pages); 2228 } while (rc == -EAGAIN); 2229 memalloc_pin_restore(flags); 2230 return rc ? rc : nr_pinned_pages; 2231 } 2232 2233 /* 2234 * Check that the given flags are valid for the exported gup/pup interface, and 2235 * update them with the required flags that the caller must have set. 2236 */ 2237 static bool is_valid_gup_args(struct page **pages, int *locked, 2238 unsigned int *gup_flags_p, unsigned int to_set) 2239 { 2240 unsigned int gup_flags = *gup_flags_p; 2241 2242 /* 2243 * These flags not allowed to be specified externally to the gup 2244 * interfaces: 2245 * - FOLL_TOUCH/FOLL_PIN/FOLL_TRIED/FOLL_FAST_ONLY are internal only 2246 * - FOLL_REMOTE is internal only and used on follow_page() 2247 * - FOLL_UNLOCKABLE is internal only and used if locked is !NULL 2248 */ 2249 if (WARN_ON_ONCE(gup_flags & INTERNAL_GUP_FLAGS)) 2250 return false; 2251 2252 gup_flags |= to_set; 2253 if (locked) { 2254 /* At the external interface locked must be set */ 2255 if (WARN_ON_ONCE(*locked != 1)) 2256 return false; 2257 2258 gup_flags |= FOLL_UNLOCKABLE; 2259 } 2260 2261 /* FOLL_GET and FOLL_PIN are mutually exclusive. */ 2262 if (WARN_ON_ONCE((gup_flags & (FOLL_PIN | FOLL_GET)) == 2263 (FOLL_PIN | FOLL_GET))) 2264 return false; 2265 2266 /* LONGTERM can only be specified when pinning */ 2267 if (WARN_ON_ONCE(!(gup_flags & FOLL_PIN) && (gup_flags & FOLL_LONGTERM))) 2268 return false; 2269 2270 /* Pages input must be given if using GET/PIN */ 2271 if (WARN_ON_ONCE((gup_flags & (FOLL_GET | FOLL_PIN)) && !pages)) 2272 return false; 2273 2274 /* We want to allow the pgmap to be hot-unplugged at all times */ 2275 if (WARN_ON_ONCE((gup_flags & FOLL_LONGTERM) && 2276 (gup_flags & FOLL_PCI_P2PDMA))) 2277 return false; 2278 2279 *gup_flags_p = gup_flags; 2280 return true; 2281 } 2282 2283 #ifdef CONFIG_MMU 2284 /** 2285 * get_user_pages_remote() - pin user pages in memory 2286 * @mm: mm_struct of target mm 2287 * @start: starting user address 2288 * @nr_pages: number of pages from start to pin 2289 * @gup_flags: flags modifying lookup behaviour 2290 * @pages: array that receives pointers to the pages pinned. 2291 * Should be at least nr_pages long. Or NULL, if caller 2292 * only intends to ensure the pages are faulted in. 2293 * @locked: pointer to lock flag indicating whether lock is held and 2294 * subsequently whether VM_FAULT_RETRY functionality can be 2295 * utilised. Lock must initially be held. 2296 * 2297 * Returns either number of pages pinned (which may be less than the 2298 * number requested), or an error. Details about the return value: 2299 * 2300 * -- If nr_pages is 0, returns 0. 2301 * -- If nr_pages is >0, but no pages were pinned, returns -errno. 2302 * -- If nr_pages is >0, and some pages were pinned, returns the number of 2303 * pages pinned. Again, this may be less than nr_pages. 2304 * 2305 * The caller is responsible for releasing returned @pages, via put_page(). 2306 * 2307 * Must be called with mmap_lock held for read or write. 2308 * 2309 * get_user_pages_remote walks a process's page tables and takes a reference 2310 * to each struct page that each user address corresponds to at a given 2311 * instant. That is, it takes the page that would be accessed if a user 2312 * thread accesses the given user virtual address at that instant. 2313 * 2314 * This does not guarantee that the page exists in the user mappings when 2315 * get_user_pages_remote returns, and there may even be a completely different 2316 * page there in some cases (eg. if mmapped pagecache has been invalidated 2317 * and subsequently re-faulted). However it does guarantee that the page 2318 * won't be freed completely. And mostly callers simply care that the page 2319 * contains data that was valid *at some point in time*. Typically, an IO 2320 * or similar operation cannot guarantee anything stronger anyway because 2321 * locks can't be held over the syscall boundary. 2322 * 2323 * If gup_flags & FOLL_WRITE == 0, the page must not be written to. If the page 2324 * is written to, set_page_dirty (or set_page_dirty_lock, as appropriate) must 2325 * be called after the page is finished with, and before put_page is called. 2326 * 2327 * get_user_pages_remote is typically used for fewer-copy IO operations, 2328 * to get a handle on the memory by some means other than accesses 2329 * via the user virtual addresses. The pages may be submitted for 2330 * DMA to devices or accessed via their kernel linear mapping (via the 2331 * kmap APIs). Care should be taken to use the correct cache flushing APIs. 2332 * 2333 * See also get_user_pages_fast, for performance critical applications. 2334 * 2335 * get_user_pages_remote should be phased out in favor of 2336 * get_user_pages_locked|unlocked or get_user_pages_fast. Nothing 2337 * should use get_user_pages_remote because it cannot pass 2338 * FAULT_FLAG_ALLOW_RETRY to handle_mm_fault. 2339 */ 2340 long get_user_pages_remote(struct mm_struct *mm, 2341 unsigned long start, unsigned long nr_pages, 2342 unsigned int gup_flags, struct page **pages, 2343 int *locked) 2344 { 2345 int local_locked = 1; 2346 2347 if (!is_valid_gup_args(pages, locked, &gup_flags, 2348 FOLL_TOUCH | FOLL_REMOTE)) 2349 return -EINVAL; 2350 2351 return __get_user_pages_locked(mm, start, nr_pages, pages, 2352 locked ? locked : &local_locked, 2353 gup_flags); 2354 } 2355 EXPORT_SYMBOL(get_user_pages_remote); 2356 2357 #else /* CONFIG_MMU */ 2358 long get_user_pages_remote(struct mm_struct *mm, 2359 unsigned long start, unsigned long nr_pages, 2360 unsigned int gup_flags, struct page **pages, 2361 int *locked) 2362 { 2363 return 0; 2364 } 2365 #endif /* !CONFIG_MMU */ 2366 2367 /** 2368 * get_user_pages() - pin user pages in memory 2369 * @start: starting user address 2370 * @nr_pages: number of pages from start to pin 2371 * @gup_flags: flags modifying lookup behaviour 2372 * @pages: array that receives pointers to the pages pinned. 2373 * Should be at least nr_pages long. Or NULL, if caller 2374 * only intends to ensure the pages are faulted in. 2375 * 2376 * This is the same as get_user_pages_remote(), just with a less-flexible 2377 * calling convention where we assume that the mm being operated on belongs to 2378 * the current task, and doesn't allow passing of a locked parameter. We also 2379 * obviously don't pass FOLL_REMOTE in here. 2380 */ 2381 long get_user_pages(unsigned long start, unsigned long nr_pages, 2382 unsigned int gup_flags, struct page **pages) 2383 { 2384 int locked = 1; 2385 2386 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_TOUCH)) 2387 return -EINVAL; 2388 2389 return __get_user_pages_locked(current->mm, start, nr_pages, pages, 2390 &locked, gup_flags); 2391 } 2392 EXPORT_SYMBOL(get_user_pages); 2393 2394 /* 2395 * get_user_pages_unlocked() is suitable to replace the form: 2396 * 2397 * mmap_read_lock(mm); 2398 * get_user_pages(mm, ..., pages, NULL); 2399 * mmap_read_unlock(mm); 2400 * 2401 * with: 2402 * 2403 * get_user_pages_unlocked(mm, ..., pages); 2404 * 2405 * It is functionally equivalent to get_user_pages_fast so 2406 * get_user_pages_fast should be used instead if specific gup_flags 2407 * (e.g. FOLL_FORCE) are not required. 2408 */ 2409 long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, 2410 struct page **pages, unsigned int gup_flags) 2411 { 2412 int locked = 0; 2413 2414 if (!is_valid_gup_args(pages, NULL, &gup_flags, 2415 FOLL_TOUCH | FOLL_UNLOCKABLE)) 2416 return -EINVAL; 2417 2418 return __get_user_pages_locked(current->mm, start, nr_pages, pages, 2419 &locked, gup_flags); 2420 } 2421 EXPORT_SYMBOL(get_user_pages_unlocked); 2422 2423 /* 2424 * Fast GUP 2425 * 2426 * get_user_pages_fast attempts to pin user pages by walking the page 2427 * tables directly and avoids taking locks. Thus the walker needs to be 2428 * protected from page table pages being freed from under it, and should 2429 * block any THP splits. 2430 * 2431 * One way to achieve this is to have the walker disable interrupts, and 2432 * rely on IPIs from the TLB flushing code blocking before the page table 2433 * pages are freed. This is unsuitable for architectures that do not need 2434 * to broadcast an IPI when invalidating TLBs. 2435 * 2436 * Another way to achieve this is to batch up page table containing pages 2437 * belonging to more than one mm_user, then rcu_sched a callback to free those 2438 * pages. Disabling interrupts will allow the fast_gup walker to both block 2439 * the rcu_sched callback, and an IPI that we broadcast for splitting THPs 2440 * (which is a relatively rare event). The code below adopts this strategy. 2441 * 2442 * Before activating this code, please be aware that the following assumptions 2443 * are currently made: 2444 * 2445 * *) Either MMU_GATHER_RCU_TABLE_FREE is enabled, and tlb_remove_table() is used to 2446 * free pages containing page tables or TLB flushing requires IPI broadcast. 2447 * 2448 * *) ptes can be read atomically by the architecture. 2449 * 2450 * *) access_ok is sufficient to validate userspace address ranges. 2451 * 2452 * The last two assumptions can be relaxed by the addition of helper functions. 2453 * 2454 * This code is based heavily on the PowerPC implementation by Nick Piggin. 2455 */ 2456 #ifdef CONFIG_HAVE_FAST_GUP 2457 2458 /* 2459 * Used in the GUP-fast path to determine whether a pin is permitted for a 2460 * specific folio. 2461 * 2462 * This call assumes the caller has pinned the folio, that the lowest page table 2463 * level still points to this folio, and that interrupts have been disabled. 2464 * 2465 * Writing to pinned file-backed dirty tracked folios is inherently problematic 2466 * (see comment describing the writable_file_mapping_allowed() function). We 2467 * therefore try to avoid the most egregious case of a long-term mapping doing 2468 * so. 2469 * 2470 * This function cannot be as thorough as that one as the VMA is not available 2471 * in the fast path, so instead we whitelist known good cases and if in doubt, 2472 * fall back to the slow path. 2473 */ 2474 static bool folio_fast_pin_allowed(struct folio *folio, unsigned int flags) 2475 { 2476 struct address_space *mapping; 2477 unsigned long mapping_flags; 2478 2479 /* 2480 * If we aren't pinning then no problematic write can occur. A long term 2481 * pin is the most egregious case so this is the one we disallow. 2482 */ 2483 if ((flags & (FOLL_PIN | FOLL_LONGTERM | FOLL_WRITE)) != 2484 (FOLL_PIN | FOLL_LONGTERM | FOLL_WRITE)) 2485 return true; 2486 2487 /* The folio is pinned, so we can safely access folio fields. */ 2488 2489 if (WARN_ON_ONCE(folio_test_slab(folio))) 2490 return false; 2491 2492 /* hugetlb mappings do not require dirty-tracking. */ 2493 if (folio_test_hugetlb(folio)) 2494 return true; 2495 2496 /* 2497 * GUP-fast disables IRQs. When IRQS are disabled, RCU grace periods 2498 * cannot proceed, which means no actions performed under RCU can 2499 * proceed either. 2500 * 2501 * inodes and thus their mappings are freed under RCU, which means the 2502 * mapping cannot be freed beneath us and thus we can safely dereference 2503 * it. 2504 */ 2505 lockdep_assert_irqs_disabled(); 2506 2507 /* 2508 * However, there may be operations which _alter_ the mapping, so ensure 2509 * we read it once and only once. 2510 */ 2511 mapping = READ_ONCE(folio->mapping); 2512 2513 /* 2514 * The mapping may have been truncated, in any case we cannot determine 2515 * if this mapping is safe - fall back to slow path to determine how to 2516 * proceed. 2517 */ 2518 if (!mapping) 2519 return false; 2520 2521 /* Anonymous folios pose no problem. */ 2522 mapping_flags = (unsigned long)mapping & PAGE_MAPPING_FLAGS; 2523 if (mapping_flags) 2524 return mapping_flags & PAGE_MAPPING_ANON; 2525 2526 /* 2527 * At this point, we know the mapping is non-null and points to an 2528 * address_space object. The only remaining whitelisted file system is 2529 * shmem. 2530 */ 2531 return shmem_mapping(mapping); 2532 } 2533 2534 static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start, 2535 unsigned int flags, 2536 struct page **pages) 2537 { 2538 while ((*nr) - nr_start) { 2539 struct page *page = pages[--(*nr)]; 2540 2541 ClearPageReferenced(page); 2542 if (flags & FOLL_PIN) 2543 unpin_user_page(page); 2544 else 2545 put_page(page); 2546 } 2547 } 2548 2549 #ifdef CONFIG_ARCH_HAS_PTE_SPECIAL 2550 /* 2551 * Fast-gup relies on pte change detection to avoid concurrent pgtable 2552 * operations. 2553 * 2554 * To pin the page, fast-gup needs to do below in order: 2555 * (1) pin the page (by prefetching pte), then (2) check pte not changed. 2556 * 2557 * For the rest of pgtable operations where pgtable updates can be racy 2558 * with fast-gup, we need to do (1) clear pte, then (2) check whether page 2559 * is pinned. 2560 * 2561 * Above will work for all pte-level operations, including THP split. 2562 * 2563 * For THP collapse, it's a bit more complicated because fast-gup may be 2564 * walking a pgtable page that is being freed (pte is still valid but pmd 2565 * can be cleared already). To avoid race in such condition, we need to 2566 * also check pmd here to make sure pmd doesn't change (corresponds to 2567 * pmdp_collapse_flush() in the THP collapse code path). 2568 */ 2569 static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, 2570 unsigned long end, unsigned int flags, 2571 struct page **pages, int *nr) 2572 { 2573 struct dev_pagemap *pgmap = NULL; 2574 int nr_start = *nr, ret = 0; 2575 pte_t *ptep, *ptem; 2576 2577 ptem = ptep = pte_offset_map(&pmd, addr); 2578 if (!ptep) 2579 return 0; 2580 do { 2581 pte_t pte = ptep_get_lockless(ptep); 2582 struct page *page; 2583 struct folio *folio; 2584 2585 /* 2586 * Always fallback to ordinary GUP on PROT_NONE-mapped pages: 2587 * pte_access_permitted() better should reject these pages 2588 * either way: otherwise, GUP-fast might succeed in 2589 * cases where ordinary GUP would fail due to VMA access 2590 * permissions. 2591 */ 2592 if (pte_protnone(pte)) 2593 goto pte_unmap; 2594 2595 if (!pte_access_permitted(pte, flags & FOLL_WRITE)) 2596 goto pte_unmap; 2597 2598 if (pte_devmap(pte)) { 2599 if (unlikely(flags & FOLL_LONGTERM)) 2600 goto pte_unmap; 2601 2602 pgmap = get_dev_pagemap(pte_pfn(pte), pgmap); 2603 if (unlikely(!pgmap)) { 2604 undo_dev_pagemap(nr, nr_start, flags, pages); 2605 goto pte_unmap; 2606 } 2607 } else if (pte_special(pte)) 2608 goto pte_unmap; 2609 2610 VM_BUG_ON(!pfn_valid(pte_pfn(pte))); 2611 page = pte_page(pte); 2612 2613 folio = try_grab_folio(page, 1, flags); 2614 if (!folio) 2615 goto pte_unmap; 2616 2617 if (unlikely(folio_is_secretmem(folio))) { 2618 gup_put_folio(folio, 1, flags); 2619 goto pte_unmap; 2620 } 2621 2622 if (unlikely(pmd_val(pmd) != pmd_val(*pmdp)) || 2623 unlikely(pte_val(pte) != pte_val(ptep_get(ptep)))) { 2624 gup_put_folio(folio, 1, flags); 2625 goto pte_unmap; 2626 } 2627 2628 if (!folio_fast_pin_allowed(folio, flags)) { 2629 gup_put_folio(folio, 1, flags); 2630 goto pte_unmap; 2631 } 2632 2633 if (!pte_write(pte) && gup_must_unshare(NULL, flags, page)) { 2634 gup_put_folio(folio, 1, flags); 2635 goto pte_unmap; 2636 } 2637 2638 /* 2639 * We need to make the page accessible if and only if we are 2640 * going to access its content (the FOLL_PIN case). Please 2641 * see Documentation/core-api/pin_user_pages.rst for 2642 * details. 2643 */ 2644 if (flags & FOLL_PIN) { 2645 ret = arch_make_page_accessible(page); 2646 if (ret) { 2647 gup_put_folio(folio, 1, flags); 2648 goto pte_unmap; 2649 } 2650 } 2651 folio_set_referenced(folio); 2652 pages[*nr] = page; 2653 (*nr)++; 2654 } while (ptep++, addr += PAGE_SIZE, addr != end); 2655 2656 ret = 1; 2657 2658 pte_unmap: 2659 if (pgmap) 2660 put_dev_pagemap(pgmap); 2661 pte_unmap(ptem); 2662 return ret; 2663 } 2664 #else 2665 2666 /* 2667 * If we can't determine whether or not a pte is special, then fail immediately 2668 * for ptes. Note, we can still pin HugeTLB and THP as these are guaranteed not 2669 * to be special. 2670 * 2671 * For a futex to be placed on a THP tail page, get_futex_key requires a 2672 * get_user_pages_fast_only implementation that can pin pages. Thus it's still 2673 * useful to have gup_huge_pmd even if we can't operate on ptes. 2674 */ 2675 static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, 2676 unsigned long end, unsigned int flags, 2677 struct page **pages, int *nr) 2678 { 2679 return 0; 2680 } 2681 #endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */ 2682 2683 #if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE) 2684 static int __gup_device_huge(unsigned long pfn, unsigned long addr, 2685 unsigned long end, unsigned int flags, 2686 struct page **pages, int *nr) 2687 { 2688 int nr_start = *nr; 2689 struct dev_pagemap *pgmap = NULL; 2690 2691 do { 2692 struct page *page = pfn_to_page(pfn); 2693 2694 pgmap = get_dev_pagemap(pfn, pgmap); 2695 if (unlikely(!pgmap)) { 2696 undo_dev_pagemap(nr, nr_start, flags, pages); 2697 break; 2698 } 2699 2700 if (!(flags & FOLL_PCI_P2PDMA) && is_pci_p2pdma_page(page)) { 2701 undo_dev_pagemap(nr, nr_start, flags, pages); 2702 break; 2703 } 2704 2705 SetPageReferenced(page); 2706 pages[*nr] = page; 2707 if (unlikely(try_grab_page(page, flags))) { 2708 undo_dev_pagemap(nr, nr_start, flags, pages); 2709 break; 2710 } 2711 (*nr)++; 2712 pfn++; 2713 } while (addr += PAGE_SIZE, addr != end); 2714 2715 put_dev_pagemap(pgmap); 2716 return addr == end; 2717 } 2718 2719 static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, 2720 unsigned long end, unsigned int flags, 2721 struct page **pages, int *nr) 2722 { 2723 unsigned long fault_pfn; 2724 int nr_start = *nr; 2725 2726 fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); 2727 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr)) 2728 return 0; 2729 2730 if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) { 2731 undo_dev_pagemap(nr, nr_start, flags, pages); 2732 return 0; 2733 } 2734 return 1; 2735 } 2736 2737 static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, 2738 unsigned long end, unsigned int flags, 2739 struct page **pages, int *nr) 2740 { 2741 unsigned long fault_pfn; 2742 int nr_start = *nr; 2743 2744 fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); 2745 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr)) 2746 return 0; 2747 2748 if (unlikely(pud_val(orig) != pud_val(*pudp))) { 2749 undo_dev_pagemap(nr, nr_start, flags, pages); 2750 return 0; 2751 } 2752 return 1; 2753 } 2754 #else 2755 static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, 2756 unsigned long end, unsigned int flags, 2757 struct page **pages, int *nr) 2758 { 2759 BUILD_BUG(); 2760 return 0; 2761 } 2762 2763 static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr, 2764 unsigned long end, unsigned int flags, 2765 struct page **pages, int *nr) 2766 { 2767 BUILD_BUG(); 2768 return 0; 2769 } 2770 #endif 2771 2772 static int record_subpages(struct page *page, unsigned long addr, 2773 unsigned long end, struct page **pages) 2774 { 2775 int nr; 2776 2777 for (nr = 0; addr != end; nr++, addr += PAGE_SIZE) 2778 pages[nr] = nth_page(page, nr); 2779 2780 return nr; 2781 } 2782 2783 #ifdef CONFIG_ARCH_HAS_HUGEPD 2784 static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end, 2785 unsigned long sz) 2786 { 2787 unsigned long __boundary = (addr + sz) & ~(sz-1); 2788 return (__boundary - 1 < end - 1) ? __boundary : end; 2789 } 2790 2791 static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, 2792 unsigned long end, unsigned int flags, 2793 struct page **pages, int *nr) 2794 { 2795 unsigned long pte_end; 2796 struct page *page; 2797 struct folio *folio; 2798 pte_t pte; 2799 int refs; 2800 2801 pte_end = (addr + sz) & ~(sz-1); 2802 if (pte_end < end) 2803 end = pte_end; 2804 2805 pte = huge_ptep_get(ptep); 2806 2807 if (!pte_access_permitted(pte, flags & FOLL_WRITE)) 2808 return 0; 2809 2810 /* hugepages are never "special" */ 2811 VM_BUG_ON(!pfn_valid(pte_pfn(pte))); 2812 2813 page = nth_page(pte_page(pte), (addr & (sz - 1)) >> PAGE_SHIFT); 2814 refs = record_subpages(page, addr, end, pages + *nr); 2815 2816 folio = try_grab_folio(page, refs, flags); 2817 if (!folio) 2818 return 0; 2819 2820 if (unlikely(pte_val(pte) != pte_val(ptep_get(ptep)))) { 2821 gup_put_folio(folio, refs, flags); 2822 return 0; 2823 } 2824 2825 if (!folio_fast_pin_allowed(folio, flags)) { 2826 gup_put_folio(folio, refs, flags); 2827 return 0; 2828 } 2829 2830 if (!pte_write(pte) && gup_must_unshare(NULL, flags, &folio->page)) { 2831 gup_put_folio(folio, refs, flags); 2832 return 0; 2833 } 2834 2835 *nr += refs; 2836 folio_set_referenced(folio); 2837 return 1; 2838 } 2839 2840 static int gup_huge_pd(hugepd_t hugepd, unsigned long addr, 2841 unsigned int pdshift, unsigned long end, unsigned int flags, 2842 struct page **pages, int *nr) 2843 { 2844 pte_t *ptep; 2845 unsigned long sz = 1UL << hugepd_shift(hugepd); 2846 unsigned long next; 2847 2848 ptep = hugepte_offset(hugepd, addr, pdshift); 2849 do { 2850 next = hugepte_addr_end(addr, end, sz); 2851 if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr)) 2852 return 0; 2853 } while (ptep++, addr = next, addr != end); 2854 2855 return 1; 2856 } 2857 #else 2858 static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr, 2859 unsigned int pdshift, unsigned long end, unsigned int flags, 2860 struct page **pages, int *nr) 2861 { 2862 return 0; 2863 } 2864 #endif /* CONFIG_ARCH_HAS_HUGEPD */ 2865 2866 static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr, 2867 unsigned long end, unsigned int flags, 2868 struct page **pages, int *nr) 2869 { 2870 struct page *page; 2871 struct folio *folio; 2872 int refs; 2873 2874 if (!pmd_access_permitted(orig, flags & FOLL_WRITE)) 2875 return 0; 2876 2877 if (pmd_devmap(orig)) { 2878 if (unlikely(flags & FOLL_LONGTERM)) 2879 return 0; 2880 return __gup_device_huge_pmd(orig, pmdp, addr, end, flags, 2881 pages, nr); 2882 } 2883 2884 page = nth_page(pmd_page(orig), (addr & ~PMD_MASK) >> PAGE_SHIFT); 2885 refs = record_subpages(page, addr, end, pages + *nr); 2886 2887 folio = try_grab_folio(page, refs, flags); 2888 if (!folio) 2889 return 0; 2890 2891 if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) { 2892 gup_put_folio(folio, refs, flags); 2893 return 0; 2894 } 2895 2896 if (!folio_fast_pin_allowed(folio, flags)) { 2897 gup_put_folio(folio, refs, flags); 2898 return 0; 2899 } 2900 if (!pmd_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) { 2901 gup_put_folio(folio, refs, flags); 2902 return 0; 2903 } 2904 2905 *nr += refs; 2906 folio_set_referenced(folio); 2907 return 1; 2908 } 2909 2910 static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr, 2911 unsigned long end, unsigned int flags, 2912 struct page **pages, int *nr) 2913 { 2914 struct page *page; 2915 struct folio *folio; 2916 int refs; 2917 2918 if (!pud_access_permitted(orig, flags & FOLL_WRITE)) 2919 return 0; 2920 2921 if (pud_devmap(orig)) { 2922 if (unlikely(flags & FOLL_LONGTERM)) 2923 return 0; 2924 return __gup_device_huge_pud(orig, pudp, addr, end, flags, 2925 pages, nr); 2926 } 2927 2928 page = nth_page(pud_page(orig), (addr & ~PUD_MASK) >> PAGE_SHIFT); 2929 refs = record_subpages(page, addr, end, pages + *nr); 2930 2931 folio = try_grab_folio(page, refs, flags); 2932 if (!folio) 2933 return 0; 2934 2935 if (unlikely(pud_val(orig) != pud_val(*pudp))) { 2936 gup_put_folio(folio, refs, flags); 2937 return 0; 2938 } 2939 2940 if (!folio_fast_pin_allowed(folio, flags)) { 2941 gup_put_folio(folio, refs, flags); 2942 return 0; 2943 } 2944 2945 if (!pud_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) { 2946 gup_put_folio(folio, refs, flags); 2947 return 0; 2948 } 2949 2950 *nr += refs; 2951 folio_set_referenced(folio); 2952 return 1; 2953 } 2954 2955 static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr, 2956 unsigned long end, unsigned int flags, 2957 struct page **pages, int *nr) 2958 { 2959 int refs; 2960 struct page *page; 2961 struct folio *folio; 2962 2963 if (!pgd_access_permitted(orig, flags & FOLL_WRITE)) 2964 return 0; 2965 2966 BUILD_BUG_ON(pgd_devmap(orig)); 2967 2968 page = nth_page(pgd_page(orig), (addr & ~PGDIR_MASK) >> PAGE_SHIFT); 2969 refs = record_subpages(page, addr, end, pages + *nr); 2970 2971 folio = try_grab_folio(page, refs, flags); 2972 if (!folio) 2973 return 0; 2974 2975 if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) { 2976 gup_put_folio(folio, refs, flags); 2977 return 0; 2978 } 2979 2980 if (!pgd_write(orig) && gup_must_unshare(NULL, flags, &folio->page)) { 2981 gup_put_folio(folio, refs, flags); 2982 return 0; 2983 } 2984 2985 if (!folio_fast_pin_allowed(folio, flags)) { 2986 gup_put_folio(folio, refs, flags); 2987 return 0; 2988 } 2989 2990 *nr += refs; 2991 folio_set_referenced(folio); 2992 return 1; 2993 } 2994 2995 static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned long end, 2996 unsigned int flags, struct page **pages, int *nr) 2997 { 2998 unsigned long next; 2999 pmd_t *pmdp; 3000 3001 pmdp = pmd_offset_lockless(pudp, pud, addr); 3002 do { 3003 pmd_t pmd = pmdp_get_lockless(pmdp); 3004 3005 next = pmd_addr_end(addr, end); 3006 if (!pmd_present(pmd)) 3007 return 0; 3008 3009 if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) || 3010 pmd_devmap(pmd))) { 3011 /* See gup_pte_range() */ 3012 if (pmd_protnone(pmd)) 3013 return 0; 3014 3015 if (!gup_huge_pmd(pmd, pmdp, addr, next, flags, 3016 pages, nr)) 3017 return 0; 3018 3019 } else if (unlikely(is_hugepd(__hugepd(pmd_val(pmd))))) { 3020 /* 3021 * architecture have different format for hugetlbfs 3022 * pmd format and THP pmd format 3023 */ 3024 if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr, 3025 PMD_SHIFT, next, flags, pages, nr)) 3026 return 0; 3027 } else if (!gup_pte_range(pmd, pmdp, addr, next, flags, pages, nr)) 3028 return 0; 3029 } while (pmdp++, addr = next, addr != end); 3030 3031 return 1; 3032 } 3033 3034 static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned long end, 3035 unsigned int flags, struct page **pages, int *nr) 3036 { 3037 unsigned long next; 3038 pud_t *pudp; 3039 3040 pudp = pud_offset_lockless(p4dp, p4d, addr); 3041 do { 3042 pud_t pud = READ_ONCE(*pudp); 3043 3044 next = pud_addr_end(addr, end); 3045 if (unlikely(!pud_present(pud))) 3046 return 0; 3047 if (unlikely(pud_huge(pud) || pud_devmap(pud))) { 3048 if (!gup_huge_pud(pud, pudp, addr, next, flags, 3049 pages, nr)) 3050 return 0; 3051 } else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) { 3052 if (!gup_huge_pd(__hugepd(pud_val(pud)), addr, 3053 PUD_SHIFT, next, flags, pages, nr)) 3054 return 0; 3055 } else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr)) 3056 return 0; 3057 } while (pudp++, addr = next, addr != end); 3058 3059 return 1; 3060 } 3061 3062 static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned long end, 3063 unsigned int flags, struct page **pages, int *nr) 3064 { 3065 unsigned long next; 3066 p4d_t *p4dp; 3067 3068 p4dp = p4d_offset_lockless(pgdp, pgd, addr); 3069 do { 3070 p4d_t p4d = READ_ONCE(*p4dp); 3071 3072 next = p4d_addr_end(addr, end); 3073 if (p4d_none(p4d)) 3074 return 0; 3075 BUILD_BUG_ON(p4d_huge(p4d)); 3076 if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) { 3077 if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr, 3078 P4D_SHIFT, next, flags, pages, nr)) 3079 return 0; 3080 } else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr)) 3081 return 0; 3082 } while (p4dp++, addr = next, addr != end); 3083 3084 return 1; 3085 } 3086 3087 static void gup_pgd_range(unsigned long addr, unsigned long end, 3088 unsigned int flags, struct page **pages, int *nr) 3089 { 3090 unsigned long next; 3091 pgd_t *pgdp; 3092 3093 pgdp = pgd_offset(current->mm, addr); 3094 do { 3095 pgd_t pgd = READ_ONCE(*pgdp); 3096 3097 next = pgd_addr_end(addr, end); 3098 if (pgd_none(pgd)) 3099 return; 3100 if (unlikely(pgd_huge(pgd))) { 3101 if (!gup_huge_pgd(pgd, pgdp, addr, next, flags, 3102 pages, nr)) 3103 return; 3104 } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) { 3105 if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr, 3106 PGDIR_SHIFT, next, flags, pages, nr)) 3107 return; 3108 } else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr)) 3109 return; 3110 } while (pgdp++, addr = next, addr != end); 3111 } 3112 #else 3113 static inline void gup_pgd_range(unsigned long addr, unsigned long end, 3114 unsigned int flags, struct page **pages, int *nr) 3115 { 3116 } 3117 #endif /* CONFIG_HAVE_FAST_GUP */ 3118 3119 #ifndef gup_fast_permitted 3120 /* 3121 * Check if it's allowed to use get_user_pages_fast_only() for the range, or 3122 * we need to fall back to the slow version: 3123 */ 3124 static bool gup_fast_permitted(unsigned long start, unsigned long end) 3125 { 3126 return true; 3127 } 3128 #endif 3129 3130 static unsigned long lockless_pages_from_mm(unsigned long start, 3131 unsigned long end, 3132 unsigned int gup_flags, 3133 struct page **pages) 3134 { 3135 unsigned long flags; 3136 int nr_pinned = 0; 3137 unsigned seq; 3138 3139 if (!IS_ENABLED(CONFIG_HAVE_FAST_GUP) || 3140 !gup_fast_permitted(start, end)) 3141 return 0; 3142 3143 if (gup_flags & FOLL_PIN) { 3144 seq = raw_read_seqcount(¤t->mm->write_protect_seq); 3145 if (seq & 1) 3146 return 0; 3147 } 3148 3149 /* 3150 * Disable interrupts. The nested form is used, in order to allow full, 3151 * general purpose use of this routine. 3152 * 3153 * With interrupts disabled, we block page table pages from being freed 3154 * from under us. See struct mmu_table_batch comments in 3155 * include/asm-generic/tlb.h for more details. 3156 * 3157 * We do not adopt an rcu_read_lock() here as we also want to block IPIs 3158 * that come from THPs splitting. 3159 */ 3160 local_irq_save(flags); 3161 gup_pgd_range(start, end, gup_flags, pages, &nr_pinned); 3162 local_irq_restore(flags); 3163 3164 /* 3165 * When pinning pages for DMA there could be a concurrent write protect 3166 * from fork() via copy_page_range(), in this case always fail fast GUP. 3167 */ 3168 if (gup_flags & FOLL_PIN) { 3169 if (read_seqcount_retry(¤t->mm->write_protect_seq, seq)) { 3170 unpin_user_pages_lockless(pages, nr_pinned); 3171 return 0; 3172 } else { 3173 sanity_check_pinned_pages(pages, nr_pinned); 3174 } 3175 } 3176 return nr_pinned; 3177 } 3178 3179 static int internal_get_user_pages_fast(unsigned long start, 3180 unsigned long nr_pages, 3181 unsigned int gup_flags, 3182 struct page **pages) 3183 { 3184 unsigned long len, end; 3185 unsigned long nr_pinned; 3186 int locked = 0; 3187 int ret; 3188 3189 if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM | 3190 FOLL_FORCE | FOLL_PIN | FOLL_GET | 3191 FOLL_FAST_ONLY | FOLL_NOFAULT | 3192 FOLL_PCI_P2PDMA | FOLL_HONOR_NUMA_FAULT))) 3193 return -EINVAL; 3194 3195 if (gup_flags & FOLL_PIN) 3196 mm_set_has_pinned_flag(¤t->mm->flags); 3197 3198 if (!(gup_flags & FOLL_FAST_ONLY)) 3199 might_lock_read(¤t->mm->mmap_lock); 3200 3201 start = untagged_addr(start) & PAGE_MASK; 3202 len = nr_pages << PAGE_SHIFT; 3203 if (check_add_overflow(start, len, &end)) 3204 return -EOVERFLOW; 3205 if (end > TASK_SIZE_MAX) 3206 return -EFAULT; 3207 if (unlikely(!access_ok((void __user *)start, len))) 3208 return -EFAULT; 3209 3210 nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages); 3211 if (nr_pinned == nr_pages || gup_flags & FOLL_FAST_ONLY) 3212 return nr_pinned; 3213 3214 /* Slow path: try to get the remaining pages with get_user_pages */ 3215 start += nr_pinned << PAGE_SHIFT; 3216 pages += nr_pinned; 3217 ret = __gup_longterm_locked(current->mm, start, nr_pages - nr_pinned, 3218 pages, &locked, 3219 gup_flags | FOLL_TOUCH | FOLL_UNLOCKABLE); 3220 if (ret < 0) { 3221 /* 3222 * The caller has to unpin the pages we already pinned so 3223 * returning -errno is not an option 3224 */ 3225 if (nr_pinned) 3226 return nr_pinned; 3227 return ret; 3228 } 3229 return ret + nr_pinned; 3230 } 3231 3232 /** 3233 * get_user_pages_fast_only() - pin user pages in memory 3234 * @start: starting user address 3235 * @nr_pages: number of pages from start to pin 3236 * @gup_flags: flags modifying pin behaviour 3237 * @pages: array that receives pointers to the pages pinned. 3238 * Should be at least nr_pages long. 3239 * 3240 * Like get_user_pages_fast() except it's IRQ-safe in that it won't fall back to 3241 * the regular GUP. 3242 * 3243 * If the architecture does not support this function, simply return with no 3244 * pages pinned. 3245 * 3246 * Careful, careful! COW breaking can go either way, so a non-write 3247 * access can get ambiguous page results. If you call this function without 3248 * 'write' set, you'd better be sure that you're ok with that ambiguity. 3249 */ 3250 int get_user_pages_fast_only(unsigned long start, int nr_pages, 3251 unsigned int gup_flags, struct page **pages) 3252 { 3253 /* 3254 * Internally (within mm/gup.c), gup fast variants must set FOLL_GET, 3255 * because gup fast is always a "pin with a +1 page refcount" request. 3256 * 3257 * FOLL_FAST_ONLY is required in order to match the API description of 3258 * this routine: no fall back to regular ("slow") GUP. 3259 */ 3260 if (!is_valid_gup_args(pages, NULL, &gup_flags, 3261 FOLL_GET | FOLL_FAST_ONLY)) 3262 return -EINVAL; 3263 3264 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); 3265 } 3266 EXPORT_SYMBOL_GPL(get_user_pages_fast_only); 3267 3268 /** 3269 * get_user_pages_fast() - pin user pages in memory 3270 * @start: starting user address 3271 * @nr_pages: number of pages from start to pin 3272 * @gup_flags: flags modifying pin behaviour 3273 * @pages: array that receives pointers to the pages pinned. 3274 * Should be at least nr_pages long. 3275 * 3276 * Attempt to pin user pages in memory without taking mm->mmap_lock. 3277 * If not successful, it will fall back to taking the lock and 3278 * calling get_user_pages(). 3279 * 3280 * Returns number of pages pinned. This may be fewer than the number requested. 3281 * If nr_pages is 0 or negative, returns 0. If no pages were pinned, returns 3282 * -errno. 3283 */ 3284 int get_user_pages_fast(unsigned long start, int nr_pages, 3285 unsigned int gup_flags, struct page **pages) 3286 { 3287 /* 3288 * The caller may or may not have explicitly set FOLL_GET; either way is 3289 * OK. However, internally (within mm/gup.c), gup fast variants must set 3290 * FOLL_GET, because gup fast is always a "pin with a +1 page refcount" 3291 * request. 3292 */ 3293 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_GET)) 3294 return -EINVAL; 3295 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); 3296 } 3297 EXPORT_SYMBOL_GPL(get_user_pages_fast); 3298 3299 /** 3300 * pin_user_pages_fast() - pin user pages in memory without taking locks 3301 * 3302 * @start: starting user address 3303 * @nr_pages: number of pages from start to pin 3304 * @gup_flags: flags modifying pin behaviour 3305 * @pages: array that receives pointers to the pages pinned. 3306 * Should be at least nr_pages long. 3307 * 3308 * Nearly the same as get_user_pages_fast(), except that FOLL_PIN is set. See 3309 * get_user_pages_fast() for documentation on the function arguments, because 3310 * the arguments here are identical. 3311 * 3312 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please 3313 * see Documentation/core-api/pin_user_pages.rst for further details. 3314 * 3315 * Note that if a zero_page is amongst the returned pages, it will not have 3316 * pins in it and unpin_user_page() will not remove pins from it. 3317 */ 3318 int pin_user_pages_fast(unsigned long start, int nr_pages, 3319 unsigned int gup_flags, struct page **pages) 3320 { 3321 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN)) 3322 return -EINVAL; 3323 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); 3324 } 3325 EXPORT_SYMBOL_GPL(pin_user_pages_fast); 3326 3327 /** 3328 * pin_user_pages_remote() - pin pages of a remote process 3329 * 3330 * @mm: mm_struct of target mm 3331 * @start: starting user address 3332 * @nr_pages: number of pages from start to pin 3333 * @gup_flags: flags modifying lookup behaviour 3334 * @pages: array that receives pointers to the pages pinned. 3335 * Should be at least nr_pages long. 3336 * @locked: pointer to lock flag indicating whether lock is held and 3337 * subsequently whether VM_FAULT_RETRY functionality can be 3338 * utilised. Lock must initially be held. 3339 * 3340 * Nearly the same as get_user_pages_remote(), except that FOLL_PIN is set. See 3341 * get_user_pages_remote() for documentation on the function arguments, because 3342 * the arguments here are identical. 3343 * 3344 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please 3345 * see Documentation/core-api/pin_user_pages.rst for details. 3346 * 3347 * Note that if a zero_page is amongst the returned pages, it will not have 3348 * pins in it and unpin_user_page*() will not remove pins from it. 3349 */ 3350 long pin_user_pages_remote(struct mm_struct *mm, 3351 unsigned long start, unsigned long nr_pages, 3352 unsigned int gup_flags, struct page **pages, 3353 int *locked) 3354 { 3355 int local_locked = 1; 3356 3357 if (!is_valid_gup_args(pages, locked, &gup_flags, 3358 FOLL_PIN | FOLL_TOUCH | FOLL_REMOTE)) 3359 return 0; 3360 return __gup_longterm_locked(mm, start, nr_pages, pages, 3361 locked ? locked : &local_locked, 3362 gup_flags); 3363 } 3364 EXPORT_SYMBOL(pin_user_pages_remote); 3365 3366 /** 3367 * pin_user_pages() - pin user pages in memory for use by other devices 3368 * 3369 * @start: starting user address 3370 * @nr_pages: number of pages from start to pin 3371 * @gup_flags: flags modifying lookup behaviour 3372 * @pages: array that receives pointers to the pages pinned. 3373 * Should be at least nr_pages long. 3374 * 3375 * Nearly the same as get_user_pages(), except that FOLL_TOUCH is not set, and 3376 * FOLL_PIN is set. 3377 * 3378 * FOLL_PIN means that the pages must be released via unpin_user_page(). Please 3379 * see Documentation/core-api/pin_user_pages.rst for details. 3380 * 3381 * Note that if a zero_page is amongst the returned pages, it will not have 3382 * pins in it and unpin_user_page*() will not remove pins from it. 3383 */ 3384 long pin_user_pages(unsigned long start, unsigned long nr_pages, 3385 unsigned int gup_flags, struct page **pages) 3386 { 3387 int locked = 1; 3388 3389 if (!is_valid_gup_args(pages, NULL, &gup_flags, FOLL_PIN)) 3390 return 0; 3391 return __gup_longterm_locked(current->mm, start, nr_pages, 3392 pages, &locked, gup_flags); 3393 } 3394 EXPORT_SYMBOL(pin_user_pages); 3395 3396 /* 3397 * pin_user_pages_unlocked() is the FOLL_PIN variant of 3398 * get_user_pages_unlocked(). Behavior is the same, except that this one sets 3399 * FOLL_PIN and rejects FOLL_GET. 3400 * 3401 * Note that if a zero_page is amongst the returned pages, it will not have 3402 * pins in it and unpin_user_page*() will not remove pins from it. 3403 */ 3404 long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages, 3405 struct page **pages, unsigned int gup_flags) 3406 { 3407 int locked = 0; 3408 3409 if (!is_valid_gup_args(pages, NULL, &gup_flags, 3410 FOLL_PIN | FOLL_TOUCH | FOLL_UNLOCKABLE)) 3411 return 0; 3412 3413 return __gup_longterm_locked(current->mm, start, nr_pages, pages, 3414 &locked, gup_flags); 3415 } 3416 EXPORT_SYMBOL(pin_user_pages_unlocked); 3417