1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2009 Red Hat, Inc. 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/mm.h> 9 #include <linux/sched.h> 10 #include <linux/sched/mm.h> 11 #include <linux/sched/coredump.h> 12 #include <linux/sched/numa_balancing.h> 13 #include <linux/highmem.h> 14 #include <linux/hugetlb.h> 15 #include <linux/mmu_notifier.h> 16 #include <linux/rmap.h> 17 #include <linux/swap.h> 18 #include <linux/shrinker.h> 19 #include <linux/mm_inline.h> 20 #include <linux/swapops.h> 21 #include <linux/dax.h> 22 #include <linux/khugepaged.h> 23 #include <linux/freezer.h> 24 #include <linux/pfn_t.h> 25 #include <linux/mman.h> 26 #include <linux/memremap.h> 27 #include <linux/pagemap.h> 28 #include <linux/debugfs.h> 29 #include <linux/migrate.h> 30 #include <linux/hashtable.h> 31 #include <linux/userfaultfd_k.h> 32 #include <linux/page_idle.h> 33 #include <linux/shmem_fs.h> 34 #include <linux/oom.h> 35 #include <linux/numa.h> 36 #include <linux/page_owner.h> 37 38 #include <asm/tlb.h> 39 #include <asm/pgalloc.h> 40 #include "internal.h" 41 42 /* 43 * By default, transparent hugepage support is disabled in order to avoid 44 * risking an increased memory footprint for applications that are not 45 * guaranteed to benefit from it. When transparent hugepage support is 46 * enabled, it is for all mappings, and khugepaged scans all mappings. 47 * Defrag is invoked by khugepaged hugepage allocations and by page faults 48 * for all hugepage allocations. 49 */ 50 unsigned long transparent_hugepage_flags __read_mostly = 51 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS 52 (1<<TRANSPARENT_HUGEPAGE_FLAG)| 53 #endif 54 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE 55 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| 56 #endif 57 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)| 58 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| 59 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 60 61 static struct shrinker deferred_split_shrinker; 62 63 static atomic_t huge_zero_refcount; 64 struct page *huge_zero_page __read_mostly; 65 unsigned long huge_zero_pfn __read_mostly = ~0UL; 66 67 static inline bool file_thp_enabled(struct vm_area_struct *vma) 68 { 69 return transhuge_vma_enabled(vma, vma->vm_flags) && vma->vm_file && 70 !inode_is_open_for_write(vma->vm_file->f_inode) && 71 (vma->vm_flags & VM_EXEC); 72 } 73 74 bool transparent_hugepage_active(struct vm_area_struct *vma) 75 { 76 /* The addr is used to check if the vma size fits */ 77 unsigned long addr = (vma->vm_end & HPAGE_PMD_MASK) - HPAGE_PMD_SIZE; 78 79 if (!transhuge_vma_suitable(vma, addr)) 80 return false; 81 if (vma_is_anonymous(vma)) 82 return __transparent_hugepage_enabled(vma); 83 if (vma_is_shmem(vma)) 84 return shmem_huge_enabled(vma); 85 if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS)) 86 return file_thp_enabled(vma); 87 88 return false; 89 } 90 91 static bool get_huge_zero_page(void) 92 { 93 struct page *zero_page; 94 retry: 95 if (likely(atomic_inc_not_zero(&huge_zero_refcount))) 96 return true; 97 98 zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE, 99 HPAGE_PMD_ORDER); 100 if (!zero_page) { 101 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); 102 return false; 103 } 104 count_vm_event(THP_ZERO_PAGE_ALLOC); 105 preempt_disable(); 106 if (cmpxchg(&huge_zero_page, NULL, zero_page)) { 107 preempt_enable(); 108 __free_pages(zero_page, compound_order(zero_page)); 109 goto retry; 110 } 111 WRITE_ONCE(huge_zero_pfn, page_to_pfn(zero_page)); 112 113 /* We take additional reference here. It will be put back by shrinker */ 114 atomic_set(&huge_zero_refcount, 2); 115 preempt_enable(); 116 return true; 117 } 118 119 static void put_huge_zero_page(void) 120 { 121 /* 122 * Counter should never go to zero here. Only shrinker can put 123 * last reference. 124 */ 125 BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); 126 } 127 128 struct page *mm_get_huge_zero_page(struct mm_struct *mm) 129 { 130 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 131 return READ_ONCE(huge_zero_page); 132 133 if (!get_huge_zero_page()) 134 return NULL; 135 136 if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 137 put_huge_zero_page(); 138 139 return READ_ONCE(huge_zero_page); 140 } 141 142 void mm_put_huge_zero_page(struct mm_struct *mm) 143 { 144 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 145 put_huge_zero_page(); 146 } 147 148 static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink, 149 struct shrink_control *sc) 150 { 151 /* we can free zero page only if last reference remains */ 152 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; 153 } 154 155 static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink, 156 struct shrink_control *sc) 157 { 158 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { 159 struct page *zero_page = xchg(&huge_zero_page, NULL); 160 BUG_ON(zero_page == NULL); 161 WRITE_ONCE(huge_zero_pfn, ~0UL); 162 __free_pages(zero_page, compound_order(zero_page)); 163 return HPAGE_PMD_NR; 164 } 165 166 return 0; 167 } 168 169 static struct shrinker huge_zero_page_shrinker = { 170 .count_objects = shrink_huge_zero_page_count, 171 .scan_objects = shrink_huge_zero_page_scan, 172 .seeks = DEFAULT_SEEKS, 173 }; 174 175 #ifdef CONFIG_SYSFS 176 static ssize_t enabled_show(struct kobject *kobj, 177 struct kobj_attribute *attr, char *buf) 178 { 179 const char *output; 180 181 if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags)) 182 output = "[always] madvise never"; 183 else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 184 &transparent_hugepage_flags)) 185 output = "always [madvise] never"; 186 else 187 output = "always madvise [never]"; 188 189 return sysfs_emit(buf, "%s\n", output); 190 } 191 192 static ssize_t enabled_store(struct kobject *kobj, 193 struct kobj_attribute *attr, 194 const char *buf, size_t count) 195 { 196 ssize_t ret = count; 197 198 if (sysfs_streq(buf, "always")) { 199 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 200 set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 201 } else if (sysfs_streq(buf, "madvise")) { 202 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 203 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 204 } else if (sysfs_streq(buf, "never")) { 205 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 206 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 207 } else 208 ret = -EINVAL; 209 210 if (ret > 0) { 211 int err = start_stop_khugepaged(); 212 if (err) 213 ret = err; 214 } 215 return ret; 216 } 217 static struct kobj_attribute enabled_attr = 218 __ATTR(enabled, 0644, enabled_show, enabled_store); 219 220 ssize_t single_hugepage_flag_show(struct kobject *kobj, 221 struct kobj_attribute *attr, char *buf, 222 enum transparent_hugepage_flag flag) 223 { 224 return sysfs_emit(buf, "%d\n", 225 !!test_bit(flag, &transparent_hugepage_flags)); 226 } 227 228 ssize_t single_hugepage_flag_store(struct kobject *kobj, 229 struct kobj_attribute *attr, 230 const char *buf, size_t count, 231 enum transparent_hugepage_flag flag) 232 { 233 unsigned long value; 234 int ret; 235 236 ret = kstrtoul(buf, 10, &value); 237 if (ret < 0) 238 return ret; 239 if (value > 1) 240 return -EINVAL; 241 242 if (value) 243 set_bit(flag, &transparent_hugepage_flags); 244 else 245 clear_bit(flag, &transparent_hugepage_flags); 246 247 return count; 248 } 249 250 static ssize_t defrag_show(struct kobject *kobj, 251 struct kobj_attribute *attr, char *buf) 252 { 253 const char *output; 254 255 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, 256 &transparent_hugepage_flags)) 257 output = "[always] defer defer+madvise madvise never"; 258 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, 259 &transparent_hugepage_flags)) 260 output = "always [defer] defer+madvise madvise never"; 261 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, 262 &transparent_hugepage_flags)) 263 output = "always defer [defer+madvise] madvise never"; 264 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, 265 &transparent_hugepage_flags)) 266 output = "always defer defer+madvise [madvise] never"; 267 else 268 output = "always defer defer+madvise madvise [never]"; 269 270 return sysfs_emit(buf, "%s\n", output); 271 } 272 273 static ssize_t defrag_store(struct kobject *kobj, 274 struct kobj_attribute *attr, 275 const char *buf, size_t count) 276 { 277 if (sysfs_streq(buf, "always")) { 278 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 279 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 280 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 281 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 282 } else if (sysfs_streq(buf, "defer+madvise")) { 283 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 284 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 285 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 286 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 287 } else if (sysfs_streq(buf, "defer")) { 288 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 289 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 290 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 291 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 292 } else if (sysfs_streq(buf, "madvise")) { 293 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 294 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 295 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 296 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 297 } else if (sysfs_streq(buf, "never")) { 298 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 299 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 300 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 301 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 302 } else 303 return -EINVAL; 304 305 return count; 306 } 307 static struct kobj_attribute defrag_attr = 308 __ATTR(defrag, 0644, defrag_show, defrag_store); 309 310 static ssize_t use_zero_page_show(struct kobject *kobj, 311 struct kobj_attribute *attr, char *buf) 312 { 313 return single_hugepage_flag_show(kobj, attr, buf, 314 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 315 } 316 static ssize_t use_zero_page_store(struct kobject *kobj, 317 struct kobj_attribute *attr, const char *buf, size_t count) 318 { 319 return single_hugepage_flag_store(kobj, attr, buf, count, 320 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 321 } 322 static struct kobj_attribute use_zero_page_attr = 323 __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store); 324 325 static ssize_t hpage_pmd_size_show(struct kobject *kobj, 326 struct kobj_attribute *attr, char *buf) 327 { 328 return sysfs_emit(buf, "%lu\n", HPAGE_PMD_SIZE); 329 } 330 static struct kobj_attribute hpage_pmd_size_attr = 331 __ATTR_RO(hpage_pmd_size); 332 333 static struct attribute *hugepage_attr[] = { 334 &enabled_attr.attr, 335 &defrag_attr.attr, 336 &use_zero_page_attr.attr, 337 &hpage_pmd_size_attr.attr, 338 #ifdef CONFIG_SHMEM 339 &shmem_enabled_attr.attr, 340 #endif 341 NULL, 342 }; 343 344 static const struct attribute_group hugepage_attr_group = { 345 .attrs = hugepage_attr, 346 }; 347 348 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj) 349 { 350 int err; 351 352 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj); 353 if (unlikely(!*hugepage_kobj)) { 354 pr_err("failed to create transparent hugepage kobject\n"); 355 return -ENOMEM; 356 } 357 358 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group); 359 if (err) { 360 pr_err("failed to register transparent hugepage group\n"); 361 goto delete_obj; 362 } 363 364 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group); 365 if (err) { 366 pr_err("failed to register transparent hugepage group\n"); 367 goto remove_hp_group; 368 } 369 370 return 0; 371 372 remove_hp_group: 373 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group); 374 delete_obj: 375 kobject_put(*hugepage_kobj); 376 return err; 377 } 378 379 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj) 380 { 381 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group); 382 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group); 383 kobject_put(hugepage_kobj); 384 } 385 #else 386 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj) 387 { 388 return 0; 389 } 390 391 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) 392 { 393 } 394 #endif /* CONFIG_SYSFS */ 395 396 static int __init hugepage_init(void) 397 { 398 int err; 399 struct kobject *hugepage_kobj; 400 401 if (!has_transparent_hugepage()) { 402 /* 403 * Hardware doesn't support hugepages, hence disable 404 * DAX PMD support. 405 */ 406 transparent_hugepage_flags = 1 << TRANSPARENT_HUGEPAGE_NEVER_DAX; 407 return -EINVAL; 408 } 409 410 /* 411 * hugepages can't be allocated by the buddy allocator 412 */ 413 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER >= MAX_ORDER); 414 /* 415 * we use page->mapping and page->index in second tail page 416 * as list_head: assuming THP order >= 2 417 */ 418 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER < 2); 419 420 err = hugepage_init_sysfs(&hugepage_kobj); 421 if (err) 422 goto err_sysfs; 423 424 err = khugepaged_init(); 425 if (err) 426 goto err_slab; 427 428 err = register_shrinker(&huge_zero_page_shrinker); 429 if (err) 430 goto err_hzp_shrinker; 431 err = register_shrinker(&deferred_split_shrinker); 432 if (err) 433 goto err_split_shrinker; 434 435 /* 436 * By default disable transparent hugepages on smaller systems, 437 * where the extra memory used could hurt more than TLB overhead 438 * is likely to save. The admin can still enable it through /sys. 439 */ 440 if (totalram_pages() < (512 << (20 - PAGE_SHIFT))) { 441 transparent_hugepage_flags = 0; 442 return 0; 443 } 444 445 err = start_stop_khugepaged(); 446 if (err) 447 goto err_khugepaged; 448 449 return 0; 450 err_khugepaged: 451 unregister_shrinker(&deferred_split_shrinker); 452 err_split_shrinker: 453 unregister_shrinker(&huge_zero_page_shrinker); 454 err_hzp_shrinker: 455 khugepaged_destroy(); 456 err_slab: 457 hugepage_exit_sysfs(hugepage_kobj); 458 err_sysfs: 459 return err; 460 } 461 subsys_initcall(hugepage_init); 462 463 static int __init setup_transparent_hugepage(char *str) 464 { 465 int ret = 0; 466 if (!str) 467 goto out; 468 if (!strcmp(str, "always")) { 469 set_bit(TRANSPARENT_HUGEPAGE_FLAG, 470 &transparent_hugepage_flags); 471 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 472 &transparent_hugepage_flags); 473 ret = 1; 474 } else if (!strcmp(str, "madvise")) { 475 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 476 &transparent_hugepage_flags); 477 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 478 &transparent_hugepage_flags); 479 ret = 1; 480 } else if (!strcmp(str, "never")) { 481 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 482 &transparent_hugepage_flags); 483 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 484 &transparent_hugepage_flags); 485 ret = 1; 486 } 487 out: 488 if (!ret) 489 pr_warn("transparent_hugepage= cannot parse, ignored\n"); 490 return ret; 491 } 492 __setup("transparent_hugepage=", setup_transparent_hugepage); 493 494 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 495 { 496 if (likely(vma->vm_flags & VM_WRITE)) 497 pmd = pmd_mkwrite(pmd); 498 return pmd; 499 } 500 501 #ifdef CONFIG_MEMCG 502 static inline struct deferred_split *get_deferred_split_queue(struct page *page) 503 { 504 struct mem_cgroup *memcg = page_memcg(compound_head(page)); 505 struct pglist_data *pgdat = NODE_DATA(page_to_nid(page)); 506 507 if (memcg) 508 return &memcg->deferred_split_queue; 509 else 510 return &pgdat->deferred_split_queue; 511 } 512 #else 513 static inline struct deferred_split *get_deferred_split_queue(struct page *page) 514 { 515 struct pglist_data *pgdat = NODE_DATA(page_to_nid(page)); 516 517 return &pgdat->deferred_split_queue; 518 } 519 #endif 520 521 void prep_transhuge_page(struct page *page) 522 { 523 /* 524 * we use page->mapping and page->indexlru in second tail page 525 * as list_head: assuming THP order >= 2 526 */ 527 528 INIT_LIST_HEAD(page_deferred_list(page)); 529 set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR); 530 } 531 532 bool is_transparent_hugepage(struct page *page) 533 { 534 if (!PageCompound(page)) 535 return false; 536 537 page = compound_head(page); 538 return is_huge_zero_page(page) || 539 page[1].compound_dtor == TRANSHUGE_PAGE_DTOR; 540 } 541 EXPORT_SYMBOL_GPL(is_transparent_hugepage); 542 543 static unsigned long __thp_get_unmapped_area(struct file *filp, 544 unsigned long addr, unsigned long len, 545 loff_t off, unsigned long flags, unsigned long size) 546 { 547 loff_t off_end = off + len; 548 loff_t off_align = round_up(off, size); 549 unsigned long len_pad, ret; 550 551 if (off_end <= off_align || (off_end - off_align) < size) 552 return 0; 553 554 len_pad = len + size; 555 if (len_pad < len || (off + len_pad) < off) 556 return 0; 557 558 ret = current->mm->get_unmapped_area(filp, addr, len_pad, 559 off >> PAGE_SHIFT, flags); 560 561 /* 562 * The failure might be due to length padding. The caller will retry 563 * without the padding. 564 */ 565 if (IS_ERR_VALUE(ret)) 566 return 0; 567 568 /* 569 * Do not try to align to THP boundary if allocation at the address 570 * hint succeeds. 571 */ 572 if (ret == addr) 573 return addr; 574 575 ret += (off - ret) & (size - 1); 576 return ret; 577 } 578 579 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, 580 unsigned long len, unsigned long pgoff, unsigned long flags) 581 { 582 unsigned long ret; 583 loff_t off = (loff_t)pgoff << PAGE_SHIFT; 584 585 if (!IS_DAX(filp->f_mapping->host) || !IS_ENABLED(CONFIG_FS_DAX_PMD)) 586 goto out; 587 588 ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE); 589 if (ret) 590 return ret; 591 out: 592 return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); 593 } 594 EXPORT_SYMBOL_GPL(thp_get_unmapped_area); 595 596 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, 597 struct page *page, gfp_t gfp) 598 { 599 struct vm_area_struct *vma = vmf->vma; 600 pgtable_t pgtable; 601 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 602 vm_fault_t ret = 0; 603 604 VM_BUG_ON_PAGE(!PageCompound(page), page); 605 606 if (mem_cgroup_charge(page_folio(page), vma->vm_mm, gfp)) { 607 put_page(page); 608 count_vm_event(THP_FAULT_FALLBACK); 609 count_vm_event(THP_FAULT_FALLBACK_CHARGE); 610 return VM_FAULT_FALLBACK; 611 } 612 cgroup_throttle_swaprate(page, gfp); 613 614 pgtable = pte_alloc_one(vma->vm_mm); 615 if (unlikely(!pgtable)) { 616 ret = VM_FAULT_OOM; 617 goto release; 618 } 619 620 clear_huge_page(page, vmf->address, HPAGE_PMD_NR); 621 /* 622 * The memory barrier inside __SetPageUptodate makes sure that 623 * clear_huge_page writes become visible before the set_pmd_at() 624 * write. 625 */ 626 __SetPageUptodate(page); 627 628 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 629 if (unlikely(!pmd_none(*vmf->pmd))) { 630 goto unlock_release; 631 } else { 632 pmd_t entry; 633 634 ret = check_stable_address_space(vma->vm_mm); 635 if (ret) 636 goto unlock_release; 637 638 /* Deliver the page fault to userland */ 639 if (userfaultfd_missing(vma)) { 640 spin_unlock(vmf->ptl); 641 put_page(page); 642 pte_free(vma->vm_mm, pgtable); 643 ret = handle_userfault(vmf, VM_UFFD_MISSING); 644 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 645 return ret; 646 } 647 648 entry = mk_huge_pmd(page, vma->vm_page_prot); 649 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 650 page_add_new_anon_rmap(page, vma, haddr, true); 651 lru_cache_add_inactive_or_unevictable(page, vma); 652 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); 653 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); 654 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 655 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); 656 mm_inc_nr_ptes(vma->vm_mm); 657 spin_unlock(vmf->ptl); 658 count_vm_event(THP_FAULT_ALLOC); 659 count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); 660 } 661 662 return 0; 663 unlock_release: 664 spin_unlock(vmf->ptl); 665 release: 666 if (pgtable) 667 pte_free(vma->vm_mm, pgtable); 668 put_page(page); 669 return ret; 670 671 } 672 673 /* 674 * always: directly stall for all thp allocations 675 * defer: wake kswapd and fail if not immediately available 676 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise 677 * fail if not immediately available 678 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately 679 * available 680 * never: never stall for any thp allocation 681 */ 682 gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma) 683 { 684 const bool vma_madvised = vma && (vma->vm_flags & VM_HUGEPAGE); 685 686 /* Always do synchronous compaction */ 687 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) 688 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY); 689 690 /* Kick kcompactd and fail quickly */ 691 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) 692 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM; 693 694 /* Synchronous compaction if madvised, otherwise kick kcompactd */ 695 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags)) 696 return GFP_TRANSHUGE_LIGHT | 697 (vma_madvised ? __GFP_DIRECT_RECLAIM : 698 __GFP_KSWAPD_RECLAIM); 699 700 /* Only do synchronous compaction if madvised */ 701 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags)) 702 return GFP_TRANSHUGE_LIGHT | 703 (vma_madvised ? __GFP_DIRECT_RECLAIM : 0); 704 705 return GFP_TRANSHUGE_LIGHT; 706 } 707 708 /* Caller must hold page table lock. */ 709 static void set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm, 710 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, 711 struct page *zero_page) 712 { 713 pmd_t entry; 714 if (!pmd_none(*pmd)) 715 return; 716 entry = mk_pmd(zero_page, vma->vm_page_prot); 717 entry = pmd_mkhuge(entry); 718 if (pgtable) 719 pgtable_trans_huge_deposit(mm, pmd, pgtable); 720 set_pmd_at(mm, haddr, pmd, entry); 721 mm_inc_nr_ptes(mm); 722 } 723 724 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) 725 { 726 struct vm_area_struct *vma = vmf->vma; 727 gfp_t gfp; 728 struct page *page; 729 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 730 731 if (!transhuge_vma_suitable(vma, haddr)) 732 return VM_FAULT_FALLBACK; 733 if (unlikely(anon_vma_prepare(vma))) 734 return VM_FAULT_OOM; 735 if (unlikely(khugepaged_enter(vma, vma->vm_flags))) 736 return VM_FAULT_OOM; 737 if (!(vmf->flags & FAULT_FLAG_WRITE) && 738 !mm_forbids_zeropage(vma->vm_mm) && 739 transparent_hugepage_use_zero_page()) { 740 pgtable_t pgtable; 741 struct page *zero_page; 742 vm_fault_t ret; 743 pgtable = pte_alloc_one(vma->vm_mm); 744 if (unlikely(!pgtable)) 745 return VM_FAULT_OOM; 746 zero_page = mm_get_huge_zero_page(vma->vm_mm); 747 if (unlikely(!zero_page)) { 748 pte_free(vma->vm_mm, pgtable); 749 count_vm_event(THP_FAULT_FALLBACK); 750 return VM_FAULT_FALLBACK; 751 } 752 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 753 ret = 0; 754 if (pmd_none(*vmf->pmd)) { 755 ret = check_stable_address_space(vma->vm_mm); 756 if (ret) { 757 spin_unlock(vmf->ptl); 758 pte_free(vma->vm_mm, pgtable); 759 } else if (userfaultfd_missing(vma)) { 760 spin_unlock(vmf->ptl); 761 pte_free(vma->vm_mm, pgtable); 762 ret = handle_userfault(vmf, VM_UFFD_MISSING); 763 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 764 } else { 765 set_huge_zero_page(pgtable, vma->vm_mm, vma, 766 haddr, vmf->pmd, zero_page); 767 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 768 spin_unlock(vmf->ptl); 769 } 770 } else { 771 spin_unlock(vmf->ptl); 772 pte_free(vma->vm_mm, pgtable); 773 } 774 return ret; 775 } 776 gfp = vma_thp_gfp_mask(vma); 777 page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER); 778 if (unlikely(!page)) { 779 count_vm_event(THP_FAULT_FALLBACK); 780 return VM_FAULT_FALLBACK; 781 } 782 prep_transhuge_page(page); 783 return __do_huge_pmd_anonymous_page(vmf, page, gfp); 784 } 785 786 static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 787 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write, 788 pgtable_t pgtable) 789 { 790 struct mm_struct *mm = vma->vm_mm; 791 pmd_t entry; 792 spinlock_t *ptl; 793 794 ptl = pmd_lock(mm, pmd); 795 if (!pmd_none(*pmd)) { 796 if (write) { 797 if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) { 798 WARN_ON_ONCE(!is_huge_zero_pmd(*pmd)); 799 goto out_unlock; 800 } 801 entry = pmd_mkyoung(*pmd); 802 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 803 if (pmdp_set_access_flags(vma, addr, pmd, entry, 1)) 804 update_mmu_cache_pmd(vma, addr, pmd); 805 } 806 807 goto out_unlock; 808 } 809 810 entry = pmd_mkhuge(pfn_t_pmd(pfn, prot)); 811 if (pfn_t_devmap(pfn)) 812 entry = pmd_mkdevmap(entry); 813 if (write) { 814 entry = pmd_mkyoung(pmd_mkdirty(entry)); 815 entry = maybe_pmd_mkwrite(entry, vma); 816 } 817 818 if (pgtable) { 819 pgtable_trans_huge_deposit(mm, pmd, pgtable); 820 mm_inc_nr_ptes(mm); 821 pgtable = NULL; 822 } 823 824 set_pmd_at(mm, addr, pmd, entry); 825 update_mmu_cache_pmd(vma, addr, pmd); 826 827 out_unlock: 828 spin_unlock(ptl); 829 if (pgtable) 830 pte_free(mm, pgtable); 831 } 832 833 /** 834 * vmf_insert_pfn_pmd_prot - insert a pmd size pfn 835 * @vmf: Structure describing the fault 836 * @pfn: pfn to insert 837 * @pgprot: page protection to use 838 * @write: whether it's a write fault 839 * 840 * Insert a pmd size pfn. See vmf_insert_pfn() for additional info and 841 * also consult the vmf_insert_mixed_prot() documentation when 842 * @pgprot != @vmf->vma->vm_page_prot. 843 * 844 * Return: vm_fault_t value. 845 */ 846 vm_fault_t vmf_insert_pfn_pmd_prot(struct vm_fault *vmf, pfn_t pfn, 847 pgprot_t pgprot, bool write) 848 { 849 unsigned long addr = vmf->address & PMD_MASK; 850 struct vm_area_struct *vma = vmf->vma; 851 pgtable_t pgtable = NULL; 852 853 /* 854 * If we had pmd_special, we could avoid all these restrictions, 855 * but we need to be consistent with PTEs and architectures that 856 * can't support a 'special' bit. 857 */ 858 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && 859 !pfn_t_devmap(pfn)); 860 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 861 (VM_PFNMAP|VM_MIXEDMAP)); 862 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 863 864 if (addr < vma->vm_start || addr >= vma->vm_end) 865 return VM_FAULT_SIGBUS; 866 867 if (arch_needs_pgtable_deposit()) { 868 pgtable = pte_alloc_one(vma->vm_mm); 869 if (!pgtable) 870 return VM_FAULT_OOM; 871 } 872 873 track_pfn_insert(vma, &pgprot, pfn); 874 875 insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable); 876 return VM_FAULT_NOPAGE; 877 } 878 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd_prot); 879 880 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 881 static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma) 882 { 883 if (likely(vma->vm_flags & VM_WRITE)) 884 pud = pud_mkwrite(pud); 885 return pud; 886 } 887 888 static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr, 889 pud_t *pud, pfn_t pfn, pgprot_t prot, bool write) 890 { 891 struct mm_struct *mm = vma->vm_mm; 892 pud_t entry; 893 spinlock_t *ptl; 894 895 ptl = pud_lock(mm, pud); 896 if (!pud_none(*pud)) { 897 if (write) { 898 if (pud_pfn(*pud) != pfn_t_to_pfn(pfn)) { 899 WARN_ON_ONCE(!is_huge_zero_pud(*pud)); 900 goto out_unlock; 901 } 902 entry = pud_mkyoung(*pud); 903 entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma); 904 if (pudp_set_access_flags(vma, addr, pud, entry, 1)) 905 update_mmu_cache_pud(vma, addr, pud); 906 } 907 goto out_unlock; 908 } 909 910 entry = pud_mkhuge(pfn_t_pud(pfn, prot)); 911 if (pfn_t_devmap(pfn)) 912 entry = pud_mkdevmap(entry); 913 if (write) { 914 entry = pud_mkyoung(pud_mkdirty(entry)); 915 entry = maybe_pud_mkwrite(entry, vma); 916 } 917 set_pud_at(mm, addr, pud, entry); 918 update_mmu_cache_pud(vma, addr, pud); 919 920 out_unlock: 921 spin_unlock(ptl); 922 } 923 924 /** 925 * vmf_insert_pfn_pud_prot - insert a pud size pfn 926 * @vmf: Structure describing the fault 927 * @pfn: pfn to insert 928 * @pgprot: page protection to use 929 * @write: whether it's a write fault 930 * 931 * Insert a pud size pfn. See vmf_insert_pfn() for additional info and 932 * also consult the vmf_insert_mixed_prot() documentation when 933 * @pgprot != @vmf->vma->vm_page_prot. 934 * 935 * Return: vm_fault_t value. 936 */ 937 vm_fault_t vmf_insert_pfn_pud_prot(struct vm_fault *vmf, pfn_t pfn, 938 pgprot_t pgprot, bool write) 939 { 940 unsigned long addr = vmf->address & PUD_MASK; 941 struct vm_area_struct *vma = vmf->vma; 942 943 /* 944 * If we had pud_special, we could avoid all these restrictions, 945 * but we need to be consistent with PTEs and architectures that 946 * can't support a 'special' bit. 947 */ 948 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && 949 !pfn_t_devmap(pfn)); 950 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 951 (VM_PFNMAP|VM_MIXEDMAP)); 952 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 953 954 if (addr < vma->vm_start || addr >= vma->vm_end) 955 return VM_FAULT_SIGBUS; 956 957 track_pfn_insert(vma, &pgprot, pfn); 958 959 insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write); 960 return VM_FAULT_NOPAGE; 961 } 962 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud_prot); 963 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 964 965 static void touch_pmd(struct vm_area_struct *vma, unsigned long addr, 966 pmd_t *pmd, int flags) 967 { 968 pmd_t _pmd; 969 970 _pmd = pmd_mkyoung(*pmd); 971 if (flags & FOLL_WRITE) 972 _pmd = pmd_mkdirty(_pmd); 973 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, 974 pmd, _pmd, flags & FOLL_WRITE)) 975 update_mmu_cache_pmd(vma, addr, pmd); 976 } 977 978 struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, 979 pmd_t *pmd, int flags, struct dev_pagemap **pgmap) 980 { 981 unsigned long pfn = pmd_pfn(*pmd); 982 struct mm_struct *mm = vma->vm_mm; 983 struct page *page; 984 985 assert_spin_locked(pmd_lockptr(mm, pmd)); 986 987 /* 988 * When we COW a devmap PMD entry, we split it into PTEs, so we should 989 * not be in this function with `flags & FOLL_COW` set. 990 */ 991 WARN_ONCE(flags & FOLL_COW, "mm: In follow_devmap_pmd with FOLL_COW set"); 992 993 /* FOLL_GET and FOLL_PIN are mutually exclusive. */ 994 if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) == 995 (FOLL_PIN | FOLL_GET))) 996 return NULL; 997 998 if (flags & FOLL_WRITE && !pmd_write(*pmd)) 999 return NULL; 1000 1001 if (pmd_present(*pmd) && pmd_devmap(*pmd)) 1002 /* pass */; 1003 else 1004 return NULL; 1005 1006 if (flags & FOLL_TOUCH) 1007 touch_pmd(vma, addr, pmd, flags); 1008 1009 /* 1010 * device mapped pages can only be returned if the 1011 * caller will manage the page reference count. 1012 */ 1013 if (!(flags & (FOLL_GET | FOLL_PIN))) 1014 return ERR_PTR(-EEXIST); 1015 1016 pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT; 1017 *pgmap = get_dev_pagemap(pfn, *pgmap); 1018 if (!*pgmap) 1019 return ERR_PTR(-EFAULT); 1020 page = pfn_to_page(pfn); 1021 if (!try_grab_page(page, flags)) 1022 page = ERR_PTR(-ENOMEM); 1023 1024 return page; 1025 } 1026 1027 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1028 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 1029 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) 1030 { 1031 spinlock_t *dst_ptl, *src_ptl; 1032 struct page *src_page; 1033 pmd_t pmd; 1034 pgtable_t pgtable = NULL; 1035 int ret = -ENOMEM; 1036 1037 /* Skip if can be re-fill on fault */ 1038 if (!vma_is_anonymous(dst_vma)) 1039 return 0; 1040 1041 pgtable = pte_alloc_one(dst_mm); 1042 if (unlikely(!pgtable)) 1043 goto out; 1044 1045 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1046 src_ptl = pmd_lockptr(src_mm, src_pmd); 1047 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1048 1049 ret = -EAGAIN; 1050 pmd = *src_pmd; 1051 1052 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 1053 if (unlikely(is_swap_pmd(pmd))) { 1054 swp_entry_t entry = pmd_to_swp_entry(pmd); 1055 1056 VM_BUG_ON(!is_pmd_migration_entry(pmd)); 1057 if (is_writable_migration_entry(entry)) { 1058 entry = make_readable_migration_entry( 1059 swp_offset(entry)); 1060 pmd = swp_entry_to_pmd(entry); 1061 if (pmd_swp_soft_dirty(*src_pmd)) 1062 pmd = pmd_swp_mksoft_dirty(pmd); 1063 if (pmd_swp_uffd_wp(*src_pmd)) 1064 pmd = pmd_swp_mkuffd_wp(pmd); 1065 set_pmd_at(src_mm, addr, src_pmd, pmd); 1066 } 1067 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1068 mm_inc_nr_ptes(dst_mm); 1069 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1070 if (!userfaultfd_wp(dst_vma)) 1071 pmd = pmd_swp_clear_uffd_wp(pmd); 1072 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1073 ret = 0; 1074 goto out_unlock; 1075 } 1076 #endif 1077 1078 if (unlikely(!pmd_trans_huge(pmd))) { 1079 pte_free(dst_mm, pgtable); 1080 goto out_unlock; 1081 } 1082 /* 1083 * When page table lock is held, the huge zero pmd should not be 1084 * under splitting since we don't split the page itself, only pmd to 1085 * a page table. 1086 */ 1087 if (is_huge_zero_pmd(pmd)) { 1088 /* 1089 * get_huge_zero_page() will never allocate a new page here, 1090 * since we already have a zero page to copy. It just takes a 1091 * reference. 1092 */ 1093 mm_get_huge_zero_page(dst_mm); 1094 goto out_zero_page; 1095 } 1096 1097 src_page = pmd_page(pmd); 1098 VM_BUG_ON_PAGE(!PageHead(src_page), src_page); 1099 1100 /* 1101 * If this page is a potentially pinned page, split and retry the fault 1102 * with smaller page size. Normally this should not happen because the 1103 * userspace should use MADV_DONTFORK upon pinned regions. This is a 1104 * best effort that the pinned pages won't be replaced by another 1105 * random page during the coming copy-on-write. 1106 */ 1107 if (unlikely(page_needs_cow_for_dma(src_vma, src_page))) { 1108 pte_free(dst_mm, pgtable); 1109 spin_unlock(src_ptl); 1110 spin_unlock(dst_ptl); 1111 __split_huge_pmd(src_vma, src_pmd, addr, false, NULL); 1112 return -EAGAIN; 1113 } 1114 1115 get_page(src_page); 1116 page_dup_rmap(src_page, true); 1117 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1118 out_zero_page: 1119 mm_inc_nr_ptes(dst_mm); 1120 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1121 pmdp_set_wrprotect(src_mm, addr, src_pmd); 1122 if (!userfaultfd_wp(dst_vma)) 1123 pmd = pmd_clear_uffd_wp(pmd); 1124 pmd = pmd_mkold(pmd_wrprotect(pmd)); 1125 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1126 1127 ret = 0; 1128 out_unlock: 1129 spin_unlock(src_ptl); 1130 spin_unlock(dst_ptl); 1131 out: 1132 return ret; 1133 } 1134 1135 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1136 static void touch_pud(struct vm_area_struct *vma, unsigned long addr, 1137 pud_t *pud, int flags) 1138 { 1139 pud_t _pud; 1140 1141 _pud = pud_mkyoung(*pud); 1142 if (flags & FOLL_WRITE) 1143 _pud = pud_mkdirty(_pud); 1144 if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK, 1145 pud, _pud, flags & FOLL_WRITE)) 1146 update_mmu_cache_pud(vma, addr, pud); 1147 } 1148 1149 struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr, 1150 pud_t *pud, int flags, struct dev_pagemap **pgmap) 1151 { 1152 unsigned long pfn = pud_pfn(*pud); 1153 struct mm_struct *mm = vma->vm_mm; 1154 struct page *page; 1155 1156 assert_spin_locked(pud_lockptr(mm, pud)); 1157 1158 if (flags & FOLL_WRITE && !pud_write(*pud)) 1159 return NULL; 1160 1161 /* FOLL_GET and FOLL_PIN are mutually exclusive. */ 1162 if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) == 1163 (FOLL_PIN | FOLL_GET))) 1164 return NULL; 1165 1166 if (pud_present(*pud) && pud_devmap(*pud)) 1167 /* pass */; 1168 else 1169 return NULL; 1170 1171 if (flags & FOLL_TOUCH) 1172 touch_pud(vma, addr, pud, flags); 1173 1174 /* 1175 * device mapped pages can only be returned if the 1176 * caller will manage the page reference count. 1177 * 1178 * At least one of FOLL_GET | FOLL_PIN must be set, so assert that here: 1179 */ 1180 if (!(flags & (FOLL_GET | FOLL_PIN))) 1181 return ERR_PTR(-EEXIST); 1182 1183 pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT; 1184 *pgmap = get_dev_pagemap(pfn, *pgmap); 1185 if (!*pgmap) 1186 return ERR_PTR(-EFAULT); 1187 page = pfn_to_page(pfn); 1188 if (!try_grab_page(page, flags)) 1189 page = ERR_PTR(-ENOMEM); 1190 1191 return page; 1192 } 1193 1194 int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1195 pud_t *dst_pud, pud_t *src_pud, unsigned long addr, 1196 struct vm_area_struct *vma) 1197 { 1198 spinlock_t *dst_ptl, *src_ptl; 1199 pud_t pud; 1200 int ret; 1201 1202 dst_ptl = pud_lock(dst_mm, dst_pud); 1203 src_ptl = pud_lockptr(src_mm, src_pud); 1204 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1205 1206 ret = -EAGAIN; 1207 pud = *src_pud; 1208 if (unlikely(!pud_trans_huge(pud) && !pud_devmap(pud))) 1209 goto out_unlock; 1210 1211 /* 1212 * When page table lock is held, the huge zero pud should not be 1213 * under splitting since we don't split the page itself, only pud to 1214 * a page table. 1215 */ 1216 if (is_huge_zero_pud(pud)) { 1217 /* No huge zero pud yet */ 1218 } 1219 1220 /* Please refer to comments in copy_huge_pmd() */ 1221 if (unlikely(page_needs_cow_for_dma(vma, pud_page(pud)))) { 1222 spin_unlock(src_ptl); 1223 spin_unlock(dst_ptl); 1224 __split_huge_pud(vma, src_pud, addr); 1225 return -EAGAIN; 1226 } 1227 1228 pudp_set_wrprotect(src_mm, addr, src_pud); 1229 pud = pud_mkold(pud_wrprotect(pud)); 1230 set_pud_at(dst_mm, addr, dst_pud, pud); 1231 1232 ret = 0; 1233 out_unlock: 1234 spin_unlock(src_ptl); 1235 spin_unlock(dst_ptl); 1236 return ret; 1237 } 1238 1239 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud) 1240 { 1241 pud_t entry; 1242 unsigned long haddr; 1243 bool write = vmf->flags & FAULT_FLAG_WRITE; 1244 1245 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); 1246 if (unlikely(!pud_same(*vmf->pud, orig_pud))) 1247 goto unlock; 1248 1249 entry = pud_mkyoung(orig_pud); 1250 if (write) 1251 entry = pud_mkdirty(entry); 1252 haddr = vmf->address & HPAGE_PUD_MASK; 1253 if (pudp_set_access_flags(vmf->vma, haddr, vmf->pud, entry, write)) 1254 update_mmu_cache_pud(vmf->vma, vmf->address, vmf->pud); 1255 1256 unlock: 1257 spin_unlock(vmf->ptl); 1258 } 1259 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1260 1261 void huge_pmd_set_accessed(struct vm_fault *vmf) 1262 { 1263 pmd_t entry; 1264 unsigned long haddr; 1265 bool write = vmf->flags & FAULT_FLAG_WRITE; 1266 pmd_t orig_pmd = vmf->orig_pmd; 1267 1268 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); 1269 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) 1270 goto unlock; 1271 1272 entry = pmd_mkyoung(orig_pmd); 1273 if (write) 1274 entry = pmd_mkdirty(entry); 1275 haddr = vmf->address & HPAGE_PMD_MASK; 1276 if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write)) 1277 update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd); 1278 1279 unlock: 1280 spin_unlock(vmf->ptl); 1281 } 1282 1283 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) 1284 { 1285 struct vm_area_struct *vma = vmf->vma; 1286 struct page *page; 1287 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1288 pmd_t orig_pmd = vmf->orig_pmd; 1289 1290 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); 1291 VM_BUG_ON_VMA(!vma->anon_vma, vma); 1292 1293 if (is_huge_zero_pmd(orig_pmd)) 1294 goto fallback; 1295 1296 spin_lock(vmf->ptl); 1297 1298 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1299 spin_unlock(vmf->ptl); 1300 return 0; 1301 } 1302 1303 page = pmd_page(orig_pmd); 1304 VM_BUG_ON_PAGE(!PageHead(page), page); 1305 1306 /* Lock page for reuse_swap_page() */ 1307 if (!trylock_page(page)) { 1308 get_page(page); 1309 spin_unlock(vmf->ptl); 1310 lock_page(page); 1311 spin_lock(vmf->ptl); 1312 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1313 spin_unlock(vmf->ptl); 1314 unlock_page(page); 1315 put_page(page); 1316 return 0; 1317 } 1318 put_page(page); 1319 } 1320 1321 /* 1322 * We can only reuse the page if nobody else maps the huge page or it's 1323 * part. 1324 */ 1325 if (reuse_swap_page(page)) { 1326 pmd_t entry; 1327 entry = pmd_mkyoung(orig_pmd); 1328 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1329 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) 1330 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1331 unlock_page(page); 1332 spin_unlock(vmf->ptl); 1333 return VM_FAULT_WRITE; 1334 } 1335 1336 unlock_page(page); 1337 spin_unlock(vmf->ptl); 1338 fallback: 1339 __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL); 1340 return VM_FAULT_FALLBACK; 1341 } 1342 1343 /* 1344 * FOLL_FORCE can write to even unwritable pmd's, but only 1345 * after we've gone through a COW cycle and they are dirty. 1346 */ 1347 static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags) 1348 { 1349 return pmd_write(pmd) || 1350 ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd)); 1351 } 1352 1353 struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, 1354 unsigned long addr, 1355 pmd_t *pmd, 1356 unsigned int flags) 1357 { 1358 struct mm_struct *mm = vma->vm_mm; 1359 struct page *page = NULL; 1360 1361 assert_spin_locked(pmd_lockptr(mm, pmd)); 1362 1363 if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags)) 1364 goto out; 1365 1366 /* Avoid dumping huge zero page */ 1367 if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd)) 1368 return ERR_PTR(-EFAULT); 1369 1370 /* Full NUMA hinting faults to serialise migration in fault paths */ 1371 if ((flags & FOLL_NUMA) && pmd_protnone(*pmd)) 1372 goto out; 1373 1374 page = pmd_page(*pmd); 1375 VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page); 1376 1377 if (!try_grab_page(page, flags)) 1378 return ERR_PTR(-ENOMEM); 1379 1380 if (flags & FOLL_TOUCH) 1381 touch_pmd(vma, addr, pmd, flags); 1382 1383 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { 1384 /* 1385 * We don't mlock() pte-mapped THPs. This way we can avoid 1386 * leaking mlocked pages into non-VM_LOCKED VMAs. 1387 * 1388 * For anon THP: 1389 * 1390 * In most cases the pmd is the only mapping of the page as we 1391 * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for 1392 * writable private mappings in populate_vma_page_range(). 1393 * 1394 * The only scenario when we have the page shared here is if we 1395 * mlocking read-only mapping shared over fork(). We skip 1396 * mlocking such pages. 1397 * 1398 * For file THP: 1399 * 1400 * We can expect PageDoubleMap() to be stable under page lock: 1401 * for file pages we set it in page_add_file_rmap(), which 1402 * requires page to be locked. 1403 */ 1404 1405 if (PageAnon(page) && compound_mapcount(page) != 1) 1406 goto skip_mlock; 1407 if (PageDoubleMap(page) || !page->mapping) 1408 goto skip_mlock; 1409 if (!trylock_page(page)) 1410 goto skip_mlock; 1411 if (page->mapping && !PageDoubleMap(page)) 1412 mlock_vma_page(page); 1413 unlock_page(page); 1414 } 1415 skip_mlock: 1416 page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT; 1417 VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page); 1418 1419 out: 1420 return page; 1421 } 1422 1423 /* NUMA hinting page fault entry point for trans huge pmds */ 1424 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) 1425 { 1426 struct vm_area_struct *vma = vmf->vma; 1427 pmd_t oldpmd = vmf->orig_pmd; 1428 pmd_t pmd; 1429 struct page *page; 1430 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1431 int page_nid = NUMA_NO_NODE; 1432 int target_nid, last_cpupid = -1; 1433 bool migrated = false; 1434 bool was_writable = pmd_savedwrite(oldpmd); 1435 int flags = 0; 1436 1437 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1438 if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) { 1439 spin_unlock(vmf->ptl); 1440 goto out; 1441 } 1442 1443 pmd = pmd_modify(oldpmd, vma->vm_page_prot); 1444 page = vm_normal_page_pmd(vma, haddr, pmd); 1445 if (!page) 1446 goto out_map; 1447 1448 /* See similar comment in do_numa_page for explanation */ 1449 if (!was_writable) 1450 flags |= TNF_NO_GROUP; 1451 1452 page_nid = page_to_nid(page); 1453 last_cpupid = page_cpupid_last(page); 1454 target_nid = numa_migrate_prep(page, vma, haddr, page_nid, 1455 &flags); 1456 1457 if (target_nid == NUMA_NO_NODE) { 1458 put_page(page); 1459 goto out_map; 1460 } 1461 1462 spin_unlock(vmf->ptl); 1463 1464 migrated = migrate_misplaced_page(page, vma, target_nid); 1465 if (migrated) { 1466 flags |= TNF_MIGRATED; 1467 page_nid = target_nid; 1468 } else { 1469 flags |= TNF_MIGRATE_FAIL; 1470 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1471 if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) { 1472 spin_unlock(vmf->ptl); 1473 goto out; 1474 } 1475 goto out_map; 1476 } 1477 1478 out: 1479 if (page_nid != NUMA_NO_NODE) 1480 task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, 1481 flags); 1482 1483 return 0; 1484 1485 out_map: 1486 /* Restore the PMD */ 1487 pmd = pmd_modify(oldpmd, vma->vm_page_prot); 1488 pmd = pmd_mkyoung(pmd); 1489 if (was_writable) 1490 pmd = pmd_mkwrite(pmd); 1491 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); 1492 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1493 spin_unlock(vmf->ptl); 1494 goto out; 1495 } 1496 1497 /* 1498 * Return true if we do MADV_FREE successfully on entire pmd page. 1499 * Otherwise, return false. 1500 */ 1501 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1502 pmd_t *pmd, unsigned long addr, unsigned long next) 1503 { 1504 spinlock_t *ptl; 1505 pmd_t orig_pmd; 1506 struct page *page; 1507 struct mm_struct *mm = tlb->mm; 1508 bool ret = false; 1509 1510 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 1511 1512 ptl = pmd_trans_huge_lock(pmd, vma); 1513 if (!ptl) 1514 goto out_unlocked; 1515 1516 orig_pmd = *pmd; 1517 if (is_huge_zero_pmd(orig_pmd)) 1518 goto out; 1519 1520 if (unlikely(!pmd_present(orig_pmd))) { 1521 VM_BUG_ON(thp_migration_supported() && 1522 !is_pmd_migration_entry(orig_pmd)); 1523 goto out; 1524 } 1525 1526 page = pmd_page(orig_pmd); 1527 /* 1528 * If other processes are mapping this page, we couldn't discard 1529 * the page unless they all do MADV_FREE so let's skip the page. 1530 */ 1531 if (total_mapcount(page) != 1) 1532 goto out; 1533 1534 if (!trylock_page(page)) 1535 goto out; 1536 1537 /* 1538 * If user want to discard part-pages of THP, split it so MADV_FREE 1539 * will deactivate only them. 1540 */ 1541 if (next - addr != HPAGE_PMD_SIZE) { 1542 get_page(page); 1543 spin_unlock(ptl); 1544 split_huge_page(page); 1545 unlock_page(page); 1546 put_page(page); 1547 goto out_unlocked; 1548 } 1549 1550 if (PageDirty(page)) 1551 ClearPageDirty(page); 1552 unlock_page(page); 1553 1554 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) { 1555 pmdp_invalidate(vma, addr, pmd); 1556 orig_pmd = pmd_mkold(orig_pmd); 1557 orig_pmd = pmd_mkclean(orig_pmd); 1558 1559 set_pmd_at(mm, addr, pmd, orig_pmd); 1560 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 1561 } 1562 1563 mark_page_lazyfree(page); 1564 ret = true; 1565 out: 1566 spin_unlock(ptl); 1567 out_unlocked: 1568 return ret; 1569 } 1570 1571 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd) 1572 { 1573 pgtable_t pgtable; 1574 1575 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 1576 pte_free(mm, pgtable); 1577 mm_dec_nr_ptes(mm); 1578 } 1579 1580 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1581 pmd_t *pmd, unsigned long addr) 1582 { 1583 pmd_t orig_pmd; 1584 spinlock_t *ptl; 1585 1586 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 1587 1588 ptl = __pmd_trans_huge_lock(pmd, vma); 1589 if (!ptl) 1590 return 0; 1591 /* 1592 * For architectures like ppc64 we look at deposited pgtable 1593 * when calling pmdp_huge_get_and_clear. So do the 1594 * pgtable_trans_huge_withdraw after finishing pmdp related 1595 * operations. 1596 */ 1597 orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd, 1598 tlb->fullmm); 1599 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 1600 if (vma_is_special_huge(vma)) { 1601 if (arch_needs_pgtable_deposit()) 1602 zap_deposited_table(tlb->mm, pmd); 1603 spin_unlock(ptl); 1604 } else if (is_huge_zero_pmd(orig_pmd)) { 1605 zap_deposited_table(tlb->mm, pmd); 1606 spin_unlock(ptl); 1607 } else { 1608 struct page *page = NULL; 1609 int flush_needed = 1; 1610 1611 if (pmd_present(orig_pmd)) { 1612 page = pmd_page(orig_pmd); 1613 page_remove_rmap(page, true); 1614 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page); 1615 VM_BUG_ON_PAGE(!PageHead(page), page); 1616 } else if (thp_migration_supported()) { 1617 swp_entry_t entry; 1618 1619 VM_BUG_ON(!is_pmd_migration_entry(orig_pmd)); 1620 entry = pmd_to_swp_entry(orig_pmd); 1621 page = pfn_swap_entry_to_page(entry); 1622 flush_needed = 0; 1623 } else 1624 WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!"); 1625 1626 if (PageAnon(page)) { 1627 zap_deposited_table(tlb->mm, pmd); 1628 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); 1629 } else { 1630 if (arch_needs_pgtable_deposit()) 1631 zap_deposited_table(tlb->mm, pmd); 1632 add_mm_counter(tlb->mm, mm_counter_file(page), -HPAGE_PMD_NR); 1633 } 1634 1635 spin_unlock(ptl); 1636 if (flush_needed) 1637 tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE); 1638 } 1639 return 1; 1640 } 1641 1642 #ifndef pmd_move_must_withdraw 1643 static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl, 1644 spinlock_t *old_pmd_ptl, 1645 struct vm_area_struct *vma) 1646 { 1647 /* 1648 * With split pmd lock we also need to move preallocated 1649 * PTE page table if new_pmd is on different PMD page table. 1650 * 1651 * We also don't deposit and withdraw tables for file pages. 1652 */ 1653 return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma); 1654 } 1655 #endif 1656 1657 static pmd_t move_soft_dirty_pmd(pmd_t pmd) 1658 { 1659 #ifdef CONFIG_MEM_SOFT_DIRTY 1660 if (unlikely(is_pmd_migration_entry(pmd))) 1661 pmd = pmd_swp_mksoft_dirty(pmd); 1662 else if (pmd_present(pmd)) 1663 pmd = pmd_mksoft_dirty(pmd); 1664 #endif 1665 return pmd; 1666 } 1667 1668 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, 1669 unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd) 1670 { 1671 spinlock_t *old_ptl, *new_ptl; 1672 pmd_t pmd; 1673 struct mm_struct *mm = vma->vm_mm; 1674 bool force_flush = false; 1675 1676 /* 1677 * The destination pmd shouldn't be established, free_pgtables() 1678 * should have release it. 1679 */ 1680 if (WARN_ON(!pmd_none(*new_pmd))) { 1681 VM_BUG_ON(pmd_trans_huge(*new_pmd)); 1682 return false; 1683 } 1684 1685 /* 1686 * We don't have to worry about the ordering of src and dst 1687 * ptlocks because exclusive mmap_lock prevents deadlock. 1688 */ 1689 old_ptl = __pmd_trans_huge_lock(old_pmd, vma); 1690 if (old_ptl) { 1691 new_ptl = pmd_lockptr(mm, new_pmd); 1692 if (new_ptl != old_ptl) 1693 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 1694 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); 1695 if (pmd_present(pmd)) 1696 force_flush = true; 1697 VM_BUG_ON(!pmd_none(*new_pmd)); 1698 1699 if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) { 1700 pgtable_t pgtable; 1701 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); 1702 pgtable_trans_huge_deposit(mm, new_pmd, pgtable); 1703 } 1704 pmd = move_soft_dirty_pmd(pmd); 1705 set_pmd_at(mm, new_addr, new_pmd, pmd); 1706 if (force_flush) 1707 flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE); 1708 if (new_ptl != old_ptl) 1709 spin_unlock(new_ptl); 1710 spin_unlock(old_ptl); 1711 return true; 1712 } 1713 return false; 1714 } 1715 1716 /* 1717 * Returns 1718 * - 0 if PMD could not be locked 1719 * - 1 if PMD was locked but protections unchanged and TLB flush unnecessary 1720 * or if prot_numa but THP migration is not supported 1721 * - HPAGE_PMD_NR if protections changed and TLB flush necessary 1722 */ 1723 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 1724 unsigned long addr, pgprot_t newprot, unsigned long cp_flags) 1725 { 1726 struct mm_struct *mm = vma->vm_mm; 1727 spinlock_t *ptl; 1728 pmd_t entry; 1729 bool preserve_write; 1730 int ret; 1731 bool prot_numa = cp_flags & MM_CP_PROT_NUMA; 1732 bool uffd_wp = cp_flags & MM_CP_UFFD_WP; 1733 bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; 1734 1735 if (prot_numa && !thp_migration_supported()) 1736 return 1; 1737 1738 ptl = __pmd_trans_huge_lock(pmd, vma); 1739 if (!ptl) 1740 return 0; 1741 1742 preserve_write = prot_numa && pmd_write(*pmd); 1743 ret = 1; 1744 1745 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 1746 if (is_swap_pmd(*pmd)) { 1747 swp_entry_t entry = pmd_to_swp_entry(*pmd); 1748 1749 VM_BUG_ON(!is_pmd_migration_entry(*pmd)); 1750 if (is_writable_migration_entry(entry)) { 1751 pmd_t newpmd; 1752 /* 1753 * A protection check is difficult so 1754 * just be safe and disable write 1755 */ 1756 entry = make_readable_migration_entry( 1757 swp_offset(entry)); 1758 newpmd = swp_entry_to_pmd(entry); 1759 if (pmd_swp_soft_dirty(*pmd)) 1760 newpmd = pmd_swp_mksoft_dirty(newpmd); 1761 if (pmd_swp_uffd_wp(*pmd)) 1762 newpmd = pmd_swp_mkuffd_wp(newpmd); 1763 set_pmd_at(mm, addr, pmd, newpmd); 1764 } 1765 goto unlock; 1766 } 1767 #endif 1768 1769 /* 1770 * Avoid trapping faults against the zero page. The read-only 1771 * data is likely to be read-cached on the local CPU and 1772 * local/remote hits to the zero page are not interesting. 1773 */ 1774 if (prot_numa && is_huge_zero_pmd(*pmd)) 1775 goto unlock; 1776 1777 if (prot_numa && pmd_protnone(*pmd)) 1778 goto unlock; 1779 1780 /* 1781 * In case prot_numa, we are under mmap_read_lock(mm). It's critical 1782 * to not clear pmd intermittently to avoid race with MADV_DONTNEED 1783 * which is also under mmap_read_lock(mm): 1784 * 1785 * CPU0: CPU1: 1786 * change_huge_pmd(prot_numa=1) 1787 * pmdp_huge_get_and_clear_notify() 1788 * madvise_dontneed() 1789 * zap_pmd_range() 1790 * pmd_trans_huge(*pmd) == 0 (without ptl) 1791 * // skip the pmd 1792 * set_pmd_at(); 1793 * // pmd is re-established 1794 * 1795 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it 1796 * which may break userspace. 1797 * 1798 * pmdp_invalidate() is required to make sure we don't miss 1799 * dirty/young flags set by hardware. 1800 */ 1801 entry = pmdp_invalidate(vma, addr, pmd); 1802 1803 entry = pmd_modify(entry, newprot); 1804 if (preserve_write) 1805 entry = pmd_mk_savedwrite(entry); 1806 if (uffd_wp) { 1807 entry = pmd_wrprotect(entry); 1808 entry = pmd_mkuffd_wp(entry); 1809 } else if (uffd_wp_resolve) { 1810 /* 1811 * Leave the write bit to be handled by PF interrupt 1812 * handler, then things like COW could be properly 1813 * handled. 1814 */ 1815 entry = pmd_clear_uffd_wp(entry); 1816 } 1817 ret = HPAGE_PMD_NR; 1818 set_pmd_at(mm, addr, pmd, entry); 1819 BUG_ON(vma_is_anonymous(vma) && !preserve_write && pmd_write(entry)); 1820 unlock: 1821 spin_unlock(ptl); 1822 return ret; 1823 } 1824 1825 /* 1826 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise. 1827 * 1828 * Note that if it returns page table lock pointer, this routine returns without 1829 * unlocking page table lock. So callers must unlock it. 1830 */ 1831 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) 1832 { 1833 spinlock_t *ptl; 1834 ptl = pmd_lock(vma->vm_mm, pmd); 1835 if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || 1836 pmd_devmap(*pmd))) 1837 return ptl; 1838 spin_unlock(ptl); 1839 return NULL; 1840 } 1841 1842 /* 1843 * Returns true if a given pud maps a thp, false otherwise. 1844 * 1845 * Note that if it returns true, this routine returns without unlocking page 1846 * table lock. So callers must unlock it. 1847 */ 1848 spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma) 1849 { 1850 spinlock_t *ptl; 1851 1852 ptl = pud_lock(vma->vm_mm, pud); 1853 if (likely(pud_trans_huge(*pud) || pud_devmap(*pud))) 1854 return ptl; 1855 spin_unlock(ptl); 1856 return NULL; 1857 } 1858 1859 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1860 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 1861 pud_t *pud, unsigned long addr) 1862 { 1863 spinlock_t *ptl; 1864 1865 ptl = __pud_trans_huge_lock(pud, vma); 1866 if (!ptl) 1867 return 0; 1868 /* 1869 * For architectures like ppc64 we look at deposited pgtable 1870 * when calling pudp_huge_get_and_clear. So do the 1871 * pgtable_trans_huge_withdraw after finishing pudp related 1872 * operations. 1873 */ 1874 pudp_huge_get_and_clear_full(tlb->mm, addr, pud, tlb->fullmm); 1875 tlb_remove_pud_tlb_entry(tlb, pud, addr); 1876 if (vma_is_special_huge(vma)) { 1877 spin_unlock(ptl); 1878 /* No zero page support yet */ 1879 } else { 1880 /* No support for anonymous PUD pages yet */ 1881 BUG(); 1882 } 1883 return 1; 1884 } 1885 1886 static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud, 1887 unsigned long haddr) 1888 { 1889 VM_BUG_ON(haddr & ~HPAGE_PUD_MASK); 1890 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 1891 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma); 1892 VM_BUG_ON(!pud_trans_huge(*pud) && !pud_devmap(*pud)); 1893 1894 count_vm_event(THP_SPLIT_PUD); 1895 1896 pudp_huge_clear_flush_notify(vma, haddr, pud); 1897 } 1898 1899 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 1900 unsigned long address) 1901 { 1902 spinlock_t *ptl; 1903 struct mmu_notifier_range range; 1904 1905 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, 1906 address & HPAGE_PUD_MASK, 1907 (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE); 1908 mmu_notifier_invalidate_range_start(&range); 1909 ptl = pud_lock(vma->vm_mm, pud); 1910 if (unlikely(!pud_trans_huge(*pud) && !pud_devmap(*pud))) 1911 goto out; 1912 __split_huge_pud_locked(vma, pud, range.start); 1913 1914 out: 1915 spin_unlock(ptl); 1916 /* 1917 * No need to double call mmu_notifier->invalidate_range() callback as 1918 * the above pudp_huge_clear_flush_notify() did already call it. 1919 */ 1920 mmu_notifier_invalidate_range_only_end(&range); 1921 } 1922 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1923 1924 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 1925 unsigned long haddr, pmd_t *pmd) 1926 { 1927 struct mm_struct *mm = vma->vm_mm; 1928 pgtable_t pgtable; 1929 pmd_t _pmd; 1930 int i; 1931 1932 /* 1933 * Leave pmd empty until pte is filled note that it is fine to delay 1934 * notification until mmu_notifier_invalidate_range_end() as we are 1935 * replacing a zero pmd write protected page with a zero pte write 1936 * protected page. 1937 * 1938 * See Documentation/vm/mmu_notifier.rst 1939 */ 1940 pmdp_huge_clear_flush(vma, haddr, pmd); 1941 1942 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 1943 pmd_populate(mm, &_pmd, pgtable); 1944 1945 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 1946 pte_t *pte, entry; 1947 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot); 1948 entry = pte_mkspecial(entry); 1949 pte = pte_offset_map(&_pmd, haddr); 1950 VM_BUG_ON(!pte_none(*pte)); 1951 set_pte_at(mm, haddr, pte, entry); 1952 pte_unmap(pte); 1953 } 1954 smp_wmb(); /* make pte visible before pmd */ 1955 pmd_populate(mm, pmd, pgtable); 1956 } 1957 1958 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 1959 unsigned long haddr, bool freeze) 1960 { 1961 struct mm_struct *mm = vma->vm_mm; 1962 struct page *page; 1963 pgtable_t pgtable; 1964 pmd_t old_pmd, _pmd; 1965 bool young, write, soft_dirty, pmd_migration = false, uffd_wp = false; 1966 unsigned long addr; 1967 int i; 1968 1969 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 1970 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 1971 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 1972 VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd) 1973 && !pmd_devmap(*pmd)); 1974 1975 count_vm_event(THP_SPLIT_PMD); 1976 1977 if (!vma_is_anonymous(vma)) { 1978 old_pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd); 1979 /* 1980 * We are going to unmap this huge page. So 1981 * just go ahead and zap it 1982 */ 1983 if (arch_needs_pgtable_deposit()) 1984 zap_deposited_table(mm, pmd); 1985 if (vma_is_special_huge(vma)) 1986 return; 1987 if (unlikely(is_pmd_migration_entry(old_pmd))) { 1988 swp_entry_t entry; 1989 1990 entry = pmd_to_swp_entry(old_pmd); 1991 page = pfn_swap_entry_to_page(entry); 1992 } else { 1993 page = pmd_page(old_pmd); 1994 if (!PageDirty(page) && pmd_dirty(old_pmd)) 1995 set_page_dirty(page); 1996 if (!PageReferenced(page) && pmd_young(old_pmd)) 1997 SetPageReferenced(page); 1998 page_remove_rmap(page, true); 1999 put_page(page); 2000 } 2001 add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR); 2002 return; 2003 } 2004 2005 if (is_huge_zero_pmd(*pmd)) { 2006 /* 2007 * FIXME: Do we want to invalidate secondary mmu by calling 2008 * mmu_notifier_invalidate_range() see comments below inside 2009 * __split_huge_pmd() ? 2010 * 2011 * We are going from a zero huge page write protected to zero 2012 * small page also write protected so it does not seems useful 2013 * to invalidate secondary mmu at this time. 2014 */ 2015 return __split_huge_zero_page_pmd(vma, haddr, pmd); 2016 } 2017 2018 /* 2019 * Up to this point the pmd is present and huge and userland has the 2020 * whole access to the hugepage during the split (which happens in 2021 * place). If we overwrite the pmd with the not-huge version pointing 2022 * to the pte here (which of course we could if all CPUs were bug 2023 * free), userland could trigger a small page size TLB miss on the 2024 * small sized TLB while the hugepage TLB entry is still established in 2025 * the huge TLB. Some CPU doesn't like that. 2026 * See http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum 2027 * 383 on page 105. Intel should be safe but is also warns that it's 2028 * only safe if the permission and cache attributes of the two entries 2029 * loaded in the two TLB is identical (which should be the case here). 2030 * But it is generally safer to never allow small and huge TLB entries 2031 * for the same virtual address to be loaded simultaneously. So instead 2032 * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the 2033 * current pmd notpresent (atomically because here the pmd_trans_huge 2034 * must remain set at all times on the pmd until the split is complete 2035 * for this pmd), then we flush the SMP TLB and finally we write the 2036 * non-huge version of the pmd entry with pmd_populate. 2037 */ 2038 old_pmd = pmdp_invalidate(vma, haddr, pmd); 2039 2040 pmd_migration = is_pmd_migration_entry(old_pmd); 2041 if (unlikely(pmd_migration)) { 2042 swp_entry_t entry; 2043 2044 entry = pmd_to_swp_entry(old_pmd); 2045 page = pfn_swap_entry_to_page(entry); 2046 write = is_writable_migration_entry(entry); 2047 young = false; 2048 soft_dirty = pmd_swp_soft_dirty(old_pmd); 2049 uffd_wp = pmd_swp_uffd_wp(old_pmd); 2050 } else { 2051 page = pmd_page(old_pmd); 2052 if (pmd_dirty(old_pmd)) 2053 SetPageDirty(page); 2054 write = pmd_write(old_pmd); 2055 young = pmd_young(old_pmd); 2056 soft_dirty = pmd_soft_dirty(old_pmd); 2057 uffd_wp = pmd_uffd_wp(old_pmd); 2058 } 2059 VM_BUG_ON_PAGE(!page_count(page), page); 2060 page_ref_add(page, HPAGE_PMD_NR - 1); 2061 2062 /* 2063 * Withdraw the table only after we mark the pmd entry invalid. 2064 * This's critical for some architectures (Power). 2065 */ 2066 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2067 pmd_populate(mm, &_pmd, pgtable); 2068 2069 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2070 pte_t entry, *pte; 2071 /* 2072 * Note that NUMA hinting access restrictions are not 2073 * transferred to avoid any possibility of altering 2074 * permissions across VMAs. 2075 */ 2076 if (freeze || pmd_migration) { 2077 swp_entry_t swp_entry; 2078 if (write) 2079 swp_entry = make_writable_migration_entry( 2080 page_to_pfn(page + i)); 2081 else 2082 swp_entry = make_readable_migration_entry( 2083 page_to_pfn(page + i)); 2084 entry = swp_entry_to_pte(swp_entry); 2085 if (soft_dirty) 2086 entry = pte_swp_mksoft_dirty(entry); 2087 if (uffd_wp) 2088 entry = pte_swp_mkuffd_wp(entry); 2089 } else { 2090 entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot)); 2091 entry = maybe_mkwrite(entry, vma); 2092 if (!write) 2093 entry = pte_wrprotect(entry); 2094 if (!young) 2095 entry = pte_mkold(entry); 2096 if (soft_dirty) 2097 entry = pte_mksoft_dirty(entry); 2098 if (uffd_wp) 2099 entry = pte_mkuffd_wp(entry); 2100 } 2101 pte = pte_offset_map(&_pmd, addr); 2102 BUG_ON(!pte_none(*pte)); 2103 set_pte_at(mm, addr, pte, entry); 2104 if (!pmd_migration) 2105 atomic_inc(&page[i]._mapcount); 2106 pte_unmap(pte); 2107 } 2108 2109 if (!pmd_migration) { 2110 /* 2111 * Set PG_double_map before dropping compound_mapcount to avoid 2112 * false-negative page_mapped(). 2113 */ 2114 if (compound_mapcount(page) > 1 && 2115 !TestSetPageDoubleMap(page)) { 2116 for (i = 0; i < HPAGE_PMD_NR; i++) 2117 atomic_inc(&page[i]._mapcount); 2118 } 2119 2120 lock_page_memcg(page); 2121 if (atomic_add_negative(-1, compound_mapcount_ptr(page))) { 2122 /* Last compound_mapcount is gone. */ 2123 __mod_lruvec_page_state(page, NR_ANON_THPS, 2124 -HPAGE_PMD_NR); 2125 if (TestClearPageDoubleMap(page)) { 2126 /* No need in mapcount reference anymore */ 2127 for (i = 0; i < HPAGE_PMD_NR; i++) 2128 atomic_dec(&page[i]._mapcount); 2129 } 2130 } 2131 unlock_page_memcg(page); 2132 } 2133 2134 smp_wmb(); /* make pte visible before pmd */ 2135 pmd_populate(mm, pmd, pgtable); 2136 2137 if (freeze) { 2138 for (i = 0; i < HPAGE_PMD_NR; i++) { 2139 page_remove_rmap(page + i, false); 2140 put_page(page + i); 2141 } 2142 } 2143 } 2144 2145 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 2146 unsigned long address, bool freeze, struct page *page) 2147 { 2148 spinlock_t *ptl; 2149 struct mmu_notifier_range range; 2150 bool do_unlock_page = false; 2151 pmd_t _pmd; 2152 2153 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, 2154 address & HPAGE_PMD_MASK, 2155 (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE); 2156 mmu_notifier_invalidate_range_start(&range); 2157 ptl = pmd_lock(vma->vm_mm, pmd); 2158 2159 /* 2160 * If caller asks to setup a migration entries, we need a page to check 2161 * pmd against. Otherwise we can end up replacing wrong page. 2162 */ 2163 VM_BUG_ON(freeze && !page); 2164 if (page) { 2165 VM_WARN_ON_ONCE(!PageLocked(page)); 2166 if (page != pmd_page(*pmd)) 2167 goto out; 2168 } 2169 2170 repeat: 2171 if (pmd_trans_huge(*pmd)) { 2172 if (!page) { 2173 page = pmd_page(*pmd); 2174 /* 2175 * An anonymous page must be locked, to ensure that a 2176 * concurrent reuse_swap_page() sees stable mapcount; 2177 * but reuse_swap_page() is not used on shmem or file, 2178 * and page lock must not be taken when zap_pmd_range() 2179 * calls __split_huge_pmd() while i_mmap_lock is held. 2180 */ 2181 if (PageAnon(page)) { 2182 if (unlikely(!trylock_page(page))) { 2183 get_page(page); 2184 _pmd = *pmd; 2185 spin_unlock(ptl); 2186 lock_page(page); 2187 spin_lock(ptl); 2188 if (unlikely(!pmd_same(*pmd, _pmd))) { 2189 unlock_page(page); 2190 put_page(page); 2191 page = NULL; 2192 goto repeat; 2193 } 2194 put_page(page); 2195 } 2196 do_unlock_page = true; 2197 } 2198 } 2199 if (PageMlocked(page)) 2200 clear_page_mlock(page); 2201 } else if (!(pmd_devmap(*pmd) || is_pmd_migration_entry(*pmd))) 2202 goto out; 2203 __split_huge_pmd_locked(vma, pmd, range.start, freeze); 2204 out: 2205 spin_unlock(ptl); 2206 if (do_unlock_page) 2207 unlock_page(page); 2208 /* 2209 * No need to double call mmu_notifier->invalidate_range() callback. 2210 * They are 3 cases to consider inside __split_huge_pmd_locked(): 2211 * 1) pmdp_huge_clear_flush_notify() call invalidate_range() obvious 2212 * 2) __split_huge_zero_page_pmd() read only zero page and any write 2213 * fault will trigger a flush_notify before pointing to a new page 2214 * (it is fine if the secondary mmu keeps pointing to the old zero 2215 * page in the meantime) 2216 * 3) Split a huge pmd into pte pointing to the same page. No need 2217 * to invalidate secondary tlb entry they are all still valid. 2218 * any further changes to individual pte will notify. So no need 2219 * to call mmu_notifier->invalidate_range() 2220 */ 2221 mmu_notifier_invalidate_range_only_end(&range); 2222 } 2223 2224 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, 2225 bool freeze, struct page *page) 2226 { 2227 pgd_t *pgd; 2228 p4d_t *p4d; 2229 pud_t *pud; 2230 pmd_t *pmd; 2231 2232 pgd = pgd_offset(vma->vm_mm, address); 2233 if (!pgd_present(*pgd)) 2234 return; 2235 2236 p4d = p4d_offset(pgd, address); 2237 if (!p4d_present(*p4d)) 2238 return; 2239 2240 pud = pud_offset(p4d, address); 2241 if (!pud_present(*pud)) 2242 return; 2243 2244 pmd = pmd_offset(pud, address); 2245 2246 __split_huge_pmd(vma, pmd, address, freeze, page); 2247 } 2248 2249 static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address) 2250 { 2251 /* 2252 * If the new address isn't hpage aligned and it could previously 2253 * contain an hugepage: check if we need to split an huge pmd. 2254 */ 2255 if (!IS_ALIGNED(address, HPAGE_PMD_SIZE) && 2256 range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE), 2257 ALIGN(address, HPAGE_PMD_SIZE))) 2258 split_huge_pmd_address(vma, address, false, NULL); 2259 } 2260 2261 void vma_adjust_trans_huge(struct vm_area_struct *vma, 2262 unsigned long start, 2263 unsigned long end, 2264 long adjust_next) 2265 { 2266 /* Check if we need to split start first. */ 2267 split_huge_pmd_if_needed(vma, start); 2268 2269 /* Check if we need to split end next. */ 2270 split_huge_pmd_if_needed(vma, end); 2271 2272 /* 2273 * If we're also updating the vma->vm_next->vm_start, 2274 * check if we need to split it. 2275 */ 2276 if (adjust_next > 0) { 2277 struct vm_area_struct *next = vma->vm_next; 2278 unsigned long nstart = next->vm_start; 2279 nstart += adjust_next; 2280 split_huge_pmd_if_needed(next, nstart); 2281 } 2282 } 2283 2284 static void unmap_page(struct page *page) 2285 { 2286 enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD | 2287 TTU_SYNC; 2288 2289 VM_BUG_ON_PAGE(!PageHead(page), page); 2290 2291 /* 2292 * Anon pages need migration entries to preserve them, but file 2293 * pages can simply be left unmapped, then faulted back on demand. 2294 * If that is ever changed (perhaps for mlock), update remap_page(). 2295 */ 2296 if (PageAnon(page)) 2297 try_to_migrate(page, ttu_flags); 2298 else 2299 try_to_unmap(page, ttu_flags | TTU_IGNORE_MLOCK); 2300 2301 VM_WARN_ON_ONCE_PAGE(page_mapped(page), page); 2302 } 2303 2304 static void remap_page(struct page *page, unsigned int nr) 2305 { 2306 int i; 2307 2308 /* If unmap_page() uses try_to_migrate() on file, remove this check */ 2309 if (!PageAnon(page)) 2310 return; 2311 if (PageTransHuge(page)) { 2312 remove_migration_ptes(page, page, true); 2313 } else { 2314 for (i = 0; i < nr; i++) 2315 remove_migration_ptes(page + i, page + i, true); 2316 } 2317 } 2318 2319 static void lru_add_page_tail(struct page *head, struct page *tail, 2320 struct lruvec *lruvec, struct list_head *list) 2321 { 2322 VM_BUG_ON_PAGE(!PageHead(head), head); 2323 VM_BUG_ON_PAGE(PageCompound(tail), head); 2324 VM_BUG_ON_PAGE(PageLRU(tail), head); 2325 lockdep_assert_held(&lruvec->lru_lock); 2326 2327 if (list) { 2328 /* page reclaim is reclaiming a huge page */ 2329 VM_WARN_ON(PageLRU(head)); 2330 get_page(tail); 2331 list_add_tail(&tail->lru, list); 2332 } else { 2333 /* head is still on lru (and we have it frozen) */ 2334 VM_WARN_ON(!PageLRU(head)); 2335 SetPageLRU(tail); 2336 list_add_tail(&tail->lru, &head->lru); 2337 } 2338 } 2339 2340 static void __split_huge_page_tail(struct page *head, int tail, 2341 struct lruvec *lruvec, struct list_head *list) 2342 { 2343 struct page *page_tail = head + tail; 2344 2345 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail); 2346 2347 /* 2348 * Clone page flags before unfreezing refcount. 2349 * 2350 * After successful get_page_unless_zero() might follow flags change, 2351 * for example lock_page() which set PG_waiters. 2352 */ 2353 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 2354 page_tail->flags |= (head->flags & 2355 ((1L << PG_referenced) | 2356 (1L << PG_swapbacked) | 2357 (1L << PG_swapcache) | 2358 (1L << PG_mlocked) | 2359 (1L << PG_uptodate) | 2360 (1L << PG_active) | 2361 (1L << PG_workingset) | 2362 (1L << PG_locked) | 2363 (1L << PG_unevictable) | 2364 #ifdef CONFIG_64BIT 2365 (1L << PG_arch_2) | 2366 #endif 2367 (1L << PG_dirty))); 2368 2369 /* ->mapping in first tail page is compound_mapcount */ 2370 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, 2371 page_tail); 2372 page_tail->mapping = head->mapping; 2373 page_tail->index = head->index + tail; 2374 2375 /* Page flags must be visible before we make the page non-compound. */ 2376 smp_wmb(); 2377 2378 /* 2379 * Clear PageTail before unfreezing page refcount. 2380 * 2381 * After successful get_page_unless_zero() might follow put_page() 2382 * which needs correct compound_head(). 2383 */ 2384 clear_compound_head(page_tail); 2385 2386 /* Finally unfreeze refcount. Additional reference from page cache. */ 2387 page_ref_unfreeze(page_tail, 1 + (!PageAnon(head) || 2388 PageSwapCache(head))); 2389 2390 if (page_is_young(head)) 2391 set_page_young(page_tail); 2392 if (page_is_idle(head)) 2393 set_page_idle(page_tail); 2394 2395 page_cpupid_xchg_last(page_tail, page_cpupid_last(head)); 2396 2397 /* 2398 * always add to the tail because some iterators expect new 2399 * pages to show after the currently processed elements - e.g. 2400 * migrate_pages 2401 */ 2402 lru_add_page_tail(head, page_tail, lruvec, list); 2403 } 2404 2405 static void __split_huge_page(struct page *page, struct list_head *list, 2406 pgoff_t end) 2407 { 2408 struct folio *folio = page_folio(page); 2409 struct page *head = &folio->page; 2410 struct lruvec *lruvec; 2411 struct address_space *swap_cache = NULL; 2412 unsigned long offset = 0; 2413 unsigned int nr = thp_nr_pages(head); 2414 int i; 2415 2416 /* complete memcg works before add pages to LRU */ 2417 split_page_memcg(head, nr); 2418 2419 if (PageAnon(head) && PageSwapCache(head)) { 2420 swp_entry_t entry = { .val = page_private(head) }; 2421 2422 offset = swp_offset(entry); 2423 swap_cache = swap_address_space(entry); 2424 xa_lock(&swap_cache->i_pages); 2425 } 2426 2427 /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ 2428 lruvec = folio_lruvec_lock(folio); 2429 2430 ClearPageHasHWPoisoned(head); 2431 2432 for (i = nr - 1; i >= 1; i--) { 2433 __split_huge_page_tail(head, i, lruvec, list); 2434 /* Some pages can be beyond EOF: drop them from page cache */ 2435 if (head[i].index >= end) { 2436 ClearPageDirty(head + i); 2437 __delete_from_page_cache(head + i, NULL); 2438 if (shmem_mapping(head->mapping)) 2439 shmem_uncharge(head->mapping->host, 1); 2440 put_page(head + i); 2441 } else if (!PageAnon(page)) { 2442 __xa_store(&head->mapping->i_pages, head[i].index, 2443 head + i, 0); 2444 } else if (swap_cache) { 2445 __xa_store(&swap_cache->i_pages, offset + i, 2446 head + i, 0); 2447 } 2448 } 2449 2450 ClearPageCompound(head); 2451 unlock_page_lruvec(lruvec); 2452 /* Caller disabled irqs, so they are still disabled here */ 2453 2454 split_page_owner(head, nr); 2455 2456 /* See comment in __split_huge_page_tail() */ 2457 if (PageAnon(head)) { 2458 /* Additional pin to swap cache */ 2459 if (PageSwapCache(head)) { 2460 page_ref_add(head, 2); 2461 xa_unlock(&swap_cache->i_pages); 2462 } else { 2463 page_ref_inc(head); 2464 } 2465 } else { 2466 /* Additional pin to page cache */ 2467 page_ref_add(head, 2); 2468 xa_unlock(&head->mapping->i_pages); 2469 } 2470 local_irq_enable(); 2471 2472 remap_page(head, nr); 2473 2474 if (PageSwapCache(head)) { 2475 swp_entry_t entry = { .val = page_private(head) }; 2476 2477 split_swap_cluster(entry); 2478 } 2479 2480 for (i = 0; i < nr; i++) { 2481 struct page *subpage = head + i; 2482 if (subpage == page) 2483 continue; 2484 unlock_page(subpage); 2485 2486 /* 2487 * Subpages may be freed if there wasn't any mapping 2488 * like if add_to_swap() is running on a lru page that 2489 * had its mapping zapped. And freeing these pages 2490 * requires taking the lru_lock so we do the put_page 2491 * of the tail pages after the split is complete. 2492 */ 2493 put_page(subpage); 2494 } 2495 } 2496 2497 int total_mapcount(struct page *page) 2498 { 2499 int i, compound, nr, ret; 2500 2501 VM_BUG_ON_PAGE(PageTail(page), page); 2502 2503 if (likely(!PageCompound(page))) 2504 return atomic_read(&page->_mapcount) + 1; 2505 2506 compound = compound_mapcount(page); 2507 nr = compound_nr(page); 2508 if (PageHuge(page)) 2509 return compound; 2510 ret = compound; 2511 for (i = 0; i < nr; i++) 2512 ret += atomic_read(&page[i]._mapcount) + 1; 2513 /* File pages has compound_mapcount included in _mapcount */ 2514 if (!PageAnon(page)) 2515 return ret - compound * nr; 2516 if (PageDoubleMap(page)) 2517 ret -= nr; 2518 return ret; 2519 } 2520 2521 /* 2522 * This calculates accurately how many mappings a transparent hugepage 2523 * has (unlike page_mapcount() which isn't fully accurate). This full 2524 * accuracy is primarily needed to know if copy-on-write faults can 2525 * reuse the page and change the mapping to read-write instead of 2526 * copying them. At the same time this returns the total_mapcount too. 2527 * 2528 * The function returns the highest mapcount any one of the subpages 2529 * has. If the return value is one, even if different processes are 2530 * mapping different subpages of the transparent hugepage, they can 2531 * all reuse it, because each process is reusing a different subpage. 2532 * 2533 * The total_mapcount is instead counting all virtual mappings of the 2534 * subpages. If the total_mapcount is equal to "one", it tells the 2535 * caller all mappings belong to the same "mm" and in turn the 2536 * anon_vma of the transparent hugepage can become the vma->anon_vma 2537 * local one as no other process may be mapping any of the subpages. 2538 * 2539 * It would be more accurate to replace page_mapcount() with 2540 * page_trans_huge_mapcount(), however we only use 2541 * page_trans_huge_mapcount() in the copy-on-write faults where we 2542 * need full accuracy to avoid breaking page pinning, because 2543 * page_trans_huge_mapcount() is slower than page_mapcount(). 2544 */ 2545 int page_trans_huge_mapcount(struct page *page) 2546 { 2547 int i, ret; 2548 2549 /* hugetlbfs shouldn't call it */ 2550 VM_BUG_ON_PAGE(PageHuge(page), page); 2551 2552 if (likely(!PageTransCompound(page))) 2553 return atomic_read(&page->_mapcount) + 1; 2554 2555 page = compound_head(page); 2556 2557 ret = 0; 2558 for (i = 0; i < thp_nr_pages(page); i++) { 2559 int mapcount = atomic_read(&page[i]._mapcount) + 1; 2560 ret = max(ret, mapcount); 2561 } 2562 2563 if (PageDoubleMap(page)) 2564 ret -= 1; 2565 2566 return ret + compound_mapcount(page); 2567 } 2568 2569 /* Racy check whether the huge page can be split */ 2570 bool can_split_huge_page(struct page *page, int *pextra_pins) 2571 { 2572 int extra_pins; 2573 2574 /* Additional pins from page cache */ 2575 if (PageAnon(page)) 2576 extra_pins = PageSwapCache(page) ? thp_nr_pages(page) : 0; 2577 else 2578 extra_pins = thp_nr_pages(page); 2579 if (pextra_pins) 2580 *pextra_pins = extra_pins; 2581 return total_mapcount(page) == page_count(page) - extra_pins - 1; 2582 } 2583 2584 /* 2585 * This function splits huge page into normal pages. @page can point to any 2586 * subpage of huge page to split. Split doesn't change the position of @page. 2587 * 2588 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY. 2589 * The huge page must be locked. 2590 * 2591 * If @list is null, tail pages will be added to LRU list, otherwise, to @list. 2592 * 2593 * Both head page and tail pages will inherit mapping, flags, and so on from 2594 * the hugepage. 2595 * 2596 * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if 2597 * they are not mapped. 2598 * 2599 * Returns 0 if the hugepage is split successfully. 2600 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under 2601 * us. 2602 */ 2603 int split_huge_page_to_list(struct page *page, struct list_head *list) 2604 { 2605 struct page *head = compound_head(page); 2606 struct deferred_split *ds_queue = get_deferred_split_queue(head); 2607 XA_STATE(xas, &head->mapping->i_pages, head->index); 2608 struct anon_vma *anon_vma = NULL; 2609 struct address_space *mapping = NULL; 2610 int extra_pins, ret; 2611 pgoff_t end; 2612 2613 VM_BUG_ON_PAGE(is_huge_zero_page(head), head); 2614 VM_BUG_ON_PAGE(!PageLocked(head), head); 2615 VM_BUG_ON_PAGE(!PageCompound(head), head); 2616 2617 if (PageWriteback(head)) 2618 return -EBUSY; 2619 2620 if (PageAnon(head)) { 2621 /* 2622 * The caller does not necessarily hold an mmap_lock that would 2623 * prevent the anon_vma disappearing so we first we take a 2624 * reference to it and then lock the anon_vma for write. This 2625 * is similar to page_lock_anon_vma_read except the write lock 2626 * is taken to serialise against parallel split or collapse 2627 * operations. 2628 */ 2629 anon_vma = page_get_anon_vma(head); 2630 if (!anon_vma) { 2631 ret = -EBUSY; 2632 goto out; 2633 } 2634 end = -1; 2635 mapping = NULL; 2636 anon_vma_lock_write(anon_vma); 2637 } else { 2638 mapping = head->mapping; 2639 2640 /* Truncated ? */ 2641 if (!mapping) { 2642 ret = -EBUSY; 2643 goto out; 2644 } 2645 2646 xas_split_alloc(&xas, head, compound_order(head), 2647 mapping_gfp_mask(mapping) & GFP_RECLAIM_MASK); 2648 if (xas_error(&xas)) { 2649 ret = xas_error(&xas); 2650 goto out; 2651 } 2652 2653 anon_vma = NULL; 2654 i_mmap_lock_read(mapping); 2655 2656 /* 2657 *__split_huge_page() may need to trim off pages beyond EOF: 2658 * but on 32-bit, i_size_read() takes an irq-unsafe seqlock, 2659 * which cannot be nested inside the page tree lock. So note 2660 * end now: i_size itself may be changed at any moment, but 2661 * head page lock is good enough to serialize the trimming. 2662 */ 2663 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); 2664 if (shmem_mapping(mapping)) 2665 end = shmem_fallocend(mapping->host, end); 2666 } 2667 2668 /* 2669 * Racy check if we can split the page, before unmap_page() will 2670 * split PMDs 2671 */ 2672 if (!can_split_huge_page(head, &extra_pins)) { 2673 ret = -EBUSY; 2674 goto out_unlock; 2675 } 2676 2677 unmap_page(head); 2678 2679 /* block interrupt reentry in xa_lock and spinlock */ 2680 local_irq_disable(); 2681 if (mapping) { 2682 /* 2683 * Check if the head page is present in page cache. 2684 * We assume all tail are present too, if head is there. 2685 */ 2686 xas_lock(&xas); 2687 xas_reset(&xas); 2688 if (xas_load(&xas) != head) 2689 goto fail; 2690 } 2691 2692 /* Prevent deferred_split_scan() touching ->_refcount */ 2693 spin_lock(&ds_queue->split_queue_lock); 2694 if (page_ref_freeze(head, 1 + extra_pins)) { 2695 if (!list_empty(page_deferred_list(head))) { 2696 ds_queue->split_queue_len--; 2697 list_del(page_deferred_list(head)); 2698 } 2699 spin_unlock(&ds_queue->split_queue_lock); 2700 if (mapping) { 2701 int nr = thp_nr_pages(head); 2702 2703 xas_split(&xas, head, thp_order(head)); 2704 if (PageSwapBacked(head)) { 2705 __mod_lruvec_page_state(head, NR_SHMEM_THPS, 2706 -nr); 2707 } else { 2708 __mod_lruvec_page_state(head, NR_FILE_THPS, 2709 -nr); 2710 filemap_nr_thps_dec(mapping); 2711 } 2712 } 2713 2714 __split_huge_page(page, list, end); 2715 ret = 0; 2716 } else { 2717 spin_unlock(&ds_queue->split_queue_lock); 2718 fail: 2719 if (mapping) 2720 xas_unlock(&xas); 2721 local_irq_enable(); 2722 remap_page(head, thp_nr_pages(head)); 2723 ret = -EBUSY; 2724 } 2725 2726 out_unlock: 2727 if (anon_vma) { 2728 anon_vma_unlock_write(anon_vma); 2729 put_anon_vma(anon_vma); 2730 } 2731 if (mapping) 2732 i_mmap_unlock_read(mapping); 2733 out: 2734 /* Free any memory we didn't use */ 2735 xas_nomem(&xas, 0); 2736 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); 2737 return ret; 2738 } 2739 2740 void free_transhuge_page(struct page *page) 2741 { 2742 struct deferred_split *ds_queue = get_deferred_split_queue(page); 2743 unsigned long flags; 2744 2745 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 2746 if (!list_empty(page_deferred_list(page))) { 2747 ds_queue->split_queue_len--; 2748 list_del(page_deferred_list(page)); 2749 } 2750 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 2751 free_compound_page(page); 2752 } 2753 2754 void deferred_split_huge_page(struct page *page) 2755 { 2756 struct deferred_split *ds_queue = get_deferred_split_queue(page); 2757 #ifdef CONFIG_MEMCG 2758 struct mem_cgroup *memcg = page_memcg(compound_head(page)); 2759 #endif 2760 unsigned long flags; 2761 2762 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 2763 2764 /* 2765 * The try_to_unmap() in page reclaim path might reach here too, 2766 * this may cause a race condition to corrupt deferred split queue. 2767 * And, if page reclaim is already handling the same page, it is 2768 * unnecessary to handle it again in shrinker. 2769 * 2770 * Check PageSwapCache to determine if the page is being 2771 * handled by page reclaim since THP swap would add the page into 2772 * swap cache before calling try_to_unmap(). 2773 */ 2774 if (PageSwapCache(page)) 2775 return; 2776 2777 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 2778 if (list_empty(page_deferred_list(page))) { 2779 count_vm_event(THP_DEFERRED_SPLIT_PAGE); 2780 list_add_tail(page_deferred_list(page), &ds_queue->split_queue); 2781 ds_queue->split_queue_len++; 2782 #ifdef CONFIG_MEMCG 2783 if (memcg) 2784 set_shrinker_bit(memcg, page_to_nid(page), 2785 deferred_split_shrinker.id); 2786 #endif 2787 } 2788 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 2789 } 2790 2791 static unsigned long deferred_split_count(struct shrinker *shrink, 2792 struct shrink_control *sc) 2793 { 2794 struct pglist_data *pgdata = NODE_DATA(sc->nid); 2795 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 2796 2797 #ifdef CONFIG_MEMCG 2798 if (sc->memcg) 2799 ds_queue = &sc->memcg->deferred_split_queue; 2800 #endif 2801 return READ_ONCE(ds_queue->split_queue_len); 2802 } 2803 2804 static unsigned long deferred_split_scan(struct shrinker *shrink, 2805 struct shrink_control *sc) 2806 { 2807 struct pglist_data *pgdata = NODE_DATA(sc->nid); 2808 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 2809 unsigned long flags; 2810 LIST_HEAD(list), *pos, *next; 2811 struct page *page; 2812 int split = 0; 2813 2814 #ifdef CONFIG_MEMCG 2815 if (sc->memcg) 2816 ds_queue = &sc->memcg->deferred_split_queue; 2817 #endif 2818 2819 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 2820 /* Take pin on all head pages to avoid freeing them under us */ 2821 list_for_each_safe(pos, next, &ds_queue->split_queue) { 2822 page = list_entry((void *)pos, struct page, deferred_list); 2823 page = compound_head(page); 2824 if (get_page_unless_zero(page)) { 2825 list_move(page_deferred_list(page), &list); 2826 } else { 2827 /* We lost race with put_compound_page() */ 2828 list_del_init(page_deferred_list(page)); 2829 ds_queue->split_queue_len--; 2830 } 2831 if (!--sc->nr_to_scan) 2832 break; 2833 } 2834 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 2835 2836 list_for_each_safe(pos, next, &list) { 2837 page = list_entry((void *)pos, struct page, deferred_list); 2838 if (!trylock_page(page)) 2839 goto next; 2840 /* split_huge_page() removes page from list on success */ 2841 if (!split_huge_page(page)) 2842 split++; 2843 unlock_page(page); 2844 next: 2845 put_page(page); 2846 } 2847 2848 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 2849 list_splice_tail(&list, &ds_queue->split_queue); 2850 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 2851 2852 /* 2853 * Stop shrinker if we didn't split any page, but the queue is empty. 2854 * This can happen if pages were freed under us. 2855 */ 2856 if (!split && list_empty(&ds_queue->split_queue)) 2857 return SHRINK_STOP; 2858 return split; 2859 } 2860 2861 static struct shrinker deferred_split_shrinker = { 2862 .count_objects = deferred_split_count, 2863 .scan_objects = deferred_split_scan, 2864 .seeks = DEFAULT_SEEKS, 2865 .flags = SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE | 2866 SHRINKER_NONSLAB, 2867 }; 2868 2869 #ifdef CONFIG_DEBUG_FS 2870 static void split_huge_pages_all(void) 2871 { 2872 struct zone *zone; 2873 struct page *page; 2874 unsigned long pfn, max_zone_pfn; 2875 unsigned long total = 0, split = 0; 2876 2877 pr_debug("Split all THPs\n"); 2878 for_each_populated_zone(zone) { 2879 max_zone_pfn = zone_end_pfn(zone); 2880 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 2881 if (!pfn_valid(pfn)) 2882 continue; 2883 2884 page = pfn_to_page(pfn); 2885 if (!get_page_unless_zero(page)) 2886 continue; 2887 2888 if (zone != page_zone(page)) 2889 goto next; 2890 2891 if (!PageHead(page) || PageHuge(page) || !PageLRU(page)) 2892 goto next; 2893 2894 total++; 2895 lock_page(page); 2896 if (!split_huge_page(page)) 2897 split++; 2898 unlock_page(page); 2899 next: 2900 put_page(page); 2901 cond_resched(); 2902 } 2903 } 2904 2905 pr_debug("%lu of %lu THP split\n", split, total); 2906 } 2907 2908 static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma) 2909 { 2910 return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) || 2911 is_vm_hugetlb_page(vma); 2912 } 2913 2914 static int split_huge_pages_pid(int pid, unsigned long vaddr_start, 2915 unsigned long vaddr_end) 2916 { 2917 int ret = 0; 2918 struct task_struct *task; 2919 struct mm_struct *mm; 2920 unsigned long total = 0, split = 0; 2921 unsigned long addr; 2922 2923 vaddr_start &= PAGE_MASK; 2924 vaddr_end &= PAGE_MASK; 2925 2926 /* Find the task_struct from pid */ 2927 rcu_read_lock(); 2928 task = find_task_by_vpid(pid); 2929 if (!task) { 2930 rcu_read_unlock(); 2931 ret = -ESRCH; 2932 goto out; 2933 } 2934 get_task_struct(task); 2935 rcu_read_unlock(); 2936 2937 /* Find the mm_struct */ 2938 mm = get_task_mm(task); 2939 put_task_struct(task); 2940 2941 if (!mm) { 2942 ret = -EINVAL; 2943 goto out; 2944 } 2945 2946 pr_debug("Split huge pages in pid: %d, vaddr: [0x%lx - 0x%lx]\n", 2947 pid, vaddr_start, vaddr_end); 2948 2949 mmap_read_lock(mm); 2950 /* 2951 * always increase addr by PAGE_SIZE, since we could have a PTE page 2952 * table filled with PTE-mapped THPs, each of which is distinct. 2953 */ 2954 for (addr = vaddr_start; addr < vaddr_end; addr += PAGE_SIZE) { 2955 struct vm_area_struct *vma = find_vma(mm, addr); 2956 unsigned int follflags; 2957 struct page *page; 2958 2959 if (!vma || addr < vma->vm_start) 2960 break; 2961 2962 /* skip special VMA and hugetlb VMA */ 2963 if (vma_not_suitable_for_thp_split(vma)) { 2964 addr = vma->vm_end; 2965 continue; 2966 } 2967 2968 /* FOLL_DUMP to ignore special (like zero) pages */ 2969 follflags = FOLL_GET | FOLL_DUMP; 2970 page = follow_page(vma, addr, follflags); 2971 2972 if (IS_ERR(page)) 2973 continue; 2974 if (!page) 2975 continue; 2976 2977 if (!is_transparent_hugepage(page)) 2978 goto next; 2979 2980 total++; 2981 if (!can_split_huge_page(compound_head(page), NULL)) 2982 goto next; 2983 2984 if (!trylock_page(page)) 2985 goto next; 2986 2987 if (!split_huge_page(page)) 2988 split++; 2989 2990 unlock_page(page); 2991 next: 2992 put_page(page); 2993 cond_resched(); 2994 } 2995 mmap_read_unlock(mm); 2996 mmput(mm); 2997 2998 pr_debug("%lu of %lu THP split\n", split, total); 2999 3000 out: 3001 return ret; 3002 } 3003 3004 static int split_huge_pages_in_file(const char *file_path, pgoff_t off_start, 3005 pgoff_t off_end) 3006 { 3007 struct filename *file; 3008 struct file *candidate; 3009 struct address_space *mapping; 3010 int ret = -EINVAL; 3011 pgoff_t index; 3012 int nr_pages = 1; 3013 unsigned long total = 0, split = 0; 3014 3015 file = getname_kernel(file_path); 3016 if (IS_ERR(file)) 3017 return ret; 3018 3019 candidate = file_open_name(file, O_RDONLY, 0); 3020 if (IS_ERR(candidate)) 3021 goto out; 3022 3023 pr_debug("split file-backed THPs in file: %s, page offset: [0x%lx - 0x%lx]\n", 3024 file_path, off_start, off_end); 3025 3026 mapping = candidate->f_mapping; 3027 3028 for (index = off_start; index < off_end; index += nr_pages) { 3029 struct page *fpage = pagecache_get_page(mapping, index, 3030 FGP_ENTRY | FGP_HEAD, 0); 3031 3032 nr_pages = 1; 3033 if (xa_is_value(fpage) || !fpage) 3034 continue; 3035 3036 if (!is_transparent_hugepage(fpage)) 3037 goto next; 3038 3039 total++; 3040 nr_pages = thp_nr_pages(fpage); 3041 3042 if (!trylock_page(fpage)) 3043 goto next; 3044 3045 if (!split_huge_page(fpage)) 3046 split++; 3047 3048 unlock_page(fpage); 3049 next: 3050 put_page(fpage); 3051 cond_resched(); 3052 } 3053 3054 filp_close(candidate, NULL); 3055 ret = 0; 3056 3057 pr_debug("%lu of %lu file-backed THP split\n", split, total); 3058 out: 3059 putname(file); 3060 return ret; 3061 } 3062 3063 #define MAX_INPUT_BUF_SZ 255 3064 3065 static ssize_t split_huge_pages_write(struct file *file, const char __user *buf, 3066 size_t count, loff_t *ppops) 3067 { 3068 static DEFINE_MUTEX(split_debug_mutex); 3069 ssize_t ret; 3070 /* hold pid, start_vaddr, end_vaddr or file_path, off_start, off_end */ 3071 char input_buf[MAX_INPUT_BUF_SZ]; 3072 int pid; 3073 unsigned long vaddr_start, vaddr_end; 3074 3075 ret = mutex_lock_interruptible(&split_debug_mutex); 3076 if (ret) 3077 return ret; 3078 3079 ret = -EFAULT; 3080 3081 memset(input_buf, 0, MAX_INPUT_BUF_SZ); 3082 if (copy_from_user(input_buf, buf, min_t(size_t, count, MAX_INPUT_BUF_SZ))) 3083 goto out; 3084 3085 input_buf[MAX_INPUT_BUF_SZ - 1] = '\0'; 3086 3087 if (input_buf[0] == '/') { 3088 char *tok; 3089 char *buf = input_buf; 3090 char file_path[MAX_INPUT_BUF_SZ]; 3091 pgoff_t off_start = 0, off_end = 0; 3092 size_t input_len = strlen(input_buf); 3093 3094 tok = strsep(&buf, ","); 3095 if (tok) { 3096 strcpy(file_path, tok); 3097 } else { 3098 ret = -EINVAL; 3099 goto out; 3100 } 3101 3102 ret = sscanf(buf, "0x%lx,0x%lx", &off_start, &off_end); 3103 if (ret != 2) { 3104 ret = -EINVAL; 3105 goto out; 3106 } 3107 ret = split_huge_pages_in_file(file_path, off_start, off_end); 3108 if (!ret) 3109 ret = input_len; 3110 3111 goto out; 3112 } 3113 3114 ret = sscanf(input_buf, "%d,0x%lx,0x%lx", &pid, &vaddr_start, &vaddr_end); 3115 if (ret == 1 && pid == 1) { 3116 split_huge_pages_all(); 3117 ret = strlen(input_buf); 3118 goto out; 3119 } else if (ret != 3) { 3120 ret = -EINVAL; 3121 goto out; 3122 } 3123 3124 ret = split_huge_pages_pid(pid, vaddr_start, vaddr_end); 3125 if (!ret) 3126 ret = strlen(input_buf); 3127 out: 3128 mutex_unlock(&split_debug_mutex); 3129 return ret; 3130 3131 } 3132 3133 static const struct file_operations split_huge_pages_fops = { 3134 .owner = THIS_MODULE, 3135 .write = split_huge_pages_write, 3136 .llseek = no_llseek, 3137 }; 3138 3139 static int __init split_huge_pages_debugfs(void) 3140 { 3141 debugfs_create_file("split_huge_pages", 0200, NULL, NULL, 3142 &split_huge_pages_fops); 3143 return 0; 3144 } 3145 late_initcall(split_huge_pages_debugfs); 3146 #endif 3147 3148 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 3149 void set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 3150 struct page *page) 3151 { 3152 struct vm_area_struct *vma = pvmw->vma; 3153 struct mm_struct *mm = vma->vm_mm; 3154 unsigned long address = pvmw->address; 3155 pmd_t pmdval; 3156 swp_entry_t entry; 3157 pmd_t pmdswp; 3158 3159 if (!(pvmw->pmd && !pvmw->pte)) 3160 return; 3161 3162 flush_cache_range(vma, address, address + HPAGE_PMD_SIZE); 3163 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); 3164 if (pmd_dirty(pmdval)) 3165 set_page_dirty(page); 3166 if (pmd_write(pmdval)) 3167 entry = make_writable_migration_entry(page_to_pfn(page)); 3168 else 3169 entry = make_readable_migration_entry(page_to_pfn(page)); 3170 pmdswp = swp_entry_to_pmd(entry); 3171 if (pmd_soft_dirty(pmdval)) 3172 pmdswp = pmd_swp_mksoft_dirty(pmdswp); 3173 set_pmd_at(mm, address, pvmw->pmd, pmdswp); 3174 page_remove_rmap(page, true); 3175 put_page(page); 3176 } 3177 3178 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) 3179 { 3180 struct vm_area_struct *vma = pvmw->vma; 3181 struct mm_struct *mm = vma->vm_mm; 3182 unsigned long address = pvmw->address; 3183 unsigned long mmun_start = address & HPAGE_PMD_MASK; 3184 pmd_t pmde; 3185 swp_entry_t entry; 3186 3187 if (!(pvmw->pmd && !pvmw->pte)) 3188 return; 3189 3190 entry = pmd_to_swp_entry(*pvmw->pmd); 3191 get_page(new); 3192 pmde = pmd_mkold(mk_huge_pmd(new, vma->vm_page_prot)); 3193 if (pmd_swp_soft_dirty(*pvmw->pmd)) 3194 pmde = pmd_mksoft_dirty(pmde); 3195 if (is_writable_migration_entry(entry)) 3196 pmde = maybe_pmd_mkwrite(pmde, vma); 3197 if (pmd_swp_uffd_wp(*pvmw->pmd)) 3198 pmde = pmd_wrprotect(pmd_mkuffd_wp(pmde)); 3199 3200 flush_cache_range(vma, mmun_start, mmun_start + HPAGE_PMD_SIZE); 3201 if (PageAnon(new)) 3202 page_add_anon_rmap(new, vma, mmun_start, true); 3203 else 3204 page_add_file_rmap(new, true); 3205 set_pmd_at(mm, mmun_start, pvmw->pmd, pmde); 3206 if ((vma->vm_flags & VM_LOCKED) && !PageDoubleMap(new)) 3207 mlock_vma_page(new); 3208 update_mmu_cache_pmd(vma, address, pvmw->pmd); 3209 } 3210 #endif 3211