1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2009 Red Hat, Inc. 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/mm.h> 9 #include <linux/sched.h> 10 #include <linux/sched/mm.h> 11 #include <linux/sched/coredump.h> 12 #include <linux/sched/numa_balancing.h> 13 #include <linux/highmem.h> 14 #include <linux/hugetlb.h> 15 #include <linux/mmu_notifier.h> 16 #include <linux/rmap.h> 17 #include <linux/swap.h> 18 #include <linux/shrinker.h> 19 #include <linux/mm_inline.h> 20 #include <linux/swapops.h> 21 #include <linux/dax.h> 22 #include <linux/khugepaged.h> 23 #include <linux/freezer.h> 24 #include <linux/pfn_t.h> 25 #include <linux/mman.h> 26 #include <linux/memremap.h> 27 #include <linux/pagemap.h> 28 #include <linux/debugfs.h> 29 #include <linux/migrate.h> 30 #include <linux/hashtable.h> 31 #include <linux/userfaultfd_k.h> 32 #include <linux/page_idle.h> 33 #include <linux/shmem_fs.h> 34 #include <linux/oom.h> 35 #include <linux/numa.h> 36 #include <linux/page_owner.h> 37 38 #include <asm/tlb.h> 39 #include <asm/pgalloc.h> 40 #include "internal.h" 41 42 /* 43 * By default, transparent hugepage support is disabled in order to avoid 44 * risking an increased memory footprint for applications that are not 45 * guaranteed to benefit from it. When transparent hugepage support is 46 * enabled, it is for all mappings, and khugepaged scans all mappings. 47 * Defrag is invoked by khugepaged hugepage allocations and by page faults 48 * for all hugepage allocations. 49 */ 50 unsigned long transparent_hugepage_flags __read_mostly = 51 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS 52 (1<<TRANSPARENT_HUGEPAGE_FLAG)| 53 #endif 54 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE 55 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| 56 #endif 57 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)| 58 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| 59 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 60 61 static struct shrinker deferred_split_shrinker; 62 63 static atomic_t huge_zero_refcount; 64 struct page *huge_zero_page __read_mostly; 65 unsigned long huge_zero_pfn __read_mostly = ~0UL; 66 67 static inline bool file_thp_enabled(struct vm_area_struct *vma) 68 { 69 return transhuge_vma_enabled(vma, vma->vm_flags) && vma->vm_file && 70 !inode_is_open_for_write(vma->vm_file->f_inode) && 71 (vma->vm_flags & VM_EXEC); 72 } 73 74 bool transparent_hugepage_active(struct vm_area_struct *vma) 75 { 76 /* The addr is used to check if the vma size fits */ 77 unsigned long addr = (vma->vm_end & HPAGE_PMD_MASK) - HPAGE_PMD_SIZE; 78 79 if (!transhuge_vma_suitable(vma, addr)) 80 return false; 81 if (vma_is_anonymous(vma)) 82 return __transparent_hugepage_enabled(vma); 83 if (vma_is_shmem(vma)) 84 return shmem_huge_enabled(vma); 85 if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS)) 86 return file_thp_enabled(vma); 87 88 return false; 89 } 90 91 static bool get_huge_zero_page(void) 92 { 93 struct page *zero_page; 94 retry: 95 if (likely(atomic_inc_not_zero(&huge_zero_refcount))) 96 return true; 97 98 zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE, 99 HPAGE_PMD_ORDER); 100 if (!zero_page) { 101 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); 102 return false; 103 } 104 count_vm_event(THP_ZERO_PAGE_ALLOC); 105 preempt_disable(); 106 if (cmpxchg(&huge_zero_page, NULL, zero_page)) { 107 preempt_enable(); 108 __free_pages(zero_page, compound_order(zero_page)); 109 goto retry; 110 } 111 WRITE_ONCE(huge_zero_pfn, page_to_pfn(zero_page)); 112 113 /* We take additional reference here. It will be put back by shrinker */ 114 atomic_set(&huge_zero_refcount, 2); 115 preempt_enable(); 116 return true; 117 } 118 119 static void put_huge_zero_page(void) 120 { 121 /* 122 * Counter should never go to zero here. Only shrinker can put 123 * last reference. 124 */ 125 BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); 126 } 127 128 struct page *mm_get_huge_zero_page(struct mm_struct *mm) 129 { 130 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 131 return READ_ONCE(huge_zero_page); 132 133 if (!get_huge_zero_page()) 134 return NULL; 135 136 if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 137 put_huge_zero_page(); 138 139 return READ_ONCE(huge_zero_page); 140 } 141 142 void mm_put_huge_zero_page(struct mm_struct *mm) 143 { 144 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 145 put_huge_zero_page(); 146 } 147 148 static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink, 149 struct shrink_control *sc) 150 { 151 /* we can free zero page only if last reference remains */ 152 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; 153 } 154 155 static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink, 156 struct shrink_control *sc) 157 { 158 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { 159 struct page *zero_page = xchg(&huge_zero_page, NULL); 160 BUG_ON(zero_page == NULL); 161 WRITE_ONCE(huge_zero_pfn, ~0UL); 162 __free_pages(zero_page, compound_order(zero_page)); 163 return HPAGE_PMD_NR; 164 } 165 166 return 0; 167 } 168 169 static struct shrinker huge_zero_page_shrinker = { 170 .count_objects = shrink_huge_zero_page_count, 171 .scan_objects = shrink_huge_zero_page_scan, 172 .seeks = DEFAULT_SEEKS, 173 }; 174 175 #ifdef CONFIG_SYSFS 176 static ssize_t enabled_show(struct kobject *kobj, 177 struct kobj_attribute *attr, char *buf) 178 { 179 const char *output; 180 181 if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags)) 182 output = "[always] madvise never"; 183 else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 184 &transparent_hugepage_flags)) 185 output = "always [madvise] never"; 186 else 187 output = "always madvise [never]"; 188 189 return sysfs_emit(buf, "%s\n", output); 190 } 191 192 static ssize_t enabled_store(struct kobject *kobj, 193 struct kobj_attribute *attr, 194 const char *buf, size_t count) 195 { 196 ssize_t ret = count; 197 198 if (sysfs_streq(buf, "always")) { 199 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 200 set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 201 } else if (sysfs_streq(buf, "madvise")) { 202 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 203 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 204 } else if (sysfs_streq(buf, "never")) { 205 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 206 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 207 } else 208 ret = -EINVAL; 209 210 if (ret > 0) { 211 int err = start_stop_khugepaged(); 212 if (err) 213 ret = err; 214 } 215 return ret; 216 } 217 static struct kobj_attribute enabled_attr = 218 __ATTR(enabled, 0644, enabled_show, enabled_store); 219 220 ssize_t single_hugepage_flag_show(struct kobject *kobj, 221 struct kobj_attribute *attr, char *buf, 222 enum transparent_hugepage_flag flag) 223 { 224 return sysfs_emit(buf, "%d\n", 225 !!test_bit(flag, &transparent_hugepage_flags)); 226 } 227 228 ssize_t single_hugepage_flag_store(struct kobject *kobj, 229 struct kobj_attribute *attr, 230 const char *buf, size_t count, 231 enum transparent_hugepage_flag flag) 232 { 233 unsigned long value; 234 int ret; 235 236 ret = kstrtoul(buf, 10, &value); 237 if (ret < 0) 238 return ret; 239 if (value > 1) 240 return -EINVAL; 241 242 if (value) 243 set_bit(flag, &transparent_hugepage_flags); 244 else 245 clear_bit(flag, &transparent_hugepage_flags); 246 247 return count; 248 } 249 250 static ssize_t defrag_show(struct kobject *kobj, 251 struct kobj_attribute *attr, char *buf) 252 { 253 const char *output; 254 255 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, 256 &transparent_hugepage_flags)) 257 output = "[always] defer defer+madvise madvise never"; 258 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, 259 &transparent_hugepage_flags)) 260 output = "always [defer] defer+madvise madvise never"; 261 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, 262 &transparent_hugepage_flags)) 263 output = "always defer [defer+madvise] madvise never"; 264 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, 265 &transparent_hugepage_flags)) 266 output = "always defer defer+madvise [madvise] never"; 267 else 268 output = "always defer defer+madvise madvise [never]"; 269 270 return sysfs_emit(buf, "%s\n", output); 271 } 272 273 static ssize_t defrag_store(struct kobject *kobj, 274 struct kobj_attribute *attr, 275 const char *buf, size_t count) 276 { 277 if (sysfs_streq(buf, "always")) { 278 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 279 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 280 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 281 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 282 } else if (sysfs_streq(buf, "defer+madvise")) { 283 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 284 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 285 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 286 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 287 } else if (sysfs_streq(buf, "defer")) { 288 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 289 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 290 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 291 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 292 } else if (sysfs_streq(buf, "madvise")) { 293 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 294 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 295 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 296 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 297 } else if (sysfs_streq(buf, "never")) { 298 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 299 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 300 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 301 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 302 } else 303 return -EINVAL; 304 305 return count; 306 } 307 static struct kobj_attribute defrag_attr = 308 __ATTR(defrag, 0644, defrag_show, defrag_store); 309 310 static ssize_t use_zero_page_show(struct kobject *kobj, 311 struct kobj_attribute *attr, char *buf) 312 { 313 return single_hugepage_flag_show(kobj, attr, buf, 314 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 315 } 316 static ssize_t use_zero_page_store(struct kobject *kobj, 317 struct kobj_attribute *attr, const char *buf, size_t count) 318 { 319 return single_hugepage_flag_store(kobj, attr, buf, count, 320 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 321 } 322 static struct kobj_attribute use_zero_page_attr = 323 __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store); 324 325 static ssize_t hpage_pmd_size_show(struct kobject *kobj, 326 struct kobj_attribute *attr, char *buf) 327 { 328 return sysfs_emit(buf, "%lu\n", HPAGE_PMD_SIZE); 329 } 330 static struct kobj_attribute hpage_pmd_size_attr = 331 __ATTR_RO(hpage_pmd_size); 332 333 static struct attribute *hugepage_attr[] = { 334 &enabled_attr.attr, 335 &defrag_attr.attr, 336 &use_zero_page_attr.attr, 337 &hpage_pmd_size_attr.attr, 338 #ifdef CONFIG_SHMEM 339 &shmem_enabled_attr.attr, 340 #endif 341 NULL, 342 }; 343 344 static const struct attribute_group hugepage_attr_group = { 345 .attrs = hugepage_attr, 346 }; 347 348 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj) 349 { 350 int err; 351 352 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj); 353 if (unlikely(!*hugepage_kobj)) { 354 pr_err("failed to create transparent hugepage kobject\n"); 355 return -ENOMEM; 356 } 357 358 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group); 359 if (err) { 360 pr_err("failed to register transparent hugepage group\n"); 361 goto delete_obj; 362 } 363 364 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group); 365 if (err) { 366 pr_err("failed to register transparent hugepage group\n"); 367 goto remove_hp_group; 368 } 369 370 return 0; 371 372 remove_hp_group: 373 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group); 374 delete_obj: 375 kobject_put(*hugepage_kobj); 376 return err; 377 } 378 379 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj) 380 { 381 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group); 382 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group); 383 kobject_put(hugepage_kobj); 384 } 385 #else 386 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj) 387 { 388 return 0; 389 } 390 391 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) 392 { 393 } 394 #endif /* CONFIG_SYSFS */ 395 396 static int __init hugepage_init(void) 397 { 398 int err; 399 struct kobject *hugepage_kobj; 400 401 if (!has_transparent_hugepage()) { 402 /* 403 * Hardware doesn't support hugepages, hence disable 404 * DAX PMD support. 405 */ 406 transparent_hugepage_flags = 1 << TRANSPARENT_HUGEPAGE_NEVER_DAX; 407 return -EINVAL; 408 } 409 410 /* 411 * hugepages can't be allocated by the buddy allocator 412 */ 413 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER >= MAX_ORDER); 414 /* 415 * we use page->mapping and page->index in second tail page 416 * as list_head: assuming THP order >= 2 417 */ 418 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER < 2); 419 420 err = hugepage_init_sysfs(&hugepage_kobj); 421 if (err) 422 goto err_sysfs; 423 424 err = khugepaged_init(); 425 if (err) 426 goto err_slab; 427 428 err = register_shrinker(&huge_zero_page_shrinker); 429 if (err) 430 goto err_hzp_shrinker; 431 err = register_shrinker(&deferred_split_shrinker); 432 if (err) 433 goto err_split_shrinker; 434 435 /* 436 * By default disable transparent hugepages on smaller systems, 437 * where the extra memory used could hurt more than TLB overhead 438 * is likely to save. The admin can still enable it through /sys. 439 */ 440 if (totalram_pages() < (512 << (20 - PAGE_SHIFT))) { 441 transparent_hugepage_flags = 0; 442 return 0; 443 } 444 445 err = start_stop_khugepaged(); 446 if (err) 447 goto err_khugepaged; 448 449 return 0; 450 err_khugepaged: 451 unregister_shrinker(&deferred_split_shrinker); 452 err_split_shrinker: 453 unregister_shrinker(&huge_zero_page_shrinker); 454 err_hzp_shrinker: 455 khugepaged_destroy(); 456 err_slab: 457 hugepage_exit_sysfs(hugepage_kobj); 458 err_sysfs: 459 return err; 460 } 461 subsys_initcall(hugepage_init); 462 463 static int __init setup_transparent_hugepage(char *str) 464 { 465 int ret = 0; 466 if (!str) 467 goto out; 468 if (!strcmp(str, "always")) { 469 set_bit(TRANSPARENT_HUGEPAGE_FLAG, 470 &transparent_hugepage_flags); 471 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 472 &transparent_hugepage_flags); 473 ret = 1; 474 } else if (!strcmp(str, "madvise")) { 475 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 476 &transparent_hugepage_flags); 477 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 478 &transparent_hugepage_flags); 479 ret = 1; 480 } else if (!strcmp(str, "never")) { 481 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 482 &transparent_hugepage_flags); 483 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 484 &transparent_hugepage_flags); 485 ret = 1; 486 } 487 out: 488 if (!ret) 489 pr_warn("transparent_hugepage= cannot parse, ignored\n"); 490 return ret; 491 } 492 __setup("transparent_hugepage=", setup_transparent_hugepage); 493 494 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 495 { 496 if (likely(vma->vm_flags & VM_WRITE)) 497 pmd = pmd_mkwrite(pmd); 498 return pmd; 499 } 500 501 #ifdef CONFIG_MEMCG 502 static inline struct deferred_split *get_deferred_split_queue(struct page *page) 503 { 504 struct mem_cgroup *memcg = page_memcg(compound_head(page)); 505 struct pglist_data *pgdat = NODE_DATA(page_to_nid(page)); 506 507 if (memcg) 508 return &memcg->deferred_split_queue; 509 else 510 return &pgdat->deferred_split_queue; 511 } 512 #else 513 static inline struct deferred_split *get_deferred_split_queue(struct page *page) 514 { 515 struct pglist_data *pgdat = NODE_DATA(page_to_nid(page)); 516 517 return &pgdat->deferred_split_queue; 518 } 519 #endif 520 521 void prep_transhuge_page(struct page *page) 522 { 523 /* 524 * we use page->mapping and page->indexlru in second tail page 525 * as list_head: assuming THP order >= 2 526 */ 527 528 INIT_LIST_HEAD(page_deferred_list(page)); 529 set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR); 530 } 531 532 bool is_transparent_hugepage(struct page *page) 533 { 534 if (!PageCompound(page)) 535 return false; 536 537 page = compound_head(page); 538 return is_huge_zero_page(page) || 539 page[1].compound_dtor == TRANSHUGE_PAGE_DTOR; 540 } 541 EXPORT_SYMBOL_GPL(is_transparent_hugepage); 542 543 static unsigned long __thp_get_unmapped_area(struct file *filp, 544 unsigned long addr, unsigned long len, 545 loff_t off, unsigned long flags, unsigned long size) 546 { 547 loff_t off_end = off + len; 548 loff_t off_align = round_up(off, size); 549 unsigned long len_pad, ret; 550 551 if (off_end <= off_align || (off_end - off_align) < size) 552 return 0; 553 554 len_pad = len + size; 555 if (len_pad < len || (off + len_pad) < off) 556 return 0; 557 558 ret = current->mm->get_unmapped_area(filp, addr, len_pad, 559 off >> PAGE_SHIFT, flags); 560 561 /* 562 * The failure might be due to length padding. The caller will retry 563 * without the padding. 564 */ 565 if (IS_ERR_VALUE(ret)) 566 return 0; 567 568 /* 569 * Do not try to align to THP boundary if allocation at the address 570 * hint succeeds. 571 */ 572 if (ret == addr) 573 return addr; 574 575 ret += (off - ret) & (size - 1); 576 return ret; 577 } 578 579 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, 580 unsigned long len, unsigned long pgoff, unsigned long flags) 581 { 582 unsigned long ret; 583 loff_t off = (loff_t)pgoff << PAGE_SHIFT; 584 585 if (!IS_DAX(filp->f_mapping->host) || !IS_ENABLED(CONFIG_FS_DAX_PMD)) 586 goto out; 587 588 ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE); 589 if (ret) 590 return ret; 591 out: 592 return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); 593 } 594 EXPORT_SYMBOL_GPL(thp_get_unmapped_area); 595 596 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, 597 struct page *page, gfp_t gfp) 598 { 599 struct vm_area_struct *vma = vmf->vma; 600 pgtable_t pgtable; 601 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 602 vm_fault_t ret = 0; 603 604 VM_BUG_ON_PAGE(!PageCompound(page), page); 605 606 if (mem_cgroup_charge(page, vma->vm_mm, gfp)) { 607 put_page(page); 608 count_vm_event(THP_FAULT_FALLBACK); 609 count_vm_event(THP_FAULT_FALLBACK_CHARGE); 610 return VM_FAULT_FALLBACK; 611 } 612 cgroup_throttle_swaprate(page, gfp); 613 614 pgtable = pte_alloc_one(vma->vm_mm); 615 if (unlikely(!pgtable)) { 616 ret = VM_FAULT_OOM; 617 goto release; 618 } 619 620 clear_huge_page(page, vmf->address, HPAGE_PMD_NR); 621 /* 622 * The memory barrier inside __SetPageUptodate makes sure that 623 * clear_huge_page writes become visible before the set_pmd_at() 624 * write. 625 */ 626 __SetPageUptodate(page); 627 628 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 629 if (unlikely(!pmd_none(*vmf->pmd))) { 630 goto unlock_release; 631 } else { 632 pmd_t entry; 633 634 ret = check_stable_address_space(vma->vm_mm); 635 if (ret) 636 goto unlock_release; 637 638 /* Deliver the page fault to userland */ 639 if (userfaultfd_missing(vma)) { 640 spin_unlock(vmf->ptl); 641 put_page(page); 642 pte_free(vma->vm_mm, pgtable); 643 ret = handle_userfault(vmf, VM_UFFD_MISSING); 644 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 645 return ret; 646 } 647 648 entry = mk_huge_pmd(page, vma->vm_page_prot); 649 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 650 page_add_new_anon_rmap(page, vma, haddr, true); 651 lru_cache_add_inactive_or_unevictable(page, vma); 652 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); 653 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); 654 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 655 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); 656 mm_inc_nr_ptes(vma->vm_mm); 657 spin_unlock(vmf->ptl); 658 count_vm_event(THP_FAULT_ALLOC); 659 count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); 660 } 661 662 return 0; 663 unlock_release: 664 spin_unlock(vmf->ptl); 665 release: 666 if (pgtable) 667 pte_free(vma->vm_mm, pgtable); 668 put_page(page); 669 return ret; 670 671 } 672 673 /* 674 * always: directly stall for all thp allocations 675 * defer: wake kswapd and fail if not immediately available 676 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise 677 * fail if not immediately available 678 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately 679 * available 680 * never: never stall for any thp allocation 681 */ 682 gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma) 683 { 684 const bool vma_madvised = vma && (vma->vm_flags & VM_HUGEPAGE); 685 686 /* Always do synchronous compaction */ 687 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) 688 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY); 689 690 /* Kick kcompactd and fail quickly */ 691 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) 692 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM; 693 694 /* Synchronous compaction if madvised, otherwise kick kcompactd */ 695 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags)) 696 return GFP_TRANSHUGE_LIGHT | 697 (vma_madvised ? __GFP_DIRECT_RECLAIM : 698 __GFP_KSWAPD_RECLAIM); 699 700 /* Only do synchronous compaction if madvised */ 701 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags)) 702 return GFP_TRANSHUGE_LIGHT | 703 (vma_madvised ? __GFP_DIRECT_RECLAIM : 0); 704 705 return GFP_TRANSHUGE_LIGHT; 706 } 707 708 /* Caller must hold page table lock. */ 709 static void set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm, 710 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, 711 struct page *zero_page) 712 { 713 pmd_t entry; 714 if (!pmd_none(*pmd)) 715 return; 716 entry = mk_pmd(zero_page, vma->vm_page_prot); 717 entry = pmd_mkhuge(entry); 718 if (pgtable) 719 pgtable_trans_huge_deposit(mm, pmd, pgtable); 720 set_pmd_at(mm, haddr, pmd, entry); 721 mm_inc_nr_ptes(mm); 722 } 723 724 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) 725 { 726 struct vm_area_struct *vma = vmf->vma; 727 gfp_t gfp; 728 struct page *page; 729 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 730 731 if (!transhuge_vma_suitable(vma, haddr)) 732 return VM_FAULT_FALLBACK; 733 if (unlikely(anon_vma_prepare(vma))) 734 return VM_FAULT_OOM; 735 if (unlikely(khugepaged_enter(vma, vma->vm_flags))) 736 return VM_FAULT_OOM; 737 if (!(vmf->flags & FAULT_FLAG_WRITE) && 738 !mm_forbids_zeropage(vma->vm_mm) && 739 transparent_hugepage_use_zero_page()) { 740 pgtable_t pgtable; 741 struct page *zero_page; 742 vm_fault_t ret; 743 pgtable = pte_alloc_one(vma->vm_mm); 744 if (unlikely(!pgtable)) 745 return VM_FAULT_OOM; 746 zero_page = mm_get_huge_zero_page(vma->vm_mm); 747 if (unlikely(!zero_page)) { 748 pte_free(vma->vm_mm, pgtable); 749 count_vm_event(THP_FAULT_FALLBACK); 750 return VM_FAULT_FALLBACK; 751 } 752 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 753 ret = 0; 754 if (pmd_none(*vmf->pmd)) { 755 ret = check_stable_address_space(vma->vm_mm); 756 if (ret) { 757 spin_unlock(vmf->ptl); 758 pte_free(vma->vm_mm, pgtable); 759 } else if (userfaultfd_missing(vma)) { 760 spin_unlock(vmf->ptl); 761 pte_free(vma->vm_mm, pgtable); 762 ret = handle_userfault(vmf, VM_UFFD_MISSING); 763 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 764 } else { 765 set_huge_zero_page(pgtable, vma->vm_mm, vma, 766 haddr, vmf->pmd, zero_page); 767 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 768 spin_unlock(vmf->ptl); 769 } 770 } else { 771 spin_unlock(vmf->ptl); 772 pte_free(vma->vm_mm, pgtable); 773 } 774 return ret; 775 } 776 gfp = vma_thp_gfp_mask(vma); 777 page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER); 778 if (unlikely(!page)) { 779 count_vm_event(THP_FAULT_FALLBACK); 780 return VM_FAULT_FALLBACK; 781 } 782 prep_transhuge_page(page); 783 return __do_huge_pmd_anonymous_page(vmf, page, gfp); 784 } 785 786 static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 787 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write, 788 pgtable_t pgtable) 789 { 790 struct mm_struct *mm = vma->vm_mm; 791 pmd_t entry; 792 spinlock_t *ptl; 793 794 ptl = pmd_lock(mm, pmd); 795 if (!pmd_none(*pmd)) { 796 if (write) { 797 if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) { 798 WARN_ON_ONCE(!is_huge_zero_pmd(*pmd)); 799 goto out_unlock; 800 } 801 entry = pmd_mkyoung(*pmd); 802 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 803 if (pmdp_set_access_flags(vma, addr, pmd, entry, 1)) 804 update_mmu_cache_pmd(vma, addr, pmd); 805 } 806 807 goto out_unlock; 808 } 809 810 entry = pmd_mkhuge(pfn_t_pmd(pfn, prot)); 811 if (pfn_t_devmap(pfn)) 812 entry = pmd_mkdevmap(entry); 813 if (write) { 814 entry = pmd_mkyoung(pmd_mkdirty(entry)); 815 entry = maybe_pmd_mkwrite(entry, vma); 816 } 817 818 if (pgtable) { 819 pgtable_trans_huge_deposit(mm, pmd, pgtable); 820 mm_inc_nr_ptes(mm); 821 pgtable = NULL; 822 } 823 824 set_pmd_at(mm, addr, pmd, entry); 825 update_mmu_cache_pmd(vma, addr, pmd); 826 827 out_unlock: 828 spin_unlock(ptl); 829 if (pgtable) 830 pte_free(mm, pgtable); 831 } 832 833 /** 834 * vmf_insert_pfn_pmd_prot - insert a pmd size pfn 835 * @vmf: Structure describing the fault 836 * @pfn: pfn to insert 837 * @pgprot: page protection to use 838 * @write: whether it's a write fault 839 * 840 * Insert a pmd size pfn. See vmf_insert_pfn() for additional info and 841 * also consult the vmf_insert_mixed_prot() documentation when 842 * @pgprot != @vmf->vma->vm_page_prot. 843 * 844 * Return: vm_fault_t value. 845 */ 846 vm_fault_t vmf_insert_pfn_pmd_prot(struct vm_fault *vmf, pfn_t pfn, 847 pgprot_t pgprot, bool write) 848 { 849 unsigned long addr = vmf->address & PMD_MASK; 850 struct vm_area_struct *vma = vmf->vma; 851 pgtable_t pgtable = NULL; 852 853 /* 854 * If we had pmd_special, we could avoid all these restrictions, 855 * but we need to be consistent with PTEs and architectures that 856 * can't support a 'special' bit. 857 */ 858 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && 859 !pfn_t_devmap(pfn)); 860 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 861 (VM_PFNMAP|VM_MIXEDMAP)); 862 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 863 864 if (addr < vma->vm_start || addr >= vma->vm_end) 865 return VM_FAULT_SIGBUS; 866 867 if (arch_needs_pgtable_deposit()) { 868 pgtable = pte_alloc_one(vma->vm_mm); 869 if (!pgtable) 870 return VM_FAULT_OOM; 871 } 872 873 track_pfn_insert(vma, &pgprot, pfn); 874 875 insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable); 876 return VM_FAULT_NOPAGE; 877 } 878 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd_prot); 879 880 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 881 static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma) 882 { 883 if (likely(vma->vm_flags & VM_WRITE)) 884 pud = pud_mkwrite(pud); 885 return pud; 886 } 887 888 static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr, 889 pud_t *pud, pfn_t pfn, pgprot_t prot, bool write) 890 { 891 struct mm_struct *mm = vma->vm_mm; 892 pud_t entry; 893 spinlock_t *ptl; 894 895 ptl = pud_lock(mm, pud); 896 if (!pud_none(*pud)) { 897 if (write) { 898 if (pud_pfn(*pud) != pfn_t_to_pfn(pfn)) { 899 WARN_ON_ONCE(!is_huge_zero_pud(*pud)); 900 goto out_unlock; 901 } 902 entry = pud_mkyoung(*pud); 903 entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma); 904 if (pudp_set_access_flags(vma, addr, pud, entry, 1)) 905 update_mmu_cache_pud(vma, addr, pud); 906 } 907 goto out_unlock; 908 } 909 910 entry = pud_mkhuge(pfn_t_pud(pfn, prot)); 911 if (pfn_t_devmap(pfn)) 912 entry = pud_mkdevmap(entry); 913 if (write) { 914 entry = pud_mkyoung(pud_mkdirty(entry)); 915 entry = maybe_pud_mkwrite(entry, vma); 916 } 917 set_pud_at(mm, addr, pud, entry); 918 update_mmu_cache_pud(vma, addr, pud); 919 920 out_unlock: 921 spin_unlock(ptl); 922 } 923 924 /** 925 * vmf_insert_pfn_pud_prot - insert a pud size pfn 926 * @vmf: Structure describing the fault 927 * @pfn: pfn to insert 928 * @pgprot: page protection to use 929 * @write: whether it's a write fault 930 * 931 * Insert a pud size pfn. See vmf_insert_pfn() for additional info and 932 * also consult the vmf_insert_mixed_prot() documentation when 933 * @pgprot != @vmf->vma->vm_page_prot. 934 * 935 * Return: vm_fault_t value. 936 */ 937 vm_fault_t vmf_insert_pfn_pud_prot(struct vm_fault *vmf, pfn_t pfn, 938 pgprot_t pgprot, bool write) 939 { 940 unsigned long addr = vmf->address & PUD_MASK; 941 struct vm_area_struct *vma = vmf->vma; 942 943 /* 944 * If we had pud_special, we could avoid all these restrictions, 945 * but we need to be consistent with PTEs and architectures that 946 * can't support a 'special' bit. 947 */ 948 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && 949 !pfn_t_devmap(pfn)); 950 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 951 (VM_PFNMAP|VM_MIXEDMAP)); 952 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 953 954 if (addr < vma->vm_start || addr >= vma->vm_end) 955 return VM_FAULT_SIGBUS; 956 957 track_pfn_insert(vma, &pgprot, pfn); 958 959 insert_pfn_pud(vma, addr, vmf->pud, pfn, pgprot, write); 960 return VM_FAULT_NOPAGE; 961 } 962 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud_prot); 963 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 964 965 static void touch_pmd(struct vm_area_struct *vma, unsigned long addr, 966 pmd_t *pmd, int flags) 967 { 968 pmd_t _pmd; 969 970 _pmd = pmd_mkyoung(*pmd); 971 if (flags & FOLL_WRITE) 972 _pmd = pmd_mkdirty(_pmd); 973 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, 974 pmd, _pmd, flags & FOLL_WRITE)) 975 update_mmu_cache_pmd(vma, addr, pmd); 976 } 977 978 struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, 979 pmd_t *pmd, int flags, struct dev_pagemap **pgmap) 980 { 981 unsigned long pfn = pmd_pfn(*pmd); 982 struct mm_struct *mm = vma->vm_mm; 983 struct page *page; 984 985 assert_spin_locked(pmd_lockptr(mm, pmd)); 986 987 /* 988 * When we COW a devmap PMD entry, we split it into PTEs, so we should 989 * not be in this function with `flags & FOLL_COW` set. 990 */ 991 WARN_ONCE(flags & FOLL_COW, "mm: In follow_devmap_pmd with FOLL_COW set"); 992 993 /* FOLL_GET and FOLL_PIN are mutually exclusive. */ 994 if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) == 995 (FOLL_PIN | FOLL_GET))) 996 return NULL; 997 998 if (flags & FOLL_WRITE && !pmd_write(*pmd)) 999 return NULL; 1000 1001 if (pmd_present(*pmd) && pmd_devmap(*pmd)) 1002 /* pass */; 1003 else 1004 return NULL; 1005 1006 if (flags & FOLL_TOUCH) 1007 touch_pmd(vma, addr, pmd, flags); 1008 1009 /* 1010 * device mapped pages can only be returned if the 1011 * caller will manage the page reference count. 1012 */ 1013 if (!(flags & (FOLL_GET | FOLL_PIN))) 1014 return ERR_PTR(-EEXIST); 1015 1016 pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT; 1017 *pgmap = get_dev_pagemap(pfn, *pgmap); 1018 if (!*pgmap) 1019 return ERR_PTR(-EFAULT); 1020 page = pfn_to_page(pfn); 1021 if (!try_grab_page(page, flags)) 1022 page = ERR_PTR(-ENOMEM); 1023 1024 return page; 1025 } 1026 1027 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1028 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 1029 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) 1030 { 1031 spinlock_t *dst_ptl, *src_ptl; 1032 struct page *src_page; 1033 pmd_t pmd; 1034 pgtable_t pgtable = NULL; 1035 int ret = -ENOMEM; 1036 1037 /* Skip if can be re-fill on fault */ 1038 if (!vma_is_anonymous(dst_vma)) 1039 return 0; 1040 1041 pgtable = pte_alloc_one(dst_mm); 1042 if (unlikely(!pgtable)) 1043 goto out; 1044 1045 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1046 src_ptl = pmd_lockptr(src_mm, src_pmd); 1047 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1048 1049 ret = -EAGAIN; 1050 pmd = *src_pmd; 1051 1052 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 1053 if (unlikely(is_swap_pmd(pmd))) { 1054 swp_entry_t entry = pmd_to_swp_entry(pmd); 1055 1056 VM_BUG_ON(!is_pmd_migration_entry(pmd)); 1057 if (is_writable_migration_entry(entry)) { 1058 entry = make_readable_migration_entry( 1059 swp_offset(entry)); 1060 pmd = swp_entry_to_pmd(entry); 1061 if (pmd_swp_soft_dirty(*src_pmd)) 1062 pmd = pmd_swp_mksoft_dirty(pmd); 1063 if (pmd_swp_uffd_wp(*src_pmd)) 1064 pmd = pmd_swp_mkuffd_wp(pmd); 1065 set_pmd_at(src_mm, addr, src_pmd, pmd); 1066 } 1067 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1068 mm_inc_nr_ptes(dst_mm); 1069 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1070 if (!userfaultfd_wp(dst_vma)) 1071 pmd = pmd_swp_clear_uffd_wp(pmd); 1072 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1073 ret = 0; 1074 goto out_unlock; 1075 } 1076 #endif 1077 1078 if (unlikely(!pmd_trans_huge(pmd))) { 1079 pte_free(dst_mm, pgtable); 1080 goto out_unlock; 1081 } 1082 /* 1083 * When page table lock is held, the huge zero pmd should not be 1084 * under splitting since we don't split the page itself, only pmd to 1085 * a page table. 1086 */ 1087 if (is_huge_zero_pmd(pmd)) { 1088 /* 1089 * get_huge_zero_page() will never allocate a new page here, 1090 * since we already have a zero page to copy. It just takes a 1091 * reference. 1092 */ 1093 mm_get_huge_zero_page(dst_mm); 1094 goto out_zero_page; 1095 } 1096 1097 src_page = pmd_page(pmd); 1098 VM_BUG_ON_PAGE(!PageHead(src_page), src_page); 1099 1100 /* 1101 * If this page is a potentially pinned page, split and retry the fault 1102 * with smaller page size. Normally this should not happen because the 1103 * userspace should use MADV_DONTFORK upon pinned regions. This is a 1104 * best effort that the pinned pages won't be replaced by another 1105 * random page during the coming copy-on-write. 1106 */ 1107 if (unlikely(page_needs_cow_for_dma(src_vma, src_page))) { 1108 pte_free(dst_mm, pgtable); 1109 spin_unlock(src_ptl); 1110 spin_unlock(dst_ptl); 1111 __split_huge_pmd(src_vma, src_pmd, addr, false, NULL); 1112 return -EAGAIN; 1113 } 1114 1115 get_page(src_page); 1116 page_dup_rmap(src_page, true); 1117 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1118 out_zero_page: 1119 mm_inc_nr_ptes(dst_mm); 1120 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1121 pmdp_set_wrprotect(src_mm, addr, src_pmd); 1122 if (!userfaultfd_wp(dst_vma)) 1123 pmd = pmd_clear_uffd_wp(pmd); 1124 pmd = pmd_mkold(pmd_wrprotect(pmd)); 1125 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1126 1127 ret = 0; 1128 out_unlock: 1129 spin_unlock(src_ptl); 1130 spin_unlock(dst_ptl); 1131 out: 1132 return ret; 1133 } 1134 1135 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1136 static void touch_pud(struct vm_area_struct *vma, unsigned long addr, 1137 pud_t *pud, int flags) 1138 { 1139 pud_t _pud; 1140 1141 _pud = pud_mkyoung(*pud); 1142 if (flags & FOLL_WRITE) 1143 _pud = pud_mkdirty(_pud); 1144 if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK, 1145 pud, _pud, flags & FOLL_WRITE)) 1146 update_mmu_cache_pud(vma, addr, pud); 1147 } 1148 1149 struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr, 1150 pud_t *pud, int flags, struct dev_pagemap **pgmap) 1151 { 1152 unsigned long pfn = pud_pfn(*pud); 1153 struct mm_struct *mm = vma->vm_mm; 1154 struct page *page; 1155 1156 assert_spin_locked(pud_lockptr(mm, pud)); 1157 1158 if (flags & FOLL_WRITE && !pud_write(*pud)) 1159 return NULL; 1160 1161 /* FOLL_GET and FOLL_PIN are mutually exclusive. */ 1162 if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) == 1163 (FOLL_PIN | FOLL_GET))) 1164 return NULL; 1165 1166 if (pud_present(*pud) && pud_devmap(*pud)) 1167 /* pass */; 1168 else 1169 return NULL; 1170 1171 if (flags & FOLL_TOUCH) 1172 touch_pud(vma, addr, pud, flags); 1173 1174 /* 1175 * device mapped pages can only be returned if the 1176 * caller will manage the page reference count. 1177 * 1178 * At least one of FOLL_GET | FOLL_PIN must be set, so assert that here: 1179 */ 1180 if (!(flags & (FOLL_GET | FOLL_PIN))) 1181 return ERR_PTR(-EEXIST); 1182 1183 pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT; 1184 *pgmap = get_dev_pagemap(pfn, *pgmap); 1185 if (!*pgmap) 1186 return ERR_PTR(-EFAULT); 1187 page = pfn_to_page(pfn); 1188 if (!try_grab_page(page, flags)) 1189 page = ERR_PTR(-ENOMEM); 1190 1191 return page; 1192 } 1193 1194 int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1195 pud_t *dst_pud, pud_t *src_pud, unsigned long addr, 1196 struct vm_area_struct *vma) 1197 { 1198 spinlock_t *dst_ptl, *src_ptl; 1199 pud_t pud; 1200 int ret; 1201 1202 dst_ptl = pud_lock(dst_mm, dst_pud); 1203 src_ptl = pud_lockptr(src_mm, src_pud); 1204 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1205 1206 ret = -EAGAIN; 1207 pud = *src_pud; 1208 if (unlikely(!pud_trans_huge(pud) && !pud_devmap(pud))) 1209 goto out_unlock; 1210 1211 /* 1212 * When page table lock is held, the huge zero pud should not be 1213 * under splitting since we don't split the page itself, only pud to 1214 * a page table. 1215 */ 1216 if (is_huge_zero_pud(pud)) { 1217 /* No huge zero pud yet */ 1218 } 1219 1220 /* Please refer to comments in copy_huge_pmd() */ 1221 if (unlikely(page_needs_cow_for_dma(vma, pud_page(pud)))) { 1222 spin_unlock(src_ptl); 1223 spin_unlock(dst_ptl); 1224 __split_huge_pud(vma, src_pud, addr); 1225 return -EAGAIN; 1226 } 1227 1228 pudp_set_wrprotect(src_mm, addr, src_pud); 1229 pud = pud_mkold(pud_wrprotect(pud)); 1230 set_pud_at(dst_mm, addr, dst_pud, pud); 1231 1232 ret = 0; 1233 out_unlock: 1234 spin_unlock(src_ptl); 1235 spin_unlock(dst_ptl); 1236 return ret; 1237 } 1238 1239 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud) 1240 { 1241 pud_t entry; 1242 unsigned long haddr; 1243 bool write = vmf->flags & FAULT_FLAG_WRITE; 1244 1245 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); 1246 if (unlikely(!pud_same(*vmf->pud, orig_pud))) 1247 goto unlock; 1248 1249 entry = pud_mkyoung(orig_pud); 1250 if (write) 1251 entry = pud_mkdirty(entry); 1252 haddr = vmf->address & HPAGE_PUD_MASK; 1253 if (pudp_set_access_flags(vmf->vma, haddr, vmf->pud, entry, write)) 1254 update_mmu_cache_pud(vmf->vma, vmf->address, vmf->pud); 1255 1256 unlock: 1257 spin_unlock(vmf->ptl); 1258 } 1259 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1260 1261 void huge_pmd_set_accessed(struct vm_fault *vmf) 1262 { 1263 pmd_t entry; 1264 unsigned long haddr; 1265 bool write = vmf->flags & FAULT_FLAG_WRITE; 1266 pmd_t orig_pmd = vmf->orig_pmd; 1267 1268 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); 1269 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) 1270 goto unlock; 1271 1272 entry = pmd_mkyoung(orig_pmd); 1273 if (write) 1274 entry = pmd_mkdirty(entry); 1275 haddr = vmf->address & HPAGE_PMD_MASK; 1276 if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write)) 1277 update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd); 1278 1279 unlock: 1280 spin_unlock(vmf->ptl); 1281 } 1282 1283 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) 1284 { 1285 struct vm_area_struct *vma = vmf->vma; 1286 struct page *page; 1287 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1288 pmd_t orig_pmd = vmf->orig_pmd; 1289 1290 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); 1291 VM_BUG_ON_VMA(!vma->anon_vma, vma); 1292 1293 if (is_huge_zero_pmd(orig_pmd)) 1294 goto fallback; 1295 1296 spin_lock(vmf->ptl); 1297 1298 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1299 spin_unlock(vmf->ptl); 1300 return 0; 1301 } 1302 1303 page = pmd_page(orig_pmd); 1304 VM_BUG_ON_PAGE(!PageHead(page), page); 1305 1306 /* Lock page for reuse_swap_page() */ 1307 if (!trylock_page(page)) { 1308 get_page(page); 1309 spin_unlock(vmf->ptl); 1310 lock_page(page); 1311 spin_lock(vmf->ptl); 1312 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1313 spin_unlock(vmf->ptl); 1314 unlock_page(page); 1315 put_page(page); 1316 return 0; 1317 } 1318 put_page(page); 1319 } 1320 1321 /* 1322 * We can only reuse the page if nobody else maps the huge page or it's 1323 * part. 1324 */ 1325 if (reuse_swap_page(page, NULL)) { 1326 pmd_t entry; 1327 entry = pmd_mkyoung(orig_pmd); 1328 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1329 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) 1330 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1331 unlock_page(page); 1332 spin_unlock(vmf->ptl); 1333 return VM_FAULT_WRITE; 1334 } 1335 1336 unlock_page(page); 1337 spin_unlock(vmf->ptl); 1338 fallback: 1339 __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL); 1340 return VM_FAULT_FALLBACK; 1341 } 1342 1343 /* 1344 * FOLL_FORCE can write to even unwritable pmd's, but only 1345 * after we've gone through a COW cycle and they are dirty. 1346 */ 1347 static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags) 1348 { 1349 return pmd_write(pmd) || 1350 ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd)); 1351 } 1352 1353 struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, 1354 unsigned long addr, 1355 pmd_t *pmd, 1356 unsigned int flags) 1357 { 1358 struct mm_struct *mm = vma->vm_mm; 1359 struct page *page = NULL; 1360 1361 assert_spin_locked(pmd_lockptr(mm, pmd)); 1362 1363 if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags)) 1364 goto out; 1365 1366 /* Avoid dumping huge zero page */ 1367 if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd)) 1368 return ERR_PTR(-EFAULT); 1369 1370 /* Full NUMA hinting faults to serialise migration in fault paths */ 1371 if ((flags & FOLL_NUMA) && pmd_protnone(*pmd)) 1372 goto out; 1373 1374 page = pmd_page(*pmd); 1375 VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page); 1376 1377 if (!try_grab_page(page, flags)) 1378 return ERR_PTR(-ENOMEM); 1379 1380 if (flags & FOLL_TOUCH) 1381 touch_pmd(vma, addr, pmd, flags); 1382 1383 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { 1384 /* 1385 * We don't mlock() pte-mapped THPs. This way we can avoid 1386 * leaking mlocked pages into non-VM_LOCKED VMAs. 1387 * 1388 * For anon THP: 1389 * 1390 * In most cases the pmd is the only mapping of the page as we 1391 * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for 1392 * writable private mappings in populate_vma_page_range(). 1393 * 1394 * The only scenario when we have the page shared here is if we 1395 * mlocking read-only mapping shared over fork(). We skip 1396 * mlocking such pages. 1397 * 1398 * For file THP: 1399 * 1400 * We can expect PageDoubleMap() to be stable under page lock: 1401 * for file pages we set it in page_add_file_rmap(), which 1402 * requires page to be locked. 1403 */ 1404 1405 if (PageAnon(page) && compound_mapcount(page) != 1) 1406 goto skip_mlock; 1407 if (PageDoubleMap(page) || !page->mapping) 1408 goto skip_mlock; 1409 if (!trylock_page(page)) 1410 goto skip_mlock; 1411 if (page->mapping && !PageDoubleMap(page)) 1412 mlock_vma_page(page); 1413 unlock_page(page); 1414 } 1415 skip_mlock: 1416 page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT; 1417 VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page); 1418 1419 out: 1420 return page; 1421 } 1422 1423 /* NUMA hinting page fault entry point for trans huge pmds */ 1424 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) 1425 { 1426 struct vm_area_struct *vma = vmf->vma; 1427 pmd_t oldpmd = vmf->orig_pmd; 1428 pmd_t pmd; 1429 struct page *page; 1430 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1431 int page_nid = NUMA_NO_NODE; 1432 int target_nid, last_cpupid = -1; 1433 bool migrated = false; 1434 bool was_writable = pmd_savedwrite(oldpmd); 1435 int flags = 0; 1436 1437 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1438 if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) { 1439 spin_unlock(vmf->ptl); 1440 goto out; 1441 } 1442 1443 pmd = pmd_modify(oldpmd, vma->vm_page_prot); 1444 page = vm_normal_page_pmd(vma, haddr, pmd); 1445 if (!page) 1446 goto out_map; 1447 1448 /* See similar comment in do_numa_page for explanation */ 1449 if (!was_writable) 1450 flags |= TNF_NO_GROUP; 1451 1452 page_nid = page_to_nid(page); 1453 last_cpupid = page_cpupid_last(page); 1454 target_nid = numa_migrate_prep(page, vma, haddr, page_nid, 1455 &flags); 1456 1457 if (target_nid == NUMA_NO_NODE) { 1458 put_page(page); 1459 goto out_map; 1460 } 1461 1462 spin_unlock(vmf->ptl); 1463 1464 migrated = migrate_misplaced_page(page, vma, target_nid); 1465 if (migrated) { 1466 flags |= TNF_MIGRATED; 1467 page_nid = target_nid; 1468 } else { 1469 flags |= TNF_MIGRATE_FAIL; 1470 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1471 if (unlikely(!pmd_same(oldpmd, *vmf->pmd))) { 1472 spin_unlock(vmf->ptl); 1473 goto out; 1474 } 1475 goto out_map; 1476 } 1477 1478 out: 1479 if (page_nid != NUMA_NO_NODE) 1480 task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, 1481 flags); 1482 1483 return 0; 1484 1485 out_map: 1486 /* Restore the PMD */ 1487 pmd = pmd_modify(oldpmd, vma->vm_page_prot); 1488 pmd = pmd_mkyoung(pmd); 1489 if (was_writable) 1490 pmd = pmd_mkwrite(pmd); 1491 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); 1492 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1493 spin_unlock(vmf->ptl); 1494 goto out; 1495 } 1496 1497 /* 1498 * Return true if we do MADV_FREE successfully on entire pmd page. 1499 * Otherwise, return false. 1500 */ 1501 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1502 pmd_t *pmd, unsigned long addr, unsigned long next) 1503 { 1504 spinlock_t *ptl; 1505 pmd_t orig_pmd; 1506 struct page *page; 1507 struct mm_struct *mm = tlb->mm; 1508 bool ret = false; 1509 1510 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 1511 1512 ptl = pmd_trans_huge_lock(pmd, vma); 1513 if (!ptl) 1514 goto out_unlocked; 1515 1516 orig_pmd = *pmd; 1517 if (is_huge_zero_pmd(orig_pmd)) 1518 goto out; 1519 1520 if (unlikely(!pmd_present(orig_pmd))) { 1521 VM_BUG_ON(thp_migration_supported() && 1522 !is_pmd_migration_entry(orig_pmd)); 1523 goto out; 1524 } 1525 1526 page = pmd_page(orig_pmd); 1527 /* 1528 * If other processes are mapping this page, we couldn't discard 1529 * the page unless they all do MADV_FREE so let's skip the page. 1530 */ 1531 if (total_mapcount(page) != 1) 1532 goto out; 1533 1534 if (!trylock_page(page)) 1535 goto out; 1536 1537 /* 1538 * If user want to discard part-pages of THP, split it so MADV_FREE 1539 * will deactivate only them. 1540 */ 1541 if (next - addr != HPAGE_PMD_SIZE) { 1542 get_page(page); 1543 spin_unlock(ptl); 1544 split_huge_page(page); 1545 unlock_page(page); 1546 put_page(page); 1547 goto out_unlocked; 1548 } 1549 1550 if (PageDirty(page)) 1551 ClearPageDirty(page); 1552 unlock_page(page); 1553 1554 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) { 1555 pmdp_invalidate(vma, addr, pmd); 1556 orig_pmd = pmd_mkold(orig_pmd); 1557 orig_pmd = pmd_mkclean(orig_pmd); 1558 1559 set_pmd_at(mm, addr, pmd, orig_pmd); 1560 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 1561 } 1562 1563 mark_page_lazyfree(page); 1564 ret = true; 1565 out: 1566 spin_unlock(ptl); 1567 out_unlocked: 1568 return ret; 1569 } 1570 1571 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd) 1572 { 1573 pgtable_t pgtable; 1574 1575 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 1576 pte_free(mm, pgtable); 1577 mm_dec_nr_ptes(mm); 1578 } 1579 1580 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1581 pmd_t *pmd, unsigned long addr) 1582 { 1583 pmd_t orig_pmd; 1584 spinlock_t *ptl; 1585 1586 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 1587 1588 ptl = __pmd_trans_huge_lock(pmd, vma); 1589 if (!ptl) 1590 return 0; 1591 /* 1592 * For architectures like ppc64 we look at deposited pgtable 1593 * when calling pmdp_huge_get_and_clear. So do the 1594 * pgtable_trans_huge_withdraw after finishing pmdp related 1595 * operations. 1596 */ 1597 orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd, 1598 tlb->fullmm); 1599 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 1600 if (vma_is_special_huge(vma)) { 1601 if (arch_needs_pgtable_deposit()) 1602 zap_deposited_table(tlb->mm, pmd); 1603 spin_unlock(ptl); 1604 } else if (is_huge_zero_pmd(orig_pmd)) { 1605 zap_deposited_table(tlb->mm, pmd); 1606 spin_unlock(ptl); 1607 } else { 1608 struct page *page = NULL; 1609 int flush_needed = 1; 1610 1611 if (pmd_present(orig_pmd)) { 1612 page = pmd_page(orig_pmd); 1613 page_remove_rmap(page, true); 1614 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page); 1615 VM_BUG_ON_PAGE(!PageHead(page), page); 1616 } else if (thp_migration_supported()) { 1617 swp_entry_t entry; 1618 1619 VM_BUG_ON(!is_pmd_migration_entry(orig_pmd)); 1620 entry = pmd_to_swp_entry(orig_pmd); 1621 page = pfn_swap_entry_to_page(entry); 1622 flush_needed = 0; 1623 } else 1624 WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!"); 1625 1626 if (PageAnon(page)) { 1627 zap_deposited_table(tlb->mm, pmd); 1628 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); 1629 } else { 1630 if (arch_needs_pgtable_deposit()) 1631 zap_deposited_table(tlb->mm, pmd); 1632 add_mm_counter(tlb->mm, mm_counter_file(page), -HPAGE_PMD_NR); 1633 } 1634 1635 spin_unlock(ptl); 1636 if (flush_needed) 1637 tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE); 1638 } 1639 return 1; 1640 } 1641 1642 #ifndef pmd_move_must_withdraw 1643 static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl, 1644 spinlock_t *old_pmd_ptl, 1645 struct vm_area_struct *vma) 1646 { 1647 /* 1648 * With split pmd lock we also need to move preallocated 1649 * PTE page table if new_pmd is on different PMD page table. 1650 * 1651 * We also don't deposit and withdraw tables for file pages. 1652 */ 1653 return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma); 1654 } 1655 #endif 1656 1657 static pmd_t move_soft_dirty_pmd(pmd_t pmd) 1658 { 1659 #ifdef CONFIG_MEM_SOFT_DIRTY 1660 if (unlikely(is_pmd_migration_entry(pmd))) 1661 pmd = pmd_swp_mksoft_dirty(pmd); 1662 else if (pmd_present(pmd)) 1663 pmd = pmd_mksoft_dirty(pmd); 1664 #endif 1665 return pmd; 1666 } 1667 1668 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, 1669 unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd) 1670 { 1671 spinlock_t *old_ptl, *new_ptl; 1672 pmd_t pmd; 1673 struct mm_struct *mm = vma->vm_mm; 1674 bool force_flush = false; 1675 1676 /* 1677 * The destination pmd shouldn't be established, free_pgtables() 1678 * should have release it. 1679 */ 1680 if (WARN_ON(!pmd_none(*new_pmd))) { 1681 VM_BUG_ON(pmd_trans_huge(*new_pmd)); 1682 return false; 1683 } 1684 1685 /* 1686 * We don't have to worry about the ordering of src and dst 1687 * ptlocks because exclusive mmap_lock prevents deadlock. 1688 */ 1689 old_ptl = __pmd_trans_huge_lock(old_pmd, vma); 1690 if (old_ptl) { 1691 new_ptl = pmd_lockptr(mm, new_pmd); 1692 if (new_ptl != old_ptl) 1693 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 1694 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); 1695 if (pmd_present(pmd)) 1696 force_flush = true; 1697 VM_BUG_ON(!pmd_none(*new_pmd)); 1698 1699 if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) { 1700 pgtable_t pgtable; 1701 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); 1702 pgtable_trans_huge_deposit(mm, new_pmd, pgtable); 1703 } 1704 pmd = move_soft_dirty_pmd(pmd); 1705 set_pmd_at(mm, new_addr, new_pmd, pmd); 1706 if (force_flush) 1707 flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE); 1708 if (new_ptl != old_ptl) 1709 spin_unlock(new_ptl); 1710 spin_unlock(old_ptl); 1711 return true; 1712 } 1713 return false; 1714 } 1715 1716 /* 1717 * Returns 1718 * - 0 if PMD could not be locked 1719 * - 1 if PMD was locked but protections unchanged and TLB flush unnecessary 1720 * or if prot_numa but THP migration is not supported 1721 * - HPAGE_PMD_NR if protections changed and TLB flush necessary 1722 */ 1723 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 1724 unsigned long addr, pgprot_t newprot, unsigned long cp_flags) 1725 { 1726 struct mm_struct *mm = vma->vm_mm; 1727 spinlock_t *ptl; 1728 pmd_t entry; 1729 bool preserve_write; 1730 int ret; 1731 bool prot_numa = cp_flags & MM_CP_PROT_NUMA; 1732 bool uffd_wp = cp_flags & MM_CP_UFFD_WP; 1733 bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; 1734 1735 if (prot_numa && !thp_migration_supported()) 1736 return 1; 1737 1738 ptl = __pmd_trans_huge_lock(pmd, vma); 1739 if (!ptl) 1740 return 0; 1741 1742 preserve_write = prot_numa && pmd_write(*pmd); 1743 ret = 1; 1744 1745 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 1746 if (is_swap_pmd(*pmd)) { 1747 swp_entry_t entry = pmd_to_swp_entry(*pmd); 1748 1749 VM_BUG_ON(!is_pmd_migration_entry(*pmd)); 1750 if (is_writable_migration_entry(entry)) { 1751 pmd_t newpmd; 1752 /* 1753 * A protection check is difficult so 1754 * just be safe and disable write 1755 */ 1756 entry = make_readable_migration_entry( 1757 swp_offset(entry)); 1758 newpmd = swp_entry_to_pmd(entry); 1759 if (pmd_swp_soft_dirty(*pmd)) 1760 newpmd = pmd_swp_mksoft_dirty(newpmd); 1761 if (pmd_swp_uffd_wp(*pmd)) 1762 newpmd = pmd_swp_mkuffd_wp(newpmd); 1763 set_pmd_at(mm, addr, pmd, newpmd); 1764 } 1765 goto unlock; 1766 } 1767 #endif 1768 1769 /* 1770 * Avoid trapping faults against the zero page. The read-only 1771 * data is likely to be read-cached on the local CPU and 1772 * local/remote hits to the zero page are not interesting. 1773 */ 1774 if (prot_numa && is_huge_zero_pmd(*pmd)) 1775 goto unlock; 1776 1777 if (prot_numa && pmd_protnone(*pmd)) 1778 goto unlock; 1779 1780 /* 1781 * In case prot_numa, we are under mmap_read_lock(mm). It's critical 1782 * to not clear pmd intermittently to avoid race with MADV_DONTNEED 1783 * which is also under mmap_read_lock(mm): 1784 * 1785 * CPU0: CPU1: 1786 * change_huge_pmd(prot_numa=1) 1787 * pmdp_huge_get_and_clear_notify() 1788 * madvise_dontneed() 1789 * zap_pmd_range() 1790 * pmd_trans_huge(*pmd) == 0 (without ptl) 1791 * // skip the pmd 1792 * set_pmd_at(); 1793 * // pmd is re-established 1794 * 1795 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it 1796 * which may break userspace. 1797 * 1798 * pmdp_invalidate() is required to make sure we don't miss 1799 * dirty/young flags set by hardware. 1800 */ 1801 entry = pmdp_invalidate(vma, addr, pmd); 1802 1803 entry = pmd_modify(entry, newprot); 1804 if (preserve_write) 1805 entry = pmd_mk_savedwrite(entry); 1806 if (uffd_wp) { 1807 entry = pmd_wrprotect(entry); 1808 entry = pmd_mkuffd_wp(entry); 1809 } else if (uffd_wp_resolve) { 1810 /* 1811 * Leave the write bit to be handled by PF interrupt 1812 * handler, then things like COW could be properly 1813 * handled. 1814 */ 1815 entry = pmd_clear_uffd_wp(entry); 1816 } 1817 ret = HPAGE_PMD_NR; 1818 set_pmd_at(mm, addr, pmd, entry); 1819 BUG_ON(vma_is_anonymous(vma) && !preserve_write && pmd_write(entry)); 1820 unlock: 1821 spin_unlock(ptl); 1822 return ret; 1823 } 1824 1825 /* 1826 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise. 1827 * 1828 * Note that if it returns page table lock pointer, this routine returns without 1829 * unlocking page table lock. So callers must unlock it. 1830 */ 1831 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) 1832 { 1833 spinlock_t *ptl; 1834 ptl = pmd_lock(vma->vm_mm, pmd); 1835 if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || 1836 pmd_devmap(*pmd))) 1837 return ptl; 1838 spin_unlock(ptl); 1839 return NULL; 1840 } 1841 1842 /* 1843 * Returns true if a given pud maps a thp, false otherwise. 1844 * 1845 * Note that if it returns true, this routine returns without unlocking page 1846 * table lock. So callers must unlock it. 1847 */ 1848 spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma) 1849 { 1850 spinlock_t *ptl; 1851 1852 ptl = pud_lock(vma->vm_mm, pud); 1853 if (likely(pud_trans_huge(*pud) || pud_devmap(*pud))) 1854 return ptl; 1855 spin_unlock(ptl); 1856 return NULL; 1857 } 1858 1859 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1860 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 1861 pud_t *pud, unsigned long addr) 1862 { 1863 spinlock_t *ptl; 1864 1865 ptl = __pud_trans_huge_lock(pud, vma); 1866 if (!ptl) 1867 return 0; 1868 /* 1869 * For architectures like ppc64 we look at deposited pgtable 1870 * when calling pudp_huge_get_and_clear. So do the 1871 * pgtable_trans_huge_withdraw after finishing pudp related 1872 * operations. 1873 */ 1874 pudp_huge_get_and_clear_full(tlb->mm, addr, pud, tlb->fullmm); 1875 tlb_remove_pud_tlb_entry(tlb, pud, addr); 1876 if (vma_is_special_huge(vma)) { 1877 spin_unlock(ptl); 1878 /* No zero page support yet */ 1879 } else { 1880 /* No support for anonymous PUD pages yet */ 1881 BUG(); 1882 } 1883 return 1; 1884 } 1885 1886 static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud, 1887 unsigned long haddr) 1888 { 1889 VM_BUG_ON(haddr & ~HPAGE_PUD_MASK); 1890 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 1891 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma); 1892 VM_BUG_ON(!pud_trans_huge(*pud) && !pud_devmap(*pud)); 1893 1894 count_vm_event(THP_SPLIT_PUD); 1895 1896 pudp_huge_clear_flush_notify(vma, haddr, pud); 1897 } 1898 1899 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 1900 unsigned long address) 1901 { 1902 spinlock_t *ptl; 1903 struct mmu_notifier_range range; 1904 1905 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, 1906 address & HPAGE_PUD_MASK, 1907 (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE); 1908 mmu_notifier_invalidate_range_start(&range); 1909 ptl = pud_lock(vma->vm_mm, pud); 1910 if (unlikely(!pud_trans_huge(*pud) && !pud_devmap(*pud))) 1911 goto out; 1912 __split_huge_pud_locked(vma, pud, range.start); 1913 1914 out: 1915 spin_unlock(ptl); 1916 /* 1917 * No need to double call mmu_notifier->invalidate_range() callback as 1918 * the above pudp_huge_clear_flush_notify() did already call it. 1919 */ 1920 mmu_notifier_invalidate_range_only_end(&range); 1921 } 1922 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1923 1924 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 1925 unsigned long haddr, pmd_t *pmd) 1926 { 1927 struct mm_struct *mm = vma->vm_mm; 1928 pgtable_t pgtable; 1929 pmd_t _pmd; 1930 int i; 1931 1932 /* 1933 * Leave pmd empty until pte is filled note that it is fine to delay 1934 * notification until mmu_notifier_invalidate_range_end() as we are 1935 * replacing a zero pmd write protected page with a zero pte write 1936 * protected page. 1937 * 1938 * See Documentation/vm/mmu_notifier.rst 1939 */ 1940 pmdp_huge_clear_flush(vma, haddr, pmd); 1941 1942 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 1943 pmd_populate(mm, &_pmd, pgtable); 1944 1945 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 1946 pte_t *pte, entry; 1947 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot); 1948 entry = pte_mkspecial(entry); 1949 pte = pte_offset_map(&_pmd, haddr); 1950 VM_BUG_ON(!pte_none(*pte)); 1951 set_pte_at(mm, haddr, pte, entry); 1952 pte_unmap(pte); 1953 } 1954 smp_wmb(); /* make pte visible before pmd */ 1955 pmd_populate(mm, pmd, pgtable); 1956 } 1957 1958 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 1959 unsigned long haddr, bool freeze) 1960 { 1961 struct mm_struct *mm = vma->vm_mm; 1962 struct page *page; 1963 pgtable_t pgtable; 1964 pmd_t old_pmd, _pmd; 1965 bool young, write, soft_dirty, pmd_migration = false, uffd_wp = false; 1966 unsigned long addr; 1967 int i; 1968 1969 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 1970 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 1971 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 1972 VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd) 1973 && !pmd_devmap(*pmd)); 1974 1975 count_vm_event(THP_SPLIT_PMD); 1976 1977 if (!vma_is_anonymous(vma)) { 1978 old_pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd); 1979 /* 1980 * We are going to unmap this huge page. So 1981 * just go ahead and zap it 1982 */ 1983 if (arch_needs_pgtable_deposit()) 1984 zap_deposited_table(mm, pmd); 1985 if (vma_is_special_huge(vma)) 1986 return; 1987 if (unlikely(is_pmd_migration_entry(old_pmd))) { 1988 swp_entry_t entry; 1989 1990 entry = pmd_to_swp_entry(old_pmd); 1991 page = pfn_swap_entry_to_page(entry); 1992 } else { 1993 page = pmd_page(old_pmd); 1994 if (!PageDirty(page) && pmd_dirty(old_pmd)) 1995 set_page_dirty(page); 1996 if (!PageReferenced(page) && pmd_young(old_pmd)) 1997 SetPageReferenced(page); 1998 page_remove_rmap(page, true); 1999 put_page(page); 2000 } 2001 add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR); 2002 return; 2003 } 2004 2005 if (is_huge_zero_pmd(*pmd)) { 2006 /* 2007 * FIXME: Do we want to invalidate secondary mmu by calling 2008 * mmu_notifier_invalidate_range() see comments below inside 2009 * __split_huge_pmd() ? 2010 * 2011 * We are going from a zero huge page write protected to zero 2012 * small page also write protected so it does not seems useful 2013 * to invalidate secondary mmu at this time. 2014 */ 2015 return __split_huge_zero_page_pmd(vma, haddr, pmd); 2016 } 2017 2018 /* 2019 * Up to this point the pmd is present and huge and userland has the 2020 * whole access to the hugepage during the split (which happens in 2021 * place). If we overwrite the pmd with the not-huge version pointing 2022 * to the pte here (which of course we could if all CPUs were bug 2023 * free), userland could trigger a small page size TLB miss on the 2024 * small sized TLB while the hugepage TLB entry is still established in 2025 * the huge TLB. Some CPU doesn't like that. 2026 * See http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum 2027 * 383 on page 105. Intel should be safe but is also warns that it's 2028 * only safe if the permission and cache attributes of the two entries 2029 * loaded in the two TLB is identical (which should be the case here). 2030 * But it is generally safer to never allow small and huge TLB entries 2031 * for the same virtual address to be loaded simultaneously. So instead 2032 * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the 2033 * current pmd notpresent (atomically because here the pmd_trans_huge 2034 * must remain set at all times on the pmd until the split is complete 2035 * for this pmd), then we flush the SMP TLB and finally we write the 2036 * non-huge version of the pmd entry with pmd_populate. 2037 */ 2038 old_pmd = pmdp_invalidate(vma, haddr, pmd); 2039 2040 pmd_migration = is_pmd_migration_entry(old_pmd); 2041 if (unlikely(pmd_migration)) { 2042 swp_entry_t entry; 2043 2044 entry = pmd_to_swp_entry(old_pmd); 2045 page = pfn_swap_entry_to_page(entry); 2046 write = is_writable_migration_entry(entry); 2047 young = false; 2048 soft_dirty = pmd_swp_soft_dirty(old_pmd); 2049 uffd_wp = pmd_swp_uffd_wp(old_pmd); 2050 } else { 2051 page = pmd_page(old_pmd); 2052 if (pmd_dirty(old_pmd)) 2053 SetPageDirty(page); 2054 write = pmd_write(old_pmd); 2055 young = pmd_young(old_pmd); 2056 soft_dirty = pmd_soft_dirty(old_pmd); 2057 uffd_wp = pmd_uffd_wp(old_pmd); 2058 } 2059 VM_BUG_ON_PAGE(!page_count(page), page); 2060 page_ref_add(page, HPAGE_PMD_NR - 1); 2061 2062 /* 2063 * Withdraw the table only after we mark the pmd entry invalid. 2064 * This's critical for some architectures (Power). 2065 */ 2066 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2067 pmd_populate(mm, &_pmd, pgtable); 2068 2069 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2070 pte_t entry, *pte; 2071 /* 2072 * Note that NUMA hinting access restrictions are not 2073 * transferred to avoid any possibility of altering 2074 * permissions across VMAs. 2075 */ 2076 if (freeze || pmd_migration) { 2077 swp_entry_t swp_entry; 2078 if (write) 2079 swp_entry = make_writable_migration_entry( 2080 page_to_pfn(page + i)); 2081 else 2082 swp_entry = make_readable_migration_entry( 2083 page_to_pfn(page + i)); 2084 entry = swp_entry_to_pte(swp_entry); 2085 if (soft_dirty) 2086 entry = pte_swp_mksoft_dirty(entry); 2087 if (uffd_wp) 2088 entry = pte_swp_mkuffd_wp(entry); 2089 } else { 2090 entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot)); 2091 entry = maybe_mkwrite(entry, vma); 2092 if (!write) 2093 entry = pte_wrprotect(entry); 2094 if (!young) 2095 entry = pte_mkold(entry); 2096 if (soft_dirty) 2097 entry = pte_mksoft_dirty(entry); 2098 if (uffd_wp) 2099 entry = pte_mkuffd_wp(entry); 2100 } 2101 pte = pte_offset_map(&_pmd, addr); 2102 BUG_ON(!pte_none(*pte)); 2103 set_pte_at(mm, addr, pte, entry); 2104 if (!pmd_migration) 2105 atomic_inc(&page[i]._mapcount); 2106 pte_unmap(pte); 2107 } 2108 2109 if (!pmd_migration) { 2110 /* 2111 * Set PG_double_map before dropping compound_mapcount to avoid 2112 * false-negative page_mapped(). 2113 */ 2114 if (compound_mapcount(page) > 1 && 2115 !TestSetPageDoubleMap(page)) { 2116 for (i = 0; i < HPAGE_PMD_NR; i++) 2117 atomic_inc(&page[i]._mapcount); 2118 } 2119 2120 lock_page_memcg(page); 2121 if (atomic_add_negative(-1, compound_mapcount_ptr(page))) { 2122 /* Last compound_mapcount is gone. */ 2123 __mod_lruvec_page_state(page, NR_ANON_THPS, 2124 -HPAGE_PMD_NR); 2125 if (TestClearPageDoubleMap(page)) { 2126 /* No need in mapcount reference anymore */ 2127 for (i = 0; i < HPAGE_PMD_NR; i++) 2128 atomic_dec(&page[i]._mapcount); 2129 } 2130 } 2131 unlock_page_memcg(page); 2132 } 2133 2134 smp_wmb(); /* make pte visible before pmd */ 2135 pmd_populate(mm, pmd, pgtable); 2136 2137 if (freeze) { 2138 for (i = 0; i < HPAGE_PMD_NR; i++) { 2139 page_remove_rmap(page + i, false); 2140 put_page(page + i); 2141 } 2142 } 2143 } 2144 2145 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 2146 unsigned long address, bool freeze, struct page *page) 2147 { 2148 spinlock_t *ptl; 2149 struct mmu_notifier_range range; 2150 bool do_unlock_page = false; 2151 pmd_t _pmd; 2152 2153 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, 2154 address & HPAGE_PMD_MASK, 2155 (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE); 2156 mmu_notifier_invalidate_range_start(&range); 2157 ptl = pmd_lock(vma->vm_mm, pmd); 2158 2159 /* 2160 * If caller asks to setup a migration entries, we need a page to check 2161 * pmd against. Otherwise we can end up replacing wrong page. 2162 */ 2163 VM_BUG_ON(freeze && !page); 2164 if (page) { 2165 VM_WARN_ON_ONCE(!PageLocked(page)); 2166 if (page != pmd_page(*pmd)) 2167 goto out; 2168 } 2169 2170 repeat: 2171 if (pmd_trans_huge(*pmd)) { 2172 if (!page) { 2173 page = pmd_page(*pmd); 2174 /* 2175 * An anonymous page must be locked, to ensure that a 2176 * concurrent reuse_swap_page() sees stable mapcount; 2177 * but reuse_swap_page() is not used on shmem or file, 2178 * and page lock must not be taken when zap_pmd_range() 2179 * calls __split_huge_pmd() while i_mmap_lock is held. 2180 */ 2181 if (PageAnon(page)) { 2182 if (unlikely(!trylock_page(page))) { 2183 get_page(page); 2184 _pmd = *pmd; 2185 spin_unlock(ptl); 2186 lock_page(page); 2187 spin_lock(ptl); 2188 if (unlikely(!pmd_same(*pmd, _pmd))) { 2189 unlock_page(page); 2190 put_page(page); 2191 page = NULL; 2192 goto repeat; 2193 } 2194 put_page(page); 2195 } 2196 do_unlock_page = true; 2197 } 2198 } 2199 if (PageMlocked(page)) 2200 clear_page_mlock(page); 2201 } else if (!(pmd_devmap(*pmd) || is_pmd_migration_entry(*pmd))) 2202 goto out; 2203 __split_huge_pmd_locked(vma, pmd, range.start, freeze); 2204 out: 2205 spin_unlock(ptl); 2206 if (do_unlock_page) 2207 unlock_page(page); 2208 /* 2209 * No need to double call mmu_notifier->invalidate_range() callback. 2210 * They are 3 cases to consider inside __split_huge_pmd_locked(): 2211 * 1) pmdp_huge_clear_flush_notify() call invalidate_range() obvious 2212 * 2) __split_huge_zero_page_pmd() read only zero page and any write 2213 * fault will trigger a flush_notify before pointing to a new page 2214 * (it is fine if the secondary mmu keeps pointing to the old zero 2215 * page in the meantime) 2216 * 3) Split a huge pmd into pte pointing to the same page. No need 2217 * to invalidate secondary tlb entry they are all still valid. 2218 * any further changes to individual pte will notify. So no need 2219 * to call mmu_notifier->invalidate_range() 2220 */ 2221 mmu_notifier_invalidate_range_only_end(&range); 2222 } 2223 2224 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, 2225 bool freeze, struct page *page) 2226 { 2227 pgd_t *pgd; 2228 p4d_t *p4d; 2229 pud_t *pud; 2230 pmd_t *pmd; 2231 2232 pgd = pgd_offset(vma->vm_mm, address); 2233 if (!pgd_present(*pgd)) 2234 return; 2235 2236 p4d = p4d_offset(pgd, address); 2237 if (!p4d_present(*p4d)) 2238 return; 2239 2240 pud = pud_offset(p4d, address); 2241 if (!pud_present(*pud)) 2242 return; 2243 2244 pmd = pmd_offset(pud, address); 2245 2246 __split_huge_pmd(vma, pmd, address, freeze, page); 2247 } 2248 2249 static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address) 2250 { 2251 /* 2252 * If the new address isn't hpage aligned and it could previously 2253 * contain an hugepage: check if we need to split an huge pmd. 2254 */ 2255 if (!IS_ALIGNED(address, HPAGE_PMD_SIZE) && 2256 range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE), 2257 ALIGN(address, HPAGE_PMD_SIZE))) 2258 split_huge_pmd_address(vma, address, false, NULL); 2259 } 2260 2261 void vma_adjust_trans_huge(struct vm_area_struct *vma, 2262 unsigned long start, 2263 unsigned long end, 2264 long adjust_next) 2265 { 2266 /* Check if we need to split start first. */ 2267 split_huge_pmd_if_needed(vma, start); 2268 2269 /* Check if we need to split end next. */ 2270 split_huge_pmd_if_needed(vma, end); 2271 2272 /* 2273 * If we're also updating the vma->vm_next->vm_start, 2274 * check if we need to split it. 2275 */ 2276 if (adjust_next > 0) { 2277 struct vm_area_struct *next = vma->vm_next; 2278 unsigned long nstart = next->vm_start; 2279 nstart += adjust_next; 2280 split_huge_pmd_if_needed(next, nstart); 2281 } 2282 } 2283 2284 static void unmap_page(struct page *page) 2285 { 2286 enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD | 2287 TTU_SYNC; 2288 2289 VM_BUG_ON_PAGE(!PageHead(page), page); 2290 2291 /* 2292 * Anon pages need migration entries to preserve them, but file 2293 * pages can simply be left unmapped, then faulted back on demand. 2294 * If that is ever changed (perhaps for mlock), update remap_page(). 2295 */ 2296 if (PageAnon(page)) 2297 try_to_migrate(page, ttu_flags); 2298 else 2299 try_to_unmap(page, ttu_flags | TTU_IGNORE_MLOCK); 2300 2301 VM_WARN_ON_ONCE_PAGE(page_mapped(page), page); 2302 } 2303 2304 static void remap_page(struct page *page, unsigned int nr) 2305 { 2306 int i; 2307 2308 /* If unmap_page() uses try_to_migrate() on file, remove this check */ 2309 if (!PageAnon(page)) 2310 return; 2311 if (PageTransHuge(page)) { 2312 remove_migration_ptes(page, page, true); 2313 } else { 2314 for (i = 0; i < nr; i++) 2315 remove_migration_ptes(page + i, page + i, true); 2316 } 2317 } 2318 2319 static void lru_add_page_tail(struct page *head, struct page *tail, 2320 struct lruvec *lruvec, struct list_head *list) 2321 { 2322 VM_BUG_ON_PAGE(!PageHead(head), head); 2323 VM_BUG_ON_PAGE(PageCompound(tail), head); 2324 VM_BUG_ON_PAGE(PageLRU(tail), head); 2325 lockdep_assert_held(&lruvec->lru_lock); 2326 2327 if (list) { 2328 /* page reclaim is reclaiming a huge page */ 2329 VM_WARN_ON(PageLRU(head)); 2330 get_page(tail); 2331 list_add_tail(&tail->lru, list); 2332 } else { 2333 /* head is still on lru (and we have it frozen) */ 2334 VM_WARN_ON(!PageLRU(head)); 2335 SetPageLRU(tail); 2336 list_add_tail(&tail->lru, &head->lru); 2337 } 2338 } 2339 2340 static void __split_huge_page_tail(struct page *head, int tail, 2341 struct lruvec *lruvec, struct list_head *list) 2342 { 2343 struct page *page_tail = head + tail; 2344 2345 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail); 2346 2347 /* 2348 * Clone page flags before unfreezing refcount. 2349 * 2350 * After successful get_page_unless_zero() might follow flags change, 2351 * for example lock_page() which set PG_waiters. 2352 */ 2353 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 2354 page_tail->flags |= (head->flags & 2355 ((1L << PG_referenced) | 2356 (1L << PG_swapbacked) | 2357 (1L << PG_swapcache) | 2358 (1L << PG_mlocked) | 2359 (1L << PG_uptodate) | 2360 (1L << PG_active) | 2361 (1L << PG_workingset) | 2362 (1L << PG_locked) | 2363 (1L << PG_unevictable) | 2364 #ifdef CONFIG_64BIT 2365 (1L << PG_arch_2) | 2366 #endif 2367 (1L << PG_dirty))); 2368 2369 /* ->mapping in first tail page is compound_mapcount */ 2370 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, 2371 page_tail); 2372 page_tail->mapping = head->mapping; 2373 page_tail->index = head->index + tail; 2374 2375 /* Page flags must be visible before we make the page non-compound. */ 2376 smp_wmb(); 2377 2378 /* 2379 * Clear PageTail before unfreezing page refcount. 2380 * 2381 * After successful get_page_unless_zero() might follow put_page() 2382 * which needs correct compound_head(). 2383 */ 2384 clear_compound_head(page_tail); 2385 2386 /* Finally unfreeze refcount. Additional reference from page cache. */ 2387 page_ref_unfreeze(page_tail, 1 + (!PageAnon(head) || 2388 PageSwapCache(head))); 2389 2390 if (page_is_young(head)) 2391 set_page_young(page_tail); 2392 if (page_is_idle(head)) 2393 set_page_idle(page_tail); 2394 2395 page_cpupid_xchg_last(page_tail, page_cpupid_last(head)); 2396 2397 /* 2398 * always add to the tail because some iterators expect new 2399 * pages to show after the currently processed elements - e.g. 2400 * migrate_pages 2401 */ 2402 lru_add_page_tail(head, page_tail, lruvec, list); 2403 } 2404 2405 static void __split_huge_page(struct page *page, struct list_head *list, 2406 pgoff_t end) 2407 { 2408 struct page *head = compound_head(page); 2409 struct lruvec *lruvec; 2410 struct address_space *swap_cache = NULL; 2411 unsigned long offset = 0; 2412 unsigned int nr = thp_nr_pages(head); 2413 int i; 2414 2415 /* complete memcg works before add pages to LRU */ 2416 split_page_memcg(head, nr); 2417 2418 if (PageAnon(head) && PageSwapCache(head)) { 2419 swp_entry_t entry = { .val = page_private(head) }; 2420 2421 offset = swp_offset(entry); 2422 swap_cache = swap_address_space(entry); 2423 xa_lock(&swap_cache->i_pages); 2424 } 2425 2426 /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ 2427 lruvec = lock_page_lruvec(head); 2428 2429 for (i = nr - 1; i >= 1; i--) { 2430 __split_huge_page_tail(head, i, lruvec, list); 2431 /* Some pages can be beyond EOF: drop them from page cache */ 2432 if (head[i].index >= end) { 2433 ClearPageDirty(head + i); 2434 __delete_from_page_cache(head + i, NULL); 2435 if (shmem_mapping(head->mapping)) 2436 shmem_uncharge(head->mapping->host, 1); 2437 put_page(head + i); 2438 } else if (!PageAnon(page)) { 2439 __xa_store(&head->mapping->i_pages, head[i].index, 2440 head + i, 0); 2441 } else if (swap_cache) { 2442 __xa_store(&swap_cache->i_pages, offset + i, 2443 head + i, 0); 2444 } 2445 } 2446 2447 ClearPageCompound(head); 2448 unlock_page_lruvec(lruvec); 2449 /* Caller disabled irqs, so they are still disabled here */ 2450 2451 split_page_owner(head, nr); 2452 2453 /* See comment in __split_huge_page_tail() */ 2454 if (PageAnon(head)) { 2455 /* Additional pin to swap cache */ 2456 if (PageSwapCache(head)) { 2457 page_ref_add(head, 2); 2458 xa_unlock(&swap_cache->i_pages); 2459 } else { 2460 page_ref_inc(head); 2461 } 2462 } else { 2463 /* Additional pin to page cache */ 2464 page_ref_add(head, 2); 2465 xa_unlock(&head->mapping->i_pages); 2466 } 2467 local_irq_enable(); 2468 2469 remap_page(head, nr); 2470 2471 if (PageSwapCache(head)) { 2472 swp_entry_t entry = { .val = page_private(head) }; 2473 2474 split_swap_cluster(entry); 2475 } 2476 2477 for (i = 0; i < nr; i++) { 2478 struct page *subpage = head + i; 2479 if (subpage == page) 2480 continue; 2481 unlock_page(subpage); 2482 2483 /* 2484 * Subpages may be freed if there wasn't any mapping 2485 * like if add_to_swap() is running on a lru page that 2486 * had its mapping zapped. And freeing these pages 2487 * requires taking the lru_lock so we do the put_page 2488 * of the tail pages after the split is complete. 2489 */ 2490 put_page(subpage); 2491 } 2492 } 2493 2494 int total_mapcount(struct page *page) 2495 { 2496 int i, compound, nr, ret; 2497 2498 VM_BUG_ON_PAGE(PageTail(page), page); 2499 2500 if (likely(!PageCompound(page))) 2501 return atomic_read(&page->_mapcount) + 1; 2502 2503 compound = compound_mapcount(page); 2504 nr = compound_nr(page); 2505 if (PageHuge(page)) 2506 return compound; 2507 ret = compound; 2508 for (i = 0; i < nr; i++) 2509 ret += atomic_read(&page[i]._mapcount) + 1; 2510 /* File pages has compound_mapcount included in _mapcount */ 2511 if (!PageAnon(page)) 2512 return ret - compound * nr; 2513 if (PageDoubleMap(page)) 2514 ret -= nr; 2515 return ret; 2516 } 2517 2518 /* 2519 * This calculates accurately how many mappings a transparent hugepage 2520 * has (unlike page_mapcount() which isn't fully accurate). This full 2521 * accuracy is primarily needed to know if copy-on-write faults can 2522 * reuse the page and change the mapping to read-write instead of 2523 * copying them. At the same time this returns the total_mapcount too. 2524 * 2525 * The function returns the highest mapcount any one of the subpages 2526 * has. If the return value is one, even if different processes are 2527 * mapping different subpages of the transparent hugepage, they can 2528 * all reuse it, because each process is reusing a different subpage. 2529 * 2530 * The total_mapcount is instead counting all virtual mappings of the 2531 * subpages. If the total_mapcount is equal to "one", it tells the 2532 * caller all mappings belong to the same "mm" and in turn the 2533 * anon_vma of the transparent hugepage can become the vma->anon_vma 2534 * local one as no other process may be mapping any of the subpages. 2535 * 2536 * It would be more accurate to replace page_mapcount() with 2537 * page_trans_huge_mapcount(), however we only use 2538 * page_trans_huge_mapcount() in the copy-on-write faults where we 2539 * need full accuracy to avoid breaking page pinning, because 2540 * page_trans_huge_mapcount() is slower than page_mapcount(). 2541 */ 2542 int page_trans_huge_mapcount(struct page *page, int *total_mapcount) 2543 { 2544 int i, ret, _total_mapcount, mapcount; 2545 2546 /* hugetlbfs shouldn't call it */ 2547 VM_BUG_ON_PAGE(PageHuge(page), page); 2548 2549 if (likely(!PageTransCompound(page))) { 2550 mapcount = atomic_read(&page->_mapcount) + 1; 2551 if (total_mapcount) 2552 *total_mapcount = mapcount; 2553 return mapcount; 2554 } 2555 2556 page = compound_head(page); 2557 2558 _total_mapcount = ret = 0; 2559 for (i = 0; i < thp_nr_pages(page); i++) { 2560 mapcount = atomic_read(&page[i]._mapcount) + 1; 2561 ret = max(ret, mapcount); 2562 _total_mapcount += mapcount; 2563 } 2564 if (PageDoubleMap(page)) { 2565 ret -= 1; 2566 _total_mapcount -= thp_nr_pages(page); 2567 } 2568 mapcount = compound_mapcount(page); 2569 ret += mapcount; 2570 _total_mapcount += mapcount; 2571 if (total_mapcount) 2572 *total_mapcount = _total_mapcount; 2573 return ret; 2574 } 2575 2576 /* Racy check whether the huge page can be split */ 2577 bool can_split_huge_page(struct page *page, int *pextra_pins) 2578 { 2579 int extra_pins; 2580 2581 /* Additional pins from page cache */ 2582 if (PageAnon(page)) 2583 extra_pins = PageSwapCache(page) ? thp_nr_pages(page) : 0; 2584 else 2585 extra_pins = thp_nr_pages(page); 2586 if (pextra_pins) 2587 *pextra_pins = extra_pins; 2588 return total_mapcount(page) == page_count(page) - extra_pins - 1; 2589 } 2590 2591 /* 2592 * This function splits huge page into normal pages. @page can point to any 2593 * subpage of huge page to split. Split doesn't change the position of @page. 2594 * 2595 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY. 2596 * The huge page must be locked. 2597 * 2598 * If @list is null, tail pages will be added to LRU list, otherwise, to @list. 2599 * 2600 * Both head page and tail pages will inherit mapping, flags, and so on from 2601 * the hugepage. 2602 * 2603 * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if 2604 * they are not mapped. 2605 * 2606 * Returns 0 if the hugepage is split successfully. 2607 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under 2608 * us. 2609 */ 2610 int split_huge_page_to_list(struct page *page, struct list_head *list) 2611 { 2612 struct page *head = compound_head(page); 2613 struct deferred_split *ds_queue = get_deferred_split_queue(head); 2614 struct anon_vma *anon_vma = NULL; 2615 struct address_space *mapping = NULL; 2616 int extra_pins, ret; 2617 pgoff_t end; 2618 2619 VM_BUG_ON_PAGE(is_huge_zero_page(head), head); 2620 VM_BUG_ON_PAGE(!PageLocked(head), head); 2621 VM_BUG_ON_PAGE(!PageCompound(head), head); 2622 2623 if (PageWriteback(head)) 2624 return -EBUSY; 2625 2626 if (PageAnon(head)) { 2627 /* 2628 * The caller does not necessarily hold an mmap_lock that would 2629 * prevent the anon_vma disappearing so we first we take a 2630 * reference to it and then lock the anon_vma for write. This 2631 * is similar to page_lock_anon_vma_read except the write lock 2632 * is taken to serialise against parallel split or collapse 2633 * operations. 2634 */ 2635 anon_vma = page_get_anon_vma(head); 2636 if (!anon_vma) { 2637 ret = -EBUSY; 2638 goto out; 2639 } 2640 end = -1; 2641 mapping = NULL; 2642 anon_vma_lock_write(anon_vma); 2643 } else { 2644 mapping = head->mapping; 2645 2646 /* Truncated ? */ 2647 if (!mapping) { 2648 ret = -EBUSY; 2649 goto out; 2650 } 2651 2652 anon_vma = NULL; 2653 i_mmap_lock_read(mapping); 2654 2655 /* 2656 *__split_huge_page() may need to trim off pages beyond EOF: 2657 * but on 32-bit, i_size_read() takes an irq-unsafe seqlock, 2658 * which cannot be nested inside the page tree lock. So note 2659 * end now: i_size itself may be changed at any moment, but 2660 * head page lock is good enough to serialize the trimming. 2661 */ 2662 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); 2663 if (shmem_mapping(mapping)) 2664 end = shmem_fallocend(mapping->host, end); 2665 } 2666 2667 /* 2668 * Racy check if we can split the page, before unmap_page() will 2669 * split PMDs 2670 */ 2671 if (!can_split_huge_page(head, &extra_pins)) { 2672 ret = -EBUSY; 2673 goto out_unlock; 2674 } 2675 2676 unmap_page(head); 2677 2678 /* block interrupt reentry in xa_lock and spinlock */ 2679 local_irq_disable(); 2680 if (mapping) { 2681 XA_STATE(xas, &mapping->i_pages, page_index(head)); 2682 2683 /* 2684 * Check if the head page is present in page cache. 2685 * We assume all tail are present too, if head is there. 2686 */ 2687 xa_lock(&mapping->i_pages); 2688 if (xas_load(&xas) != head) 2689 goto fail; 2690 } 2691 2692 /* Prevent deferred_split_scan() touching ->_refcount */ 2693 spin_lock(&ds_queue->split_queue_lock); 2694 if (page_ref_freeze(head, 1 + extra_pins)) { 2695 if (!list_empty(page_deferred_list(head))) { 2696 ds_queue->split_queue_len--; 2697 list_del(page_deferred_list(head)); 2698 } 2699 spin_unlock(&ds_queue->split_queue_lock); 2700 if (mapping) { 2701 int nr = thp_nr_pages(head); 2702 2703 if (PageSwapBacked(head)) 2704 __mod_lruvec_page_state(head, NR_SHMEM_THPS, 2705 -nr); 2706 else 2707 __mod_lruvec_page_state(head, NR_FILE_THPS, 2708 -nr); 2709 } 2710 2711 __split_huge_page(page, list, end); 2712 ret = 0; 2713 } else { 2714 spin_unlock(&ds_queue->split_queue_lock); 2715 fail: 2716 if (mapping) 2717 xa_unlock(&mapping->i_pages); 2718 local_irq_enable(); 2719 remap_page(head, thp_nr_pages(head)); 2720 ret = -EBUSY; 2721 } 2722 2723 out_unlock: 2724 if (anon_vma) { 2725 anon_vma_unlock_write(anon_vma); 2726 put_anon_vma(anon_vma); 2727 } 2728 if (mapping) 2729 i_mmap_unlock_read(mapping); 2730 out: 2731 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); 2732 return ret; 2733 } 2734 2735 void free_transhuge_page(struct page *page) 2736 { 2737 struct deferred_split *ds_queue = get_deferred_split_queue(page); 2738 unsigned long flags; 2739 2740 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 2741 if (!list_empty(page_deferred_list(page))) { 2742 ds_queue->split_queue_len--; 2743 list_del(page_deferred_list(page)); 2744 } 2745 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 2746 free_compound_page(page); 2747 } 2748 2749 void deferred_split_huge_page(struct page *page) 2750 { 2751 struct deferred_split *ds_queue = get_deferred_split_queue(page); 2752 #ifdef CONFIG_MEMCG 2753 struct mem_cgroup *memcg = page_memcg(compound_head(page)); 2754 #endif 2755 unsigned long flags; 2756 2757 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 2758 2759 /* 2760 * The try_to_unmap() in page reclaim path might reach here too, 2761 * this may cause a race condition to corrupt deferred split queue. 2762 * And, if page reclaim is already handling the same page, it is 2763 * unnecessary to handle it again in shrinker. 2764 * 2765 * Check PageSwapCache to determine if the page is being 2766 * handled by page reclaim since THP swap would add the page into 2767 * swap cache before calling try_to_unmap(). 2768 */ 2769 if (PageSwapCache(page)) 2770 return; 2771 2772 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 2773 if (list_empty(page_deferred_list(page))) { 2774 count_vm_event(THP_DEFERRED_SPLIT_PAGE); 2775 list_add_tail(page_deferred_list(page), &ds_queue->split_queue); 2776 ds_queue->split_queue_len++; 2777 #ifdef CONFIG_MEMCG 2778 if (memcg) 2779 set_shrinker_bit(memcg, page_to_nid(page), 2780 deferred_split_shrinker.id); 2781 #endif 2782 } 2783 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 2784 } 2785 2786 static unsigned long deferred_split_count(struct shrinker *shrink, 2787 struct shrink_control *sc) 2788 { 2789 struct pglist_data *pgdata = NODE_DATA(sc->nid); 2790 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 2791 2792 #ifdef CONFIG_MEMCG 2793 if (sc->memcg) 2794 ds_queue = &sc->memcg->deferred_split_queue; 2795 #endif 2796 return READ_ONCE(ds_queue->split_queue_len); 2797 } 2798 2799 static unsigned long deferred_split_scan(struct shrinker *shrink, 2800 struct shrink_control *sc) 2801 { 2802 struct pglist_data *pgdata = NODE_DATA(sc->nid); 2803 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 2804 unsigned long flags; 2805 LIST_HEAD(list), *pos, *next; 2806 struct page *page; 2807 int split = 0; 2808 2809 #ifdef CONFIG_MEMCG 2810 if (sc->memcg) 2811 ds_queue = &sc->memcg->deferred_split_queue; 2812 #endif 2813 2814 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 2815 /* Take pin on all head pages to avoid freeing them under us */ 2816 list_for_each_safe(pos, next, &ds_queue->split_queue) { 2817 page = list_entry((void *)pos, struct page, deferred_list); 2818 page = compound_head(page); 2819 if (get_page_unless_zero(page)) { 2820 list_move(page_deferred_list(page), &list); 2821 } else { 2822 /* We lost race with put_compound_page() */ 2823 list_del_init(page_deferred_list(page)); 2824 ds_queue->split_queue_len--; 2825 } 2826 if (!--sc->nr_to_scan) 2827 break; 2828 } 2829 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 2830 2831 list_for_each_safe(pos, next, &list) { 2832 page = list_entry((void *)pos, struct page, deferred_list); 2833 if (!trylock_page(page)) 2834 goto next; 2835 /* split_huge_page() removes page from list on success */ 2836 if (!split_huge_page(page)) 2837 split++; 2838 unlock_page(page); 2839 next: 2840 put_page(page); 2841 } 2842 2843 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 2844 list_splice_tail(&list, &ds_queue->split_queue); 2845 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 2846 2847 /* 2848 * Stop shrinker if we didn't split any page, but the queue is empty. 2849 * This can happen if pages were freed under us. 2850 */ 2851 if (!split && list_empty(&ds_queue->split_queue)) 2852 return SHRINK_STOP; 2853 return split; 2854 } 2855 2856 static struct shrinker deferred_split_shrinker = { 2857 .count_objects = deferred_split_count, 2858 .scan_objects = deferred_split_scan, 2859 .seeks = DEFAULT_SEEKS, 2860 .flags = SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE | 2861 SHRINKER_NONSLAB, 2862 }; 2863 2864 #ifdef CONFIG_DEBUG_FS 2865 static void split_huge_pages_all(void) 2866 { 2867 struct zone *zone; 2868 struct page *page; 2869 unsigned long pfn, max_zone_pfn; 2870 unsigned long total = 0, split = 0; 2871 2872 pr_debug("Split all THPs\n"); 2873 for_each_populated_zone(zone) { 2874 max_zone_pfn = zone_end_pfn(zone); 2875 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 2876 if (!pfn_valid(pfn)) 2877 continue; 2878 2879 page = pfn_to_page(pfn); 2880 if (!get_page_unless_zero(page)) 2881 continue; 2882 2883 if (zone != page_zone(page)) 2884 goto next; 2885 2886 if (!PageHead(page) || PageHuge(page) || !PageLRU(page)) 2887 goto next; 2888 2889 total++; 2890 lock_page(page); 2891 if (!split_huge_page(page)) 2892 split++; 2893 unlock_page(page); 2894 next: 2895 put_page(page); 2896 cond_resched(); 2897 } 2898 } 2899 2900 pr_debug("%lu of %lu THP split\n", split, total); 2901 } 2902 2903 static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma) 2904 { 2905 return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) || 2906 is_vm_hugetlb_page(vma); 2907 } 2908 2909 static int split_huge_pages_pid(int pid, unsigned long vaddr_start, 2910 unsigned long vaddr_end) 2911 { 2912 int ret = 0; 2913 struct task_struct *task; 2914 struct mm_struct *mm; 2915 unsigned long total = 0, split = 0; 2916 unsigned long addr; 2917 2918 vaddr_start &= PAGE_MASK; 2919 vaddr_end &= PAGE_MASK; 2920 2921 /* Find the task_struct from pid */ 2922 rcu_read_lock(); 2923 task = find_task_by_vpid(pid); 2924 if (!task) { 2925 rcu_read_unlock(); 2926 ret = -ESRCH; 2927 goto out; 2928 } 2929 get_task_struct(task); 2930 rcu_read_unlock(); 2931 2932 /* Find the mm_struct */ 2933 mm = get_task_mm(task); 2934 put_task_struct(task); 2935 2936 if (!mm) { 2937 ret = -EINVAL; 2938 goto out; 2939 } 2940 2941 pr_debug("Split huge pages in pid: %d, vaddr: [0x%lx - 0x%lx]\n", 2942 pid, vaddr_start, vaddr_end); 2943 2944 mmap_read_lock(mm); 2945 /* 2946 * always increase addr by PAGE_SIZE, since we could have a PTE page 2947 * table filled with PTE-mapped THPs, each of which is distinct. 2948 */ 2949 for (addr = vaddr_start; addr < vaddr_end; addr += PAGE_SIZE) { 2950 struct vm_area_struct *vma = find_vma(mm, addr); 2951 unsigned int follflags; 2952 struct page *page; 2953 2954 if (!vma || addr < vma->vm_start) 2955 break; 2956 2957 /* skip special VMA and hugetlb VMA */ 2958 if (vma_not_suitable_for_thp_split(vma)) { 2959 addr = vma->vm_end; 2960 continue; 2961 } 2962 2963 /* FOLL_DUMP to ignore special (like zero) pages */ 2964 follflags = FOLL_GET | FOLL_DUMP; 2965 page = follow_page(vma, addr, follflags); 2966 2967 if (IS_ERR(page)) 2968 continue; 2969 if (!page) 2970 continue; 2971 2972 if (!is_transparent_hugepage(page)) 2973 goto next; 2974 2975 total++; 2976 if (!can_split_huge_page(compound_head(page), NULL)) 2977 goto next; 2978 2979 if (!trylock_page(page)) 2980 goto next; 2981 2982 if (!split_huge_page(page)) 2983 split++; 2984 2985 unlock_page(page); 2986 next: 2987 put_page(page); 2988 cond_resched(); 2989 } 2990 mmap_read_unlock(mm); 2991 mmput(mm); 2992 2993 pr_debug("%lu of %lu THP split\n", split, total); 2994 2995 out: 2996 return ret; 2997 } 2998 2999 static int split_huge_pages_in_file(const char *file_path, pgoff_t off_start, 3000 pgoff_t off_end) 3001 { 3002 struct filename *file; 3003 struct file *candidate; 3004 struct address_space *mapping; 3005 int ret = -EINVAL; 3006 pgoff_t index; 3007 int nr_pages = 1; 3008 unsigned long total = 0, split = 0; 3009 3010 file = getname_kernel(file_path); 3011 if (IS_ERR(file)) 3012 return ret; 3013 3014 candidate = file_open_name(file, O_RDONLY, 0); 3015 if (IS_ERR(candidate)) 3016 goto out; 3017 3018 pr_debug("split file-backed THPs in file: %s, page offset: [0x%lx - 0x%lx]\n", 3019 file_path, off_start, off_end); 3020 3021 mapping = candidate->f_mapping; 3022 3023 for (index = off_start; index < off_end; index += nr_pages) { 3024 struct page *fpage = pagecache_get_page(mapping, index, 3025 FGP_ENTRY | FGP_HEAD, 0); 3026 3027 nr_pages = 1; 3028 if (xa_is_value(fpage) || !fpage) 3029 continue; 3030 3031 if (!is_transparent_hugepage(fpage)) 3032 goto next; 3033 3034 total++; 3035 nr_pages = thp_nr_pages(fpage); 3036 3037 if (!trylock_page(fpage)) 3038 goto next; 3039 3040 if (!split_huge_page(fpage)) 3041 split++; 3042 3043 unlock_page(fpage); 3044 next: 3045 put_page(fpage); 3046 cond_resched(); 3047 } 3048 3049 filp_close(candidate, NULL); 3050 ret = 0; 3051 3052 pr_debug("%lu of %lu file-backed THP split\n", split, total); 3053 out: 3054 putname(file); 3055 return ret; 3056 } 3057 3058 #define MAX_INPUT_BUF_SZ 255 3059 3060 static ssize_t split_huge_pages_write(struct file *file, const char __user *buf, 3061 size_t count, loff_t *ppops) 3062 { 3063 static DEFINE_MUTEX(split_debug_mutex); 3064 ssize_t ret; 3065 /* hold pid, start_vaddr, end_vaddr or file_path, off_start, off_end */ 3066 char input_buf[MAX_INPUT_BUF_SZ]; 3067 int pid; 3068 unsigned long vaddr_start, vaddr_end; 3069 3070 ret = mutex_lock_interruptible(&split_debug_mutex); 3071 if (ret) 3072 return ret; 3073 3074 ret = -EFAULT; 3075 3076 memset(input_buf, 0, MAX_INPUT_BUF_SZ); 3077 if (copy_from_user(input_buf, buf, min_t(size_t, count, MAX_INPUT_BUF_SZ))) 3078 goto out; 3079 3080 input_buf[MAX_INPUT_BUF_SZ - 1] = '\0'; 3081 3082 if (input_buf[0] == '/') { 3083 char *tok; 3084 char *buf = input_buf; 3085 char file_path[MAX_INPUT_BUF_SZ]; 3086 pgoff_t off_start = 0, off_end = 0; 3087 size_t input_len = strlen(input_buf); 3088 3089 tok = strsep(&buf, ","); 3090 if (tok) { 3091 strcpy(file_path, tok); 3092 } else { 3093 ret = -EINVAL; 3094 goto out; 3095 } 3096 3097 ret = sscanf(buf, "0x%lx,0x%lx", &off_start, &off_end); 3098 if (ret != 2) { 3099 ret = -EINVAL; 3100 goto out; 3101 } 3102 ret = split_huge_pages_in_file(file_path, off_start, off_end); 3103 if (!ret) 3104 ret = input_len; 3105 3106 goto out; 3107 } 3108 3109 ret = sscanf(input_buf, "%d,0x%lx,0x%lx", &pid, &vaddr_start, &vaddr_end); 3110 if (ret == 1 && pid == 1) { 3111 split_huge_pages_all(); 3112 ret = strlen(input_buf); 3113 goto out; 3114 } else if (ret != 3) { 3115 ret = -EINVAL; 3116 goto out; 3117 } 3118 3119 ret = split_huge_pages_pid(pid, vaddr_start, vaddr_end); 3120 if (!ret) 3121 ret = strlen(input_buf); 3122 out: 3123 mutex_unlock(&split_debug_mutex); 3124 return ret; 3125 3126 } 3127 3128 static const struct file_operations split_huge_pages_fops = { 3129 .owner = THIS_MODULE, 3130 .write = split_huge_pages_write, 3131 .llseek = no_llseek, 3132 }; 3133 3134 static int __init split_huge_pages_debugfs(void) 3135 { 3136 debugfs_create_file("split_huge_pages", 0200, NULL, NULL, 3137 &split_huge_pages_fops); 3138 return 0; 3139 } 3140 late_initcall(split_huge_pages_debugfs); 3141 #endif 3142 3143 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 3144 void set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 3145 struct page *page) 3146 { 3147 struct vm_area_struct *vma = pvmw->vma; 3148 struct mm_struct *mm = vma->vm_mm; 3149 unsigned long address = pvmw->address; 3150 pmd_t pmdval; 3151 swp_entry_t entry; 3152 pmd_t pmdswp; 3153 3154 if (!(pvmw->pmd && !pvmw->pte)) 3155 return; 3156 3157 flush_cache_range(vma, address, address + HPAGE_PMD_SIZE); 3158 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); 3159 if (pmd_dirty(pmdval)) 3160 set_page_dirty(page); 3161 if (pmd_write(pmdval)) 3162 entry = make_writable_migration_entry(page_to_pfn(page)); 3163 else 3164 entry = make_readable_migration_entry(page_to_pfn(page)); 3165 pmdswp = swp_entry_to_pmd(entry); 3166 if (pmd_soft_dirty(pmdval)) 3167 pmdswp = pmd_swp_mksoft_dirty(pmdswp); 3168 set_pmd_at(mm, address, pvmw->pmd, pmdswp); 3169 page_remove_rmap(page, true); 3170 put_page(page); 3171 } 3172 3173 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) 3174 { 3175 struct vm_area_struct *vma = pvmw->vma; 3176 struct mm_struct *mm = vma->vm_mm; 3177 unsigned long address = pvmw->address; 3178 unsigned long mmun_start = address & HPAGE_PMD_MASK; 3179 pmd_t pmde; 3180 swp_entry_t entry; 3181 3182 if (!(pvmw->pmd && !pvmw->pte)) 3183 return; 3184 3185 entry = pmd_to_swp_entry(*pvmw->pmd); 3186 get_page(new); 3187 pmde = pmd_mkold(mk_huge_pmd(new, vma->vm_page_prot)); 3188 if (pmd_swp_soft_dirty(*pvmw->pmd)) 3189 pmde = pmd_mksoft_dirty(pmde); 3190 if (is_writable_migration_entry(entry)) 3191 pmde = maybe_pmd_mkwrite(pmde, vma); 3192 if (pmd_swp_uffd_wp(*pvmw->pmd)) 3193 pmde = pmd_wrprotect(pmd_mkuffd_wp(pmde)); 3194 3195 flush_cache_range(vma, mmun_start, mmun_start + HPAGE_PMD_SIZE); 3196 if (PageAnon(new)) 3197 page_add_anon_rmap(new, vma, mmun_start, true); 3198 else 3199 page_add_file_rmap(new, true); 3200 set_pmd_at(mm, mmun_start, pvmw->pmd, pmde); 3201 if ((vma->vm_flags & VM_LOCKED) && !PageDoubleMap(new)) 3202 mlock_vma_page(new); 3203 update_mmu_cache_pmd(vma, address, pvmw->pmd); 3204 } 3205 #endif 3206