1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2009 Red Hat, Inc. 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/mm.h> 9 #include <linux/sched.h> 10 #include <linux/sched/mm.h> 11 #include <linux/sched/numa_balancing.h> 12 #include <linux/highmem.h> 13 #include <linux/hugetlb.h> 14 #include <linux/mmu_notifier.h> 15 #include <linux/rmap.h> 16 #include <linux/swap.h> 17 #include <linux/shrinker.h> 18 #include <linux/mm_inline.h> 19 #include <linux/swapops.h> 20 #include <linux/backing-dev.h> 21 #include <linux/dax.h> 22 #include <linux/mm_types.h> 23 #include <linux/khugepaged.h> 24 #include <linux/freezer.h> 25 #include <linux/pfn_t.h> 26 #include <linux/mman.h> 27 #include <linux/memremap.h> 28 #include <linux/pagemap.h> 29 #include <linux/debugfs.h> 30 #include <linux/migrate.h> 31 #include <linux/hashtable.h> 32 #include <linux/userfaultfd_k.h> 33 #include <linux/page_idle.h> 34 #include <linux/shmem_fs.h> 35 #include <linux/oom.h> 36 #include <linux/numa.h> 37 #include <linux/page_owner.h> 38 #include <linux/sched/sysctl.h> 39 #include <linux/memory-tiers.h> 40 #include <linux/compat.h> 41 #include <linux/pgalloc_tag.h> 42 #include <linux/pagewalk.h> 43 44 #include <asm/tlb.h> 45 #include <asm/pgalloc.h> 46 #include "internal.h" 47 #include "swap.h" 48 49 #define CREATE_TRACE_POINTS 50 #include <trace/events/thp.h> 51 52 /* 53 * By default, transparent hugepage support is disabled in order to avoid 54 * risking an increased memory footprint for applications that are not 55 * guaranteed to benefit from it. When transparent hugepage support is 56 * enabled, it is for all mappings, and khugepaged scans all mappings. 57 * Defrag is invoked by khugepaged hugepage allocations and by page faults 58 * for all hugepage allocations. 59 */ 60 unsigned long transparent_hugepage_flags __read_mostly = 61 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS 62 (1<<TRANSPARENT_HUGEPAGE_FLAG)| 63 #endif 64 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE 65 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| 66 #endif 67 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)| 68 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| 69 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 70 71 static struct shrinker *deferred_split_shrinker; 72 static unsigned long deferred_split_count(struct shrinker *shrink, 73 struct shrink_control *sc); 74 static unsigned long deferred_split_scan(struct shrinker *shrink, 75 struct shrink_control *sc); 76 static bool split_underused_thp = true; 77 78 static atomic_t huge_zero_refcount; 79 struct folio *huge_zero_folio __read_mostly; 80 unsigned long huge_zero_pfn __read_mostly = ~0UL; 81 unsigned long huge_anon_orders_always __read_mostly; 82 unsigned long huge_anon_orders_madvise __read_mostly; 83 unsigned long huge_anon_orders_inherit __read_mostly; 84 static bool anon_orders_configured __initdata; 85 86 static inline bool file_thp_enabled(struct vm_area_struct *vma) 87 { 88 struct inode *inode; 89 90 if (!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS)) 91 return false; 92 93 if (!vma->vm_file) 94 return false; 95 96 inode = file_inode(vma->vm_file); 97 98 return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); 99 } 100 101 unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, 102 unsigned long vm_flags, 103 unsigned long tva_flags, 104 unsigned long orders) 105 { 106 bool smaps = tva_flags & TVA_SMAPS; 107 bool in_pf = tva_flags & TVA_IN_PF; 108 bool enforce_sysfs = tva_flags & TVA_ENFORCE_SYSFS; 109 unsigned long supported_orders; 110 111 /* Check the intersection of requested and supported orders. */ 112 if (vma_is_anonymous(vma)) 113 supported_orders = THP_ORDERS_ALL_ANON; 114 else if (vma_is_special_huge(vma)) 115 supported_orders = THP_ORDERS_ALL_SPECIAL; 116 else 117 supported_orders = THP_ORDERS_ALL_FILE_DEFAULT; 118 119 orders &= supported_orders; 120 if (!orders) 121 return 0; 122 123 if (!vma->vm_mm) /* vdso */ 124 return 0; 125 126 if (thp_disabled_by_hw() || vma_thp_disabled(vma, vm_flags)) 127 return 0; 128 129 /* khugepaged doesn't collapse DAX vma, but page fault is fine. */ 130 if (vma_is_dax(vma)) 131 return in_pf ? orders : 0; 132 133 /* 134 * khugepaged special VMA and hugetlb VMA. 135 * Must be checked after dax since some dax mappings may have 136 * VM_MIXEDMAP set. 137 */ 138 if (!in_pf && !smaps && (vm_flags & VM_NO_KHUGEPAGED)) 139 return 0; 140 141 /* 142 * Check alignment for file vma and size for both file and anon vma by 143 * filtering out the unsuitable orders. 144 * 145 * Skip the check for page fault. Huge fault does the check in fault 146 * handlers. 147 */ 148 if (!in_pf) { 149 int order = highest_order(orders); 150 unsigned long addr; 151 152 while (orders) { 153 addr = vma->vm_end - (PAGE_SIZE << order); 154 if (thp_vma_suitable_order(vma, addr, order)) 155 break; 156 order = next_order(&orders, order); 157 } 158 159 if (!orders) 160 return 0; 161 } 162 163 /* 164 * Enabled via shmem mount options or sysfs settings. 165 * Must be done before hugepage flags check since shmem has its 166 * own flags. 167 */ 168 if (!in_pf && shmem_file(vma->vm_file)) 169 return shmem_allowable_huge_orders(file_inode(vma->vm_file), 170 vma, vma->vm_pgoff, 0, 171 !enforce_sysfs); 172 173 if (!vma_is_anonymous(vma)) { 174 /* 175 * Enforce sysfs THP requirements as necessary. Anonymous vmas 176 * were already handled in thp_vma_allowable_orders(). 177 */ 178 if (enforce_sysfs && 179 (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) && 180 !hugepage_global_always()))) 181 return 0; 182 183 /* 184 * Trust that ->huge_fault() handlers know what they are doing 185 * in fault path. 186 */ 187 if (((in_pf || smaps)) && vma->vm_ops->huge_fault) 188 return orders; 189 /* Only regular file is valid in collapse path */ 190 if (((!in_pf || smaps)) && file_thp_enabled(vma)) 191 return orders; 192 return 0; 193 } 194 195 if (vma_is_temporary_stack(vma)) 196 return 0; 197 198 /* 199 * THPeligible bit of smaps should show 1 for proper VMAs even 200 * though anon_vma is not initialized yet. 201 * 202 * Allow page fault since anon_vma may be not initialized until 203 * the first page fault. 204 */ 205 if (!vma->anon_vma) 206 return (smaps || in_pf) ? orders : 0; 207 208 return orders; 209 } 210 211 static bool get_huge_zero_page(void) 212 { 213 struct folio *zero_folio; 214 retry: 215 if (likely(atomic_inc_not_zero(&huge_zero_refcount))) 216 return true; 217 218 zero_folio = folio_alloc((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE, 219 HPAGE_PMD_ORDER); 220 if (!zero_folio) { 221 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); 222 return false; 223 } 224 /* Ensure zero folio won't have large_rmappable flag set. */ 225 folio_clear_large_rmappable(zero_folio); 226 preempt_disable(); 227 if (cmpxchg(&huge_zero_folio, NULL, zero_folio)) { 228 preempt_enable(); 229 folio_put(zero_folio); 230 goto retry; 231 } 232 WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio)); 233 234 /* We take additional reference here. It will be put back by shrinker */ 235 atomic_set(&huge_zero_refcount, 2); 236 preempt_enable(); 237 count_vm_event(THP_ZERO_PAGE_ALLOC); 238 return true; 239 } 240 241 static void put_huge_zero_page(void) 242 { 243 /* 244 * Counter should never go to zero here. Only shrinker can put 245 * last reference. 246 */ 247 BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); 248 } 249 250 struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) 251 { 252 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 253 return READ_ONCE(huge_zero_folio); 254 255 if (!get_huge_zero_page()) 256 return NULL; 257 258 if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 259 put_huge_zero_page(); 260 261 return READ_ONCE(huge_zero_folio); 262 } 263 264 void mm_put_huge_zero_folio(struct mm_struct *mm) 265 { 266 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 267 put_huge_zero_page(); 268 } 269 270 static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink, 271 struct shrink_control *sc) 272 { 273 /* we can free zero page only if last reference remains */ 274 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; 275 } 276 277 static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink, 278 struct shrink_control *sc) 279 { 280 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { 281 struct folio *zero_folio = xchg(&huge_zero_folio, NULL); 282 BUG_ON(zero_folio == NULL); 283 WRITE_ONCE(huge_zero_pfn, ~0UL); 284 folio_put(zero_folio); 285 return HPAGE_PMD_NR; 286 } 287 288 return 0; 289 } 290 291 static struct shrinker *huge_zero_page_shrinker; 292 293 #ifdef CONFIG_SYSFS 294 static ssize_t enabled_show(struct kobject *kobj, 295 struct kobj_attribute *attr, char *buf) 296 { 297 const char *output; 298 299 if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags)) 300 output = "[always] madvise never"; 301 else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 302 &transparent_hugepage_flags)) 303 output = "always [madvise] never"; 304 else 305 output = "always madvise [never]"; 306 307 return sysfs_emit(buf, "%s\n", output); 308 } 309 310 static ssize_t enabled_store(struct kobject *kobj, 311 struct kobj_attribute *attr, 312 const char *buf, size_t count) 313 { 314 ssize_t ret = count; 315 316 if (sysfs_streq(buf, "always")) { 317 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 318 set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 319 } else if (sysfs_streq(buf, "madvise")) { 320 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 321 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 322 } else if (sysfs_streq(buf, "never")) { 323 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 324 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 325 } else 326 ret = -EINVAL; 327 328 if (ret > 0) { 329 int err = start_stop_khugepaged(); 330 if (err) 331 ret = err; 332 } 333 return ret; 334 } 335 336 static struct kobj_attribute enabled_attr = __ATTR_RW(enabled); 337 338 ssize_t single_hugepage_flag_show(struct kobject *kobj, 339 struct kobj_attribute *attr, char *buf, 340 enum transparent_hugepage_flag flag) 341 { 342 return sysfs_emit(buf, "%d\n", 343 !!test_bit(flag, &transparent_hugepage_flags)); 344 } 345 346 ssize_t single_hugepage_flag_store(struct kobject *kobj, 347 struct kobj_attribute *attr, 348 const char *buf, size_t count, 349 enum transparent_hugepage_flag flag) 350 { 351 unsigned long value; 352 int ret; 353 354 ret = kstrtoul(buf, 10, &value); 355 if (ret < 0) 356 return ret; 357 if (value > 1) 358 return -EINVAL; 359 360 if (value) 361 set_bit(flag, &transparent_hugepage_flags); 362 else 363 clear_bit(flag, &transparent_hugepage_flags); 364 365 return count; 366 } 367 368 static ssize_t defrag_show(struct kobject *kobj, 369 struct kobj_attribute *attr, char *buf) 370 { 371 const char *output; 372 373 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, 374 &transparent_hugepage_flags)) 375 output = "[always] defer defer+madvise madvise never"; 376 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, 377 &transparent_hugepage_flags)) 378 output = "always [defer] defer+madvise madvise never"; 379 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, 380 &transparent_hugepage_flags)) 381 output = "always defer [defer+madvise] madvise never"; 382 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, 383 &transparent_hugepage_flags)) 384 output = "always defer defer+madvise [madvise] never"; 385 else 386 output = "always defer defer+madvise madvise [never]"; 387 388 return sysfs_emit(buf, "%s\n", output); 389 } 390 391 static ssize_t defrag_store(struct kobject *kobj, 392 struct kobj_attribute *attr, 393 const char *buf, size_t count) 394 { 395 if (sysfs_streq(buf, "always")) { 396 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 397 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 398 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 399 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 400 } else if (sysfs_streq(buf, "defer+madvise")) { 401 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 402 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 403 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 404 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 405 } else if (sysfs_streq(buf, "defer")) { 406 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 407 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 408 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 409 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 410 } else if (sysfs_streq(buf, "madvise")) { 411 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 412 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 413 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 414 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 415 } else if (sysfs_streq(buf, "never")) { 416 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 417 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 418 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 419 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 420 } else 421 return -EINVAL; 422 423 return count; 424 } 425 static struct kobj_attribute defrag_attr = __ATTR_RW(defrag); 426 427 static ssize_t use_zero_page_show(struct kobject *kobj, 428 struct kobj_attribute *attr, char *buf) 429 { 430 return single_hugepage_flag_show(kobj, attr, buf, 431 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 432 } 433 static ssize_t use_zero_page_store(struct kobject *kobj, 434 struct kobj_attribute *attr, const char *buf, size_t count) 435 { 436 return single_hugepage_flag_store(kobj, attr, buf, count, 437 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 438 } 439 static struct kobj_attribute use_zero_page_attr = __ATTR_RW(use_zero_page); 440 441 static ssize_t hpage_pmd_size_show(struct kobject *kobj, 442 struct kobj_attribute *attr, char *buf) 443 { 444 return sysfs_emit(buf, "%lu\n", HPAGE_PMD_SIZE); 445 } 446 static struct kobj_attribute hpage_pmd_size_attr = 447 __ATTR_RO(hpage_pmd_size); 448 449 static ssize_t split_underused_thp_show(struct kobject *kobj, 450 struct kobj_attribute *attr, char *buf) 451 { 452 return sysfs_emit(buf, "%d\n", split_underused_thp); 453 } 454 455 static ssize_t split_underused_thp_store(struct kobject *kobj, 456 struct kobj_attribute *attr, 457 const char *buf, size_t count) 458 { 459 int err = kstrtobool(buf, &split_underused_thp); 460 461 if (err < 0) 462 return err; 463 464 return count; 465 } 466 467 static struct kobj_attribute split_underused_thp_attr = __ATTR( 468 shrink_underused, 0644, split_underused_thp_show, split_underused_thp_store); 469 470 static struct attribute *hugepage_attr[] = { 471 &enabled_attr.attr, 472 &defrag_attr.attr, 473 &use_zero_page_attr.attr, 474 &hpage_pmd_size_attr.attr, 475 #ifdef CONFIG_SHMEM 476 &shmem_enabled_attr.attr, 477 #endif 478 &split_underused_thp_attr.attr, 479 NULL, 480 }; 481 482 static const struct attribute_group hugepage_attr_group = { 483 .attrs = hugepage_attr, 484 }; 485 486 static void hugepage_exit_sysfs(struct kobject *hugepage_kobj); 487 static void thpsize_release(struct kobject *kobj); 488 static DEFINE_SPINLOCK(huge_anon_orders_lock); 489 static LIST_HEAD(thpsize_list); 490 491 static ssize_t anon_enabled_show(struct kobject *kobj, 492 struct kobj_attribute *attr, char *buf) 493 { 494 int order = to_thpsize(kobj)->order; 495 const char *output; 496 497 if (test_bit(order, &huge_anon_orders_always)) 498 output = "[always] inherit madvise never"; 499 else if (test_bit(order, &huge_anon_orders_inherit)) 500 output = "always [inherit] madvise never"; 501 else if (test_bit(order, &huge_anon_orders_madvise)) 502 output = "always inherit [madvise] never"; 503 else 504 output = "always inherit madvise [never]"; 505 506 return sysfs_emit(buf, "%s\n", output); 507 } 508 509 static ssize_t anon_enabled_store(struct kobject *kobj, 510 struct kobj_attribute *attr, 511 const char *buf, size_t count) 512 { 513 int order = to_thpsize(kobj)->order; 514 ssize_t ret = count; 515 516 if (sysfs_streq(buf, "always")) { 517 spin_lock(&huge_anon_orders_lock); 518 clear_bit(order, &huge_anon_orders_inherit); 519 clear_bit(order, &huge_anon_orders_madvise); 520 set_bit(order, &huge_anon_orders_always); 521 spin_unlock(&huge_anon_orders_lock); 522 } else if (sysfs_streq(buf, "inherit")) { 523 spin_lock(&huge_anon_orders_lock); 524 clear_bit(order, &huge_anon_orders_always); 525 clear_bit(order, &huge_anon_orders_madvise); 526 set_bit(order, &huge_anon_orders_inherit); 527 spin_unlock(&huge_anon_orders_lock); 528 } else if (sysfs_streq(buf, "madvise")) { 529 spin_lock(&huge_anon_orders_lock); 530 clear_bit(order, &huge_anon_orders_always); 531 clear_bit(order, &huge_anon_orders_inherit); 532 set_bit(order, &huge_anon_orders_madvise); 533 spin_unlock(&huge_anon_orders_lock); 534 } else if (sysfs_streq(buf, "never")) { 535 spin_lock(&huge_anon_orders_lock); 536 clear_bit(order, &huge_anon_orders_always); 537 clear_bit(order, &huge_anon_orders_inherit); 538 clear_bit(order, &huge_anon_orders_madvise); 539 spin_unlock(&huge_anon_orders_lock); 540 } else 541 ret = -EINVAL; 542 543 if (ret > 0) { 544 int err; 545 546 err = start_stop_khugepaged(); 547 if (err) 548 ret = err; 549 } 550 return ret; 551 } 552 553 static struct kobj_attribute anon_enabled_attr = 554 __ATTR(enabled, 0644, anon_enabled_show, anon_enabled_store); 555 556 static struct attribute *anon_ctrl_attrs[] = { 557 &anon_enabled_attr.attr, 558 NULL, 559 }; 560 561 static const struct attribute_group anon_ctrl_attr_grp = { 562 .attrs = anon_ctrl_attrs, 563 }; 564 565 static struct attribute *file_ctrl_attrs[] = { 566 #ifdef CONFIG_SHMEM 567 &thpsize_shmem_enabled_attr.attr, 568 #endif 569 NULL, 570 }; 571 572 static const struct attribute_group file_ctrl_attr_grp = { 573 .attrs = file_ctrl_attrs, 574 }; 575 576 static struct attribute *any_ctrl_attrs[] = { 577 NULL, 578 }; 579 580 static const struct attribute_group any_ctrl_attr_grp = { 581 .attrs = any_ctrl_attrs, 582 }; 583 584 static const struct kobj_type thpsize_ktype = { 585 .release = &thpsize_release, 586 .sysfs_ops = &kobj_sysfs_ops, 587 }; 588 589 DEFINE_PER_CPU(struct mthp_stat, mthp_stats) = {{{0}}}; 590 591 static unsigned long sum_mthp_stat(int order, enum mthp_stat_item item) 592 { 593 unsigned long sum = 0; 594 int cpu; 595 596 for_each_possible_cpu(cpu) { 597 struct mthp_stat *this = &per_cpu(mthp_stats, cpu); 598 599 sum += this->stats[order][item]; 600 } 601 602 return sum; 603 } 604 605 #define DEFINE_MTHP_STAT_ATTR(_name, _index) \ 606 static ssize_t _name##_show(struct kobject *kobj, \ 607 struct kobj_attribute *attr, char *buf) \ 608 { \ 609 int order = to_thpsize(kobj)->order; \ 610 \ 611 return sysfs_emit(buf, "%lu\n", sum_mthp_stat(order, _index)); \ 612 } \ 613 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 614 615 DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC); 616 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK); 617 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); 618 DEFINE_MTHP_STAT_ATTR(zswpout, MTHP_STAT_ZSWPOUT); 619 DEFINE_MTHP_STAT_ATTR(swpin, MTHP_STAT_SWPIN); 620 DEFINE_MTHP_STAT_ATTR(swpin_fallback, MTHP_STAT_SWPIN_FALLBACK); 621 DEFINE_MTHP_STAT_ATTR(swpin_fallback_charge, MTHP_STAT_SWPIN_FALLBACK_CHARGE); 622 DEFINE_MTHP_STAT_ATTR(swpout, MTHP_STAT_SWPOUT); 623 DEFINE_MTHP_STAT_ATTR(swpout_fallback, MTHP_STAT_SWPOUT_FALLBACK); 624 #ifdef CONFIG_SHMEM 625 DEFINE_MTHP_STAT_ATTR(shmem_alloc, MTHP_STAT_SHMEM_ALLOC); 626 DEFINE_MTHP_STAT_ATTR(shmem_fallback, MTHP_STAT_SHMEM_FALLBACK); 627 DEFINE_MTHP_STAT_ATTR(shmem_fallback_charge, MTHP_STAT_SHMEM_FALLBACK_CHARGE); 628 #endif 629 DEFINE_MTHP_STAT_ATTR(split, MTHP_STAT_SPLIT); 630 DEFINE_MTHP_STAT_ATTR(split_failed, MTHP_STAT_SPLIT_FAILED); 631 DEFINE_MTHP_STAT_ATTR(split_deferred, MTHP_STAT_SPLIT_DEFERRED); 632 DEFINE_MTHP_STAT_ATTR(nr_anon, MTHP_STAT_NR_ANON); 633 DEFINE_MTHP_STAT_ATTR(nr_anon_partially_mapped, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED); 634 635 static struct attribute *anon_stats_attrs[] = { 636 &anon_fault_alloc_attr.attr, 637 &anon_fault_fallback_attr.attr, 638 &anon_fault_fallback_charge_attr.attr, 639 #ifndef CONFIG_SHMEM 640 &zswpout_attr.attr, 641 &swpin_attr.attr, 642 &swpin_fallback_attr.attr, 643 &swpin_fallback_charge_attr.attr, 644 &swpout_attr.attr, 645 &swpout_fallback_attr.attr, 646 #endif 647 &split_deferred_attr.attr, 648 &nr_anon_attr.attr, 649 &nr_anon_partially_mapped_attr.attr, 650 NULL, 651 }; 652 653 static struct attribute_group anon_stats_attr_grp = { 654 .name = "stats", 655 .attrs = anon_stats_attrs, 656 }; 657 658 static struct attribute *file_stats_attrs[] = { 659 #ifdef CONFIG_SHMEM 660 &shmem_alloc_attr.attr, 661 &shmem_fallback_attr.attr, 662 &shmem_fallback_charge_attr.attr, 663 #endif 664 NULL, 665 }; 666 667 static struct attribute_group file_stats_attr_grp = { 668 .name = "stats", 669 .attrs = file_stats_attrs, 670 }; 671 672 static struct attribute *any_stats_attrs[] = { 673 #ifdef CONFIG_SHMEM 674 &zswpout_attr.attr, 675 &swpin_attr.attr, 676 &swpin_fallback_attr.attr, 677 &swpin_fallback_charge_attr.attr, 678 &swpout_attr.attr, 679 &swpout_fallback_attr.attr, 680 #endif 681 &split_attr.attr, 682 &split_failed_attr.attr, 683 NULL, 684 }; 685 686 static struct attribute_group any_stats_attr_grp = { 687 .name = "stats", 688 .attrs = any_stats_attrs, 689 }; 690 691 static int sysfs_add_group(struct kobject *kobj, 692 const struct attribute_group *grp) 693 { 694 int ret = -ENOENT; 695 696 /* 697 * If the group is named, try to merge first, assuming the subdirectory 698 * was already created. This avoids the warning emitted by 699 * sysfs_create_group() if the directory already exists. 700 */ 701 if (grp->name) 702 ret = sysfs_merge_group(kobj, grp); 703 if (ret) 704 ret = sysfs_create_group(kobj, grp); 705 706 return ret; 707 } 708 709 static struct thpsize *thpsize_create(int order, struct kobject *parent) 710 { 711 unsigned long size = (PAGE_SIZE << order) / SZ_1K; 712 struct thpsize *thpsize; 713 int ret = -ENOMEM; 714 715 thpsize = kzalloc(sizeof(*thpsize), GFP_KERNEL); 716 if (!thpsize) 717 goto err; 718 719 thpsize->order = order; 720 721 ret = kobject_init_and_add(&thpsize->kobj, &thpsize_ktype, parent, 722 "hugepages-%lukB", size); 723 if (ret) { 724 kfree(thpsize); 725 goto err; 726 } 727 728 729 ret = sysfs_add_group(&thpsize->kobj, &any_ctrl_attr_grp); 730 if (ret) 731 goto err_put; 732 733 ret = sysfs_add_group(&thpsize->kobj, &any_stats_attr_grp); 734 if (ret) 735 goto err_put; 736 737 if (BIT(order) & THP_ORDERS_ALL_ANON) { 738 ret = sysfs_add_group(&thpsize->kobj, &anon_ctrl_attr_grp); 739 if (ret) 740 goto err_put; 741 742 ret = sysfs_add_group(&thpsize->kobj, &anon_stats_attr_grp); 743 if (ret) 744 goto err_put; 745 } 746 747 if (BIT(order) & THP_ORDERS_ALL_FILE_DEFAULT) { 748 ret = sysfs_add_group(&thpsize->kobj, &file_ctrl_attr_grp); 749 if (ret) 750 goto err_put; 751 752 ret = sysfs_add_group(&thpsize->kobj, &file_stats_attr_grp); 753 if (ret) 754 goto err_put; 755 } 756 757 return thpsize; 758 err_put: 759 kobject_put(&thpsize->kobj); 760 err: 761 return ERR_PTR(ret); 762 } 763 764 static void thpsize_release(struct kobject *kobj) 765 { 766 kfree(to_thpsize(kobj)); 767 } 768 769 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj) 770 { 771 int err; 772 struct thpsize *thpsize; 773 unsigned long orders; 774 int order; 775 776 /* 777 * Default to setting PMD-sized THP to inherit the global setting and 778 * disable all other sizes. powerpc's PMD_ORDER isn't a compile-time 779 * constant so we have to do this here. 780 */ 781 if (!anon_orders_configured) 782 huge_anon_orders_inherit = BIT(PMD_ORDER); 783 784 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj); 785 if (unlikely(!*hugepage_kobj)) { 786 pr_err("failed to create transparent hugepage kobject\n"); 787 return -ENOMEM; 788 } 789 790 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group); 791 if (err) { 792 pr_err("failed to register transparent hugepage group\n"); 793 goto delete_obj; 794 } 795 796 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group); 797 if (err) { 798 pr_err("failed to register transparent hugepage group\n"); 799 goto remove_hp_group; 800 } 801 802 orders = THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_FILE_DEFAULT; 803 order = highest_order(orders); 804 while (orders) { 805 thpsize = thpsize_create(order, *hugepage_kobj); 806 if (IS_ERR(thpsize)) { 807 pr_err("failed to create thpsize for order %d\n", order); 808 err = PTR_ERR(thpsize); 809 goto remove_all; 810 } 811 list_add(&thpsize->node, &thpsize_list); 812 order = next_order(&orders, order); 813 } 814 815 return 0; 816 817 remove_all: 818 hugepage_exit_sysfs(*hugepage_kobj); 819 return err; 820 remove_hp_group: 821 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group); 822 delete_obj: 823 kobject_put(*hugepage_kobj); 824 return err; 825 } 826 827 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj) 828 { 829 struct thpsize *thpsize, *tmp; 830 831 list_for_each_entry_safe(thpsize, tmp, &thpsize_list, node) { 832 list_del(&thpsize->node); 833 kobject_put(&thpsize->kobj); 834 } 835 836 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group); 837 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group); 838 kobject_put(hugepage_kobj); 839 } 840 #else 841 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj) 842 { 843 return 0; 844 } 845 846 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) 847 { 848 } 849 #endif /* CONFIG_SYSFS */ 850 851 static int __init thp_shrinker_init(void) 852 { 853 huge_zero_page_shrinker = shrinker_alloc(0, "thp-zero"); 854 if (!huge_zero_page_shrinker) 855 return -ENOMEM; 856 857 deferred_split_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE | 858 SHRINKER_MEMCG_AWARE | 859 SHRINKER_NONSLAB, 860 "thp-deferred_split"); 861 if (!deferred_split_shrinker) { 862 shrinker_free(huge_zero_page_shrinker); 863 return -ENOMEM; 864 } 865 866 huge_zero_page_shrinker->count_objects = shrink_huge_zero_page_count; 867 huge_zero_page_shrinker->scan_objects = shrink_huge_zero_page_scan; 868 shrinker_register(huge_zero_page_shrinker); 869 870 deferred_split_shrinker->count_objects = deferred_split_count; 871 deferred_split_shrinker->scan_objects = deferred_split_scan; 872 shrinker_register(deferred_split_shrinker); 873 874 return 0; 875 } 876 877 static void __init thp_shrinker_exit(void) 878 { 879 shrinker_free(huge_zero_page_shrinker); 880 shrinker_free(deferred_split_shrinker); 881 } 882 883 static int __init hugepage_init(void) 884 { 885 int err; 886 struct kobject *hugepage_kobj; 887 888 if (!has_transparent_hugepage()) { 889 transparent_hugepage_flags = 1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED; 890 return -EINVAL; 891 } 892 893 /* 894 * hugepages can't be allocated by the buddy allocator 895 */ 896 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER > MAX_PAGE_ORDER); 897 898 err = hugepage_init_sysfs(&hugepage_kobj); 899 if (err) 900 goto err_sysfs; 901 902 err = khugepaged_init(); 903 if (err) 904 goto err_slab; 905 906 err = thp_shrinker_init(); 907 if (err) 908 goto err_shrinker; 909 910 /* 911 * By default disable transparent hugepages on smaller systems, 912 * where the extra memory used could hurt more than TLB overhead 913 * is likely to save. The admin can still enable it through /sys. 914 */ 915 if (totalram_pages() < (512 << (20 - PAGE_SHIFT))) { 916 transparent_hugepage_flags = 0; 917 return 0; 918 } 919 920 err = start_stop_khugepaged(); 921 if (err) 922 goto err_khugepaged; 923 924 return 0; 925 err_khugepaged: 926 thp_shrinker_exit(); 927 err_shrinker: 928 khugepaged_destroy(); 929 err_slab: 930 hugepage_exit_sysfs(hugepage_kobj); 931 err_sysfs: 932 return err; 933 } 934 subsys_initcall(hugepage_init); 935 936 static int __init setup_transparent_hugepage(char *str) 937 { 938 int ret = 0; 939 if (!str) 940 goto out; 941 if (!strcmp(str, "always")) { 942 set_bit(TRANSPARENT_HUGEPAGE_FLAG, 943 &transparent_hugepage_flags); 944 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 945 &transparent_hugepage_flags); 946 ret = 1; 947 } else if (!strcmp(str, "madvise")) { 948 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 949 &transparent_hugepage_flags); 950 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 951 &transparent_hugepage_flags); 952 ret = 1; 953 } else if (!strcmp(str, "never")) { 954 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 955 &transparent_hugepage_flags); 956 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 957 &transparent_hugepage_flags); 958 ret = 1; 959 } 960 out: 961 if (!ret) 962 pr_warn("transparent_hugepage= cannot parse, ignored\n"); 963 return ret; 964 } 965 __setup("transparent_hugepage=", setup_transparent_hugepage); 966 967 static char str_dup[PAGE_SIZE] __initdata; 968 static int __init setup_thp_anon(char *str) 969 { 970 char *token, *range, *policy, *subtoken; 971 unsigned long always, inherit, madvise; 972 char *start_size, *end_size; 973 int start, end, nr; 974 char *p; 975 976 if (!str || strlen(str) + 1 > PAGE_SIZE) 977 goto err; 978 strscpy(str_dup, str); 979 980 always = huge_anon_orders_always; 981 madvise = huge_anon_orders_madvise; 982 inherit = huge_anon_orders_inherit; 983 p = str_dup; 984 while ((token = strsep(&p, ";")) != NULL) { 985 range = strsep(&token, ":"); 986 policy = token; 987 988 if (!policy) 989 goto err; 990 991 while ((subtoken = strsep(&range, ",")) != NULL) { 992 if (strchr(subtoken, '-')) { 993 start_size = strsep(&subtoken, "-"); 994 end_size = subtoken; 995 996 start = get_order_from_str(start_size, THP_ORDERS_ALL_ANON); 997 end = get_order_from_str(end_size, THP_ORDERS_ALL_ANON); 998 } else { 999 start_size = end_size = subtoken; 1000 start = end = get_order_from_str(subtoken, 1001 THP_ORDERS_ALL_ANON); 1002 } 1003 1004 if (start == -EINVAL) { 1005 pr_err("invalid size %s in thp_anon boot parameter\n", start_size); 1006 goto err; 1007 } 1008 1009 if (end == -EINVAL) { 1010 pr_err("invalid size %s in thp_anon boot parameter\n", end_size); 1011 goto err; 1012 } 1013 1014 if (start < 0 || end < 0 || start > end) 1015 goto err; 1016 1017 nr = end - start + 1; 1018 if (!strcmp(policy, "always")) { 1019 bitmap_set(&always, start, nr); 1020 bitmap_clear(&inherit, start, nr); 1021 bitmap_clear(&madvise, start, nr); 1022 } else if (!strcmp(policy, "madvise")) { 1023 bitmap_set(&madvise, start, nr); 1024 bitmap_clear(&inherit, start, nr); 1025 bitmap_clear(&always, start, nr); 1026 } else if (!strcmp(policy, "inherit")) { 1027 bitmap_set(&inherit, start, nr); 1028 bitmap_clear(&madvise, start, nr); 1029 bitmap_clear(&always, start, nr); 1030 } else if (!strcmp(policy, "never")) { 1031 bitmap_clear(&inherit, start, nr); 1032 bitmap_clear(&madvise, start, nr); 1033 bitmap_clear(&always, start, nr); 1034 } else { 1035 pr_err("invalid policy %s in thp_anon boot parameter\n", policy); 1036 goto err; 1037 } 1038 } 1039 } 1040 1041 huge_anon_orders_always = always; 1042 huge_anon_orders_madvise = madvise; 1043 huge_anon_orders_inherit = inherit; 1044 anon_orders_configured = true; 1045 return 1; 1046 1047 err: 1048 pr_warn("thp_anon=%s: error parsing string, ignoring setting\n", str); 1049 return 0; 1050 } 1051 __setup("thp_anon=", setup_thp_anon); 1052 1053 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 1054 { 1055 if (likely(vma->vm_flags & VM_WRITE)) 1056 pmd = pmd_mkwrite(pmd, vma); 1057 return pmd; 1058 } 1059 1060 #ifdef CONFIG_MEMCG 1061 static inline 1062 struct deferred_split *get_deferred_split_queue(struct folio *folio) 1063 { 1064 struct mem_cgroup *memcg = folio_memcg(folio); 1065 struct pglist_data *pgdat = NODE_DATA(folio_nid(folio)); 1066 1067 if (memcg) 1068 return &memcg->deferred_split_queue; 1069 else 1070 return &pgdat->deferred_split_queue; 1071 } 1072 #else 1073 static inline 1074 struct deferred_split *get_deferred_split_queue(struct folio *folio) 1075 { 1076 struct pglist_data *pgdat = NODE_DATA(folio_nid(folio)); 1077 1078 return &pgdat->deferred_split_queue; 1079 } 1080 #endif 1081 1082 static inline bool is_transparent_hugepage(const struct folio *folio) 1083 { 1084 if (!folio_test_large(folio)) 1085 return false; 1086 1087 return is_huge_zero_folio(folio) || 1088 folio_test_large_rmappable(folio); 1089 } 1090 1091 static unsigned long __thp_get_unmapped_area(struct file *filp, 1092 unsigned long addr, unsigned long len, 1093 loff_t off, unsigned long flags, unsigned long size, 1094 vm_flags_t vm_flags) 1095 { 1096 loff_t off_end = off + len; 1097 loff_t off_align = round_up(off, size); 1098 unsigned long len_pad, ret, off_sub; 1099 1100 if (!IS_ENABLED(CONFIG_64BIT) || in_compat_syscall()) 1101 return 0; 1102 1103 if (off_end <= off_align || (off_end - off_align) < size) 1104 return 0; 1105 1106 len_pad = len + size; 1107 if (len_pad < len || (off + len_pad) < off) 1108 return 0; 1109 1110 ret = mm_get_unmapped_area_vmflags(current->mm, filp, addr, len_pad, 1111 off >> PAGE_SHIFT, flags, vm_flags); 1112 1113 /* 1114 * The failure might be due to length padding. The caller will retry 1115 * without the padding. 1116 */ 1117 if (IS_ERR_VALUE(ret)) 1118 return 0; 1119 1120 /* 1121 * Do not try to align to THP boundary if allocation at the address 1122 * hint succeeds. 1123 */ 1124 if (ret == addr) 1125 return addr; 1126 1127 off_sub = (off - ret) & (size - 1); 1128 1129 if (test_bit(MMF_TOPDOWN, ¤t->mm->flags) && !off_sub) 1130 return ret + size; 1131 1132 ret += off_sub; 1133 return ret; 1134 } 1135 1136 unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, 1137 unsigned long len, unsigned long pgoff, unsigned long flags, 1138 vm_flags_t vm_flags) 1139 { 1140 unsigned long ret; 1141 loff_t off = (loff_t)pgoff << PAGE_SHIFT; 1142 1143 ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, vm_flags); 1144 if (ret) 1145 return ret; 1146 1147 return mm_get_unmapped_area_vmflags(current->mm, filp, addr, len, pgoff, flags, 1148 vm_flags); 1149 } 1150 1151 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, 1152 unsigned long len, unsigned long pgoff, unsigned long flags) 1153 { 1154 return thp_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0); 1155 } 1156 EXPORT_SYMBOL_GPL(thp_get_unmapped_area); 1157 1158 static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma, 1159 unsigned long addr) 1160 { 1161 gfp_t gfp = vma_thp_gfp_mask(vma); 1162 const int order = HPAGE_PMD_ORDER; 1163 struct folio *folio; 1164 1165 folio = vma_alloc_folio(gfp, order, vma, addr & HPAGE_PMD_MASK); 1166 1167 if (unlikely(!folio)) { 1168 count_vm_event(THP_FAULT_FALLBACK); 1169 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK); 1170 return NULL; 1171 } 1172 1173 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 1174 if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) { 1175 folio_put(folio); 1176 count_vm_event(THP_FAULT_FALLBACK); 1177 count_vm_event(THP_FAULT_FALLBACK_CHARGE); 1178 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK); 1179 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); 1180 return NULL; 1181 } 1182 folio_throttle_swaprate(folio, gfp); 1183 1184 /* 1185 * When a folio is not zeroed during allocation (__GFP_ZERO not used) 1186 * or user folios require special handling, folio_zero_user() is used to 1187 * make sure that the page corresponding to the faulting address will be 1188 * hot in the cache after zeroing. 1189 */ 1190 if (user_alloc_needs_zeroing()) 1191 folio_zero_user(folio, addr); 1192 /* 1193 * The memory barrier inside __folio_mark_uptodate makes sure that 1194 * folio_zero_user writes become visible before the set_pmd_at() 1195 * write. 1196 */ 1197 __folio_mark_uptodate(folio); 1198 return folio; 1199 } 1200 1201 static void map_anon_folio_pmd(struct folio *folio, pmd_t *pmd, 1202 struct vm_area_struct *vma, unsigned long haddr) 1203 { 1204 pmd_t entry; 1205 1206 entry = folio_mk_pmd(folio, vma->vm_page_prot); 1207 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1208 folio_add_new_anon_rmap(folio, vma, haddr, RMAP_EXCLUSIVE); 1209 folio_add_lru_vma(folio, vma); 1210 set_pmd_at(vma->vm_mm, haddr, pmd, entry); 1211 update_mmu_cache_pmd(vma, haddr, pmd); 1212 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1213 count_vm_event(THP_FAULT_ALLOC); 1214 count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_ALLOC); 1215 count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); 1216 } 1217 1218 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf) 1219 { 1220 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1221 struct vm_area_struct *vma = vmf->vma; 1222 struct folio *folio; 1223 pgtable_t pgtable; 1224 vm_fault_t ret = 0; 1225 1226 folio = vma_alloc_anon_folio_pmd(vma, vmf->address); 1227 if (unlikely(!folio)) 1228 return VM_FAULT_FALLBACK; 1229 1230 pgtable = pte_alloc_one(vma->vm_mm); 1231 if (unlikely(!pgtable)) { 1232 ret = VM_FAULT_OOM; 1233 goto release; 1234 } 1235 1236 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1237 if (unlikely(!pmd_none(*vmf->pmd))) { 1238 goto unlock_release; 1239 } else { 1240 ret = check_stable_address_space(vma->vm_mm); 1241 if (ret) 1242 goto unlock_release; 1243 1244 /* Deliver the page fault to userland */ 1245 if (userfaultfd_missing(vma)) { 1246 spin_unlock(vmf->ptl); 1247 folio_put(folio); 1248 pte_free(vma->vm_mm, pgtable); 1249 ret = handle_userfault(vmf, VM_UFFD_MISSING); 1250 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 1251 return ret; 1252 } 1253 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); 1254 map_anon_folio_pmd(folio, vmf->pmd, vma, haddr); 1255 mm_inc_nr_ptes(vma->vm_mm); 1256 deferred_split_folio(folio, false); 1257 spin_unlock(vmf->ptl); 1258 } 1259 1260 return 0; 1261 unlock_release: 1262 spin_unlock(vmf->ptl); 1263 release: 1264 if (pgtable) 1265 pte_free(vma->vm_mm, pgtable); 1266 folio_put(folio); 1267 return ret; 1268 1269 } 1270 1271 /* 1272 * always: directly stall for all thp allocations 1273 * defer: wake kswapd and fail if not immediately available 1274 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise 1275 * fail if not immediately available 1276 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately 1277 * available 1278 * never: never stall for any thp allocation 1279 */ 1280 gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma) 1281 { 1282 const bool vma_madvised = vma && (vma->vm_flags & VM_HUGEPAGE); 1283 1284 /* Always do synchronous compaction */ 1285 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) 1286 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY); 1287 1288 /* Kick kcompactd and fail quickly */ 1289 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) 1290 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM; 1291 1292 /* Synchronous compaction if madvised, otherwise kick kcompactd */ 1293 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags)) 1294 return GFP_TRANSHUGE_LIGHT | 1295 (vma_madvised ? __GFP_DIRECT_RECLAIM : 1296 __GFP_KSWAPD_RECLAIM); 1297 1298 /* Only do synchronous compaction if madvised */ 1299 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags)) 1300 return GFP_TRANSHUGE_LIGHT | 1301 (vma_madvised ? __GFP_DIRECT_RECLAIM : 0); 1302 1303 return GFP_TRANSHUGE_LIGHT; 1304 } 1305 1306 /* Caller must hold page table lock. */ 1307 static void set_huge_zero_folio(pgtable_t pgtable, struct mm_struct *mm, 1308 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, 1309 struct folio *zero_folio) 1310 { 1311 pmd_t entry; 1312 entry = folio_mk_pmd(zero_folio, vma->vm_page_prot); 1313 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1314 set_pmd_at(mm, haddr, pmd, entry); 1315 mm_inc_nr_ptes(mm); 1316 } 1317 1318 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) 1319 { 1320 struct vm_area_struct *vma = vmf->vma; 1321 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1322 vm_fault_t ret; 1323 1324 if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER)) 1325 return VM_FAULT_FALLBACK; 1326 ret = vmf_anon_prepare(vmf); 1327 if (ret) 1328 return ret; 1329 khugepaged_enter_vma(vma, vma->vm_flags); 1330 1331 if (!(vmf->flags & FAULT_FLAG_WRITE) && 1332 !mm_forbids_zeropage(vma->vm_mm) && 1333 transparent_hugepage_use_zero_page()) { 1334 pgtable_t pgtable; 1335 struct folio *zero_folio; 1336 vm_fault_t ret; 1337 1338 pgtable = pte_alloc_one(vma->vm_mm); 1339 if (unlikely(!pgtable)) 1340 return VM_FAULT_OOM; 1341 zero_folio = mm_get_huge_zero_folio(vma->vm_mm); 1342 if (unlikely(!zero_folio)) { 1343 pte_free(vma->vm_mm, pgtable); 1344 count_vm_event(THP_FAULT_FALLBACK); 1345 return VM_FAULT_FALLBACK; 1346 } 1347 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1348 ret = 0; 1349 if (pmd_none(*vmf->pmd)) { 1350 ret = check_stable_address_space(vma->vm_mm); 1351 if (ret) { 1352 spin_unlock(vmf->ptl); 1353 pte_free(vma->vm_mm, pgtable); 1354 } else if (userfaultfd_missing(vma)) { 1355 spin_unlock(vmf->ptl); 1356 pte_free(vma->vm_mm, pgtable); 1357 ret = handle_userfault(vmf, VM_UFFD_MISSING); 1358 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 1359 } else { 1360 set_huge_zero_folio(pgtable, vma->vm_mm, vma, 1361 haddr, vmf->pmd, zero_folio); 1362 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1363 spin_unlock(vmf->ptl); 1364 } 1365 } else { 1366 spin_unlock(vmf->ptl); 1367 pte_free(vma->vm_mm, pgtable); 1368 } 1369 return ret; 1370 } 1371 1372 return __do_huge_pmd_anonymous_page(vmf); 1373 } 1374 1375 static int insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 1376 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write, 1377 pgtable_t pgtable) 1378 { 1379 struct mm_struct *mm = vma->vm_mm; 1380 pmd_t entry; 1381 1382 lockdep_assert_held(pmd_lockptr(mm, pmd)); 1383 1384 if (!pmd_none(*pmd)) { 1385 if (write) { 1386 if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) { 1387 WARN_ON_ONCE(!is_huge_zero_pmd(*pmd)); 1388 return -EEXIST; 1389 } 1390 entry = pmd_mkyoung(*pmd); 1391 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1392 if (pmdp_set_access_flags(vma, addr, pmd, entry, 1)) 1393 update_mmu_cache_pmd(vma, addr, pmd); 1394 } 1395 1396 return -EEXIST; 1397 } 1398 1399 entry = pmd_mkhuge(pfn_t_pmd(pfn, prot)); 1400 if (pfn_t_devmap(pfn)) 1401 entry = pmd_mkdevmap(entry); 1402 else 1403 entry = pmd_mkspecial(entry); 1404 if (write) { 1405 entry = pmd_mkyoung(pmd_mkdirty(entry)); 1406 entry = maybe_pmd_mkwrite(entry, vma); 1407 } 1408 1409 if (pgtable) { 1410 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1411 mm_inc_nr_ptes(mm); 1412 } 1413 1414 set_pmd_at(mm, addr, pmd, entry); 1415 update_mmu_cache_pmd(vma, addr, pmd); 1416 return 0; 1417 } 1418 1419 /** 1420 * vmf_insert_pfn_pmd - insert a pmd size pfn 1421 * @vmf: Structure describing the fault 1422 * @pfn: pfn to insert 1423 * @write: whether it's a write fault 1424 * 1425 * Insert a pmd size pfn. See vmf_insert_pfn() for additional info. 1426 * 1427 * Return: vm_fault_t value. 1428 */ 1429 vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write) 1430 { 1431 unsigned long addr = vmf->address & PMD_MASK; 1432 struct vm_area_struct *vma = vmf->vma; 1433 pgprot_t pgprot = vma->vm_page_prot; 1434 pgtable_t pgtable = NULL; 1435 spinlock_t *ptl; 1436 int error; 1437 1438 /* 1439 * If we had pmd_special, we could avoid all these restrictions, 1440 * but we need to be consistent with PTEs and architectures that 1441 * can't support a 'special' bit. 1442 */ 1443 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && 1444 !pfn_t_devmap(pfn)); 1445 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1446 (VM_PFNMAP|VM_MIXEDMAP)); 1447 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1448 1449 if (addr < vma->vm_start || addr >= vma->vm_end) 1450 return VM_FAULT_SIGBUS; 1451 1452 if (arch_needs_pgtable_deposit()) { 1453 pgtable = pte_alloc_one(vma->vm_mm); 1454 if (!pgtable) 1455 return VM_FAULT_OOM; 1456 } 1457 1458 track_pfn_insert(vma, &pgprot, pfn); 1459 ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1460 error = insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, 1461 pgtable); 1462 spin_unlock(ptl); 1463 if (error && pgtable) 1464 pte_free(vma->vm_mm, pgtable); 1465 1466 return VM_FAULT_NOPAGE; 1467 } 1468 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd); 1469 1470 vm_fault_t vmf_insert_folio_pmd(struct vm_fault *vmf, struct folio *folio, 1471 bool write) 1472 { 1473 struct vm_area_struct *vma = vmf->vma; 1474 unsigned long addr = vmf->address & PMD_MASK; 1475 struct mm_struct *mm = vma->vm_mm; 1476 spinlock_t *ptl; 1477 pgtable_t pgtable = NULL; 1478 int error; 1479 1480 if (addr < vma->vm_start || addr >= vma->vm_end) 1481 return VM_FAULT_SIGBUS; 1482 1483 if (WARN_ON_ONCE(folio_order(folio) != PMD_ORDER)) 1484 return VM_FAULT_SIGBUS; 1485 1486 if (arch_needs_pgtable_deposit()) { 1487 pgtable = pte_alloc_one(vma->vm_mm); 1488 if (!pgtable) 1489 return VM_FAULT_OOM; 1490 } 1491 1492 ptl = pmd_lock(mm, vmf->pmd); 1493 if (pmd_none(*vmf->pmd)) { 1494 folio_get(folio); 1495 folio_add_file_rmap_pmd(folio, &folio->page, vma); 1496 add_mm_counter(mm, mm_counter_file(folio), HPAGE_PMD_NR); 1497 } 1498 error = insert_pfn_pmd(vma, addr, vmf->pmd, 1499 pfn_to_pfn_t(folio_pfn(folio)), vma->vm_page_prot, 1500 write, pgtable); 1501 spin_unlock(ptl); 1502 if (error && pgtable) 1503 pte_free(mm, pgtable); 1504 1505 return VM_FAULT_NOPAGE; 1506 } 1507 EXPORT_SYMBOL_GPL(vmf_insert_folio_pmd); 1508 1509 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1510 static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma) 1511 { 1512 if (likely(vma->vm_flags & VM_WRITE)) 1513 pud = pud_mkwrite(pud); 1514 return pud; 1515 } 1516 1517 static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr, 1518 pud_t *pud, pfn_t pfn, bool write) 1519 { 1520 struct mm_struct *mm = vma->vm_mm; 1521 pgprot_t prot = vma->vm_page_prot; 1522 pud_t entry; 1523 1524 if (!pud_none(*pud)) { 1525 if (write) { 1526 if (WARN_ON_ONCE(pud_pfn(*pud) != pfn_t_to_pfn(pfn))) 1527 return; 1528 entry = pud_mkyoung(*pud); 1529 entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma); 1530 if (pudp_set_access_flags(vma, addr, pud, entry, 1)) 1531 update_mmu_cache_pud(vma, addr, pud); 1532 } 1533 return; 1534 } 1535 1536 entry = pud_mkhuge(pfn_t_pud(pfn, prot)); 1537 if (pfn_t_devmap(pfn)) 1538 entry = pud_mkdevmap(entry); 1539 else 1540 entry = pud_mkspecial(entry); 1541 if (write) { 1542 entry = pud_mkyoung(pud_mkdirty(entry)); 1543 entry = maybe_pud_mkwrite(entry, vma); 1544 } 1545 set_pud_at(mm, addr, pud, entry); 1546 update_mmu_cache_pud(vma, addr, pud); 1547 } 1548 1549 /** 1550 * vmf_insert_pfn_pud - insert a pud size pfn 1551 * @vmf: Structure describing the fault 1552 * @pfn: pfn to insert 1553 * @write: whether it's a write fault 1554 * 1555 * Insert a pud size pfn. See vmf_insert_pfn() for additional info. 1556 * 1557 * Return: vm_fault_t value. 1558 */ 1559 vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write) 1560 { 1561 unsigned long addr = vmf->address & PUD_MASK; 1562 struct vm_area_struct *vma = vmf->vma; 1563 pgprot_t pgprot = vma->vm_page_prot; 1564 spinlock_t *ptl; 1565 1566 /* 1567 * If we had pud_special, we could avoid all these restrictions, 1568 * but we need to be consistent with PTEs and architectures that 1569 * can't support a 'special' bit. 1570 */ 1571 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && 1572 !pfn_t_devmap(pfn)); 1573 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1574 (VM_PFNMAP|VM_MIXEDMAP)); 1575 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1576 1577 if (addr < vma->vm_start || addr >= vma->vm_end) 1578 return VM_FAULT_SIGBUS; 1579 1580 track_pfn_insert(vma, &pgprot, pfn); 1581 1582 ptl = pud_lock(vma->vm_mm, vmf->pud); 1583 insert_pfn_pud(vma, addr, vmf->pud, pfn, write); 1584 spin_unlock(ptl); 1585 1586 return VM_FAULT_NOPAGE; 1587 } 1588 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud); 1589 1590 /** 1591 * vmf_insert_folio_pud - insert a pud size folio mapped by a pud entry 1592 * @vmf: Structure describing the fault 1593 * @folio: folio to insert 1594 * @write: whether it's a write fault 1595 * 1596 * Return: vm_fault_t value. 1597 */ 1598 vm_fault_t vmf_insert_folio_pud(struct vm_fault *vmf, struct folio *folio, 1599 bool write) 1600 { 1601 struct vm_area_struct *vma = vmf->vma; 1602 unsigned long addr = vmf->address & PUD_MASK; 1603 pud_t *pud = vmf->pud; 1604 struct mm_struct *mm = vma->vm_mm; 1605 spinlock_t *ptl; 1606 1607 if (addr < vma->vm_start || addr >= vma->vm_end) 1608 return VM_FAULT_SIGBUS; 1609 1610 if (WARN_ON_ONCE(folio_order(folio) != PUD_ORDER)) 1611 return VM_FAULT_SIGBUS; 1612 1613 ptl = pud_lock(mm, pud); 1614 1615 /* 1616 * If there is already an entry present we assume the folio is 1617 * already mapped, hence no need to take another reference. We 1618 * still call insert_pfn_pud() though in case the mapping needs 1619 * upgrading to writeable. 1620 */ 1621 if (pud_none(*vmf->pud)) { 1622 folio_get(folio); 1623 folio_add_file_rmap_pud(folio, &folio->page, vma); 1624 add_mm_counter(mm, mm_counter_file(folio), HPAGE_PUD_NR); 1625 } 1626 insert_pfn_pud(vma, addr, vmf->pud, pfn_to_pfn_t(folio_pfn(folio)), 1627 write); 1628 spin_unlock(ptl); 1629 1630 return VM_FAULT_NOPAGE; 1631 } 1632 EXPORT_SYMBOL_GPL(vmf_insert_folio_pud); 1633 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1634 1635 void touch_pmd(struct vm_area_struct *vma, unsigned long addr, 1636 pmd_t *pmd, bool write) 1637 { 1638 pmd_t _pmd; 1639 1640 _pmd = pmd_mkyoung(*pmd); 1641 if (write) 1642 _pmd = pmd_mkdirty(_pmd); 1643 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, 1644 pmd, _pmd, write)) 1645 update_mmu_cache_pmd(vma, addr, pmd); 1646 } 1647 1648 struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, 1649 pmd_t *pmd, int flags, struct dev_pagemap **pgmap) 1650 { 1651 unsigned long pfn = pmd_pfn(*pmd); 1652 struct mm_struct *mm = vma->vm_mm; 1653 struct page *page; 1654 int ret; 1655 1656 assert_spin_locked(pmd_lockptr(mm, pmd)); 1657 1658 if (flags & FOLL_WRITE && !pmd_write(*pmd)) 1659 return NULL; 1660 1661 if (pmd_present(*pmd) && pmd_devmap(*pmd)) 1662 /* pass */; 1663 else 1664 return NULL; 1665 1666 if (flags & FOLL_TOUCH) 1667 touch_pmd(vma, addr, pmd, flags & FOLL_WRITE); 1668 1669 /* 1670 * device mapped pages can only be returned if the 1671 * caller will manage the page reference count. 1672 */ 1673 if (!(flags & (FOLL_GET | FOLL_PIN))) 1674 return ERR_PTR(-EEXIST); 1675 1676 pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT; 1677 *pgmap = get_dev_pagemap(pfn, *pgmap); 1678 if (!*pgmap) 1679 return ERR_PTR(-EFAULT); 1680 page = pfn_to_page(pfn); 1681 ret = try_grab_folio(page_folio(page), 1, flags); 1682 if (ret) 1683 page = ERR_PTR(ret); 1684 1685 return page; 1686 } 1687 1688 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1689 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 1690 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) 1691 { 1692 spinlock_t *dst_ptl, *src_ptl; 1693 struct page *src_page; 1694 struct folio *src_folio; 1695 pmd_t pmd; 1696 pgtable_t pgtable = NULL; 1697 int ret = -ENOMEM; 1698 1699 pmd = pmdp_get_lockless(src_pmd); 1700 if (unlikely(pmd_present(pmd) && pmd_special(pmd))) { 1701 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1702 src_ptl = pmd_lockptr(src_mm, src_pmd); 1703 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1704 /* 1705 * No need to recheck the pmd, it can't change with write 1706 * mmap lock held here. 1707 * 1708 * Meanwhile, making sure it's not a CoW VMA with writable 1709 * mapping, otherwise it means either the anon page wrongly 1710 * applied special bit, or we made the PRIVATE mapping be 1711 * able to wrongly write to the backend MMIO. 1712 */ 1713 VM_WARN_ON_ONCE(is_cow_mapping(src_vma->vm_flags) && pmd_write(pmd)); 1714 goto set_pmd; 1715 } 1716 1717 /* Skip if can be re-fill on fault */ 1718 if (!vma_is_anonymous(dst_vma)) 1719 return 0; 1720 1721 pgtable = pte_alloc_one(dst_mm); 1722 if (unlikely(!pgtable)) 1723 goto out; 1724 1725 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1726 src_ptl = pmd_lockptr(src_mm, src_pmd); 1727 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1728 1729 ret = -EAGAIN; 1730 pmd = *src_pmd; 1731 1732 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 1733 if (unlikely(is_swap_pmd(pmd))) { 1734 swp_entry_t entry = pmd_to_swp_entry(pmd); 1735 1736 VM_BUG_ON(!is_pmd_migration_entry(pmd)); 1737 if (!is_readable_migration_entry(entry)) { 1738 entry = make_readable_migration_entry( 1739 swp_offset(entry)); 1740 pmd = swp_entry_to_pmd(entry); 1741 if (pmd_swp_soft_dirty(*src_pmd)) 1742 pmd = pmd_swp_mksoft_dirty(pmd); 1743 if (pmd_swp_uffd_wp(*src_pmd)) 1744 pmd = pmd_swp_mkuffd_wp(pmd); 1745 set_pmd_at(src_mm, addr, src_pmd, pmd); 1746 } 1747 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1748 mm_inc_nr_ptes(dst_mm); 1749 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1750 if (!userfaultfd_wp(dst_vma)) 1751 pmd = pmd_swp_clear_uffd_wp(pmd); 1752 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1753 ret = 0; 1754 goto out_unlock; 1755 } 1756 #endif 1757 1758 if (unlikely(!pmd_trans_huge(pmd))) { 1759 pte_free(dst_mm, pgtable); 1760 goto out_unlock; 1761 } 1762 /* 1763 * When page table lock is held, the huge zero pmd should not be 1764 * under splitting since we don't split the page itself, only pmd to 1765 * a page table. 1766 */ 1767 if (is_huge_zero_pmd(pmd)) { 1768 /* 1769 * mm_get_huge_zero_folio() will never allocate a new 1770 * folio here, since we already have a zero page to 1771 * copy. It just takes a reference. 1772 */ 1773 mm_get_huge_zero_folio(dst_mm); 1774 goto out_zero_page; 1775 } 1776 1777 src_page = pmd_page(pmd); 1778 VM_BUG_ON_PAGE(!PageHead(src_page), src_page); 1779 src_folio = page_folio(src_page); 1780 1781 folio_get(src_folio); 1782 if (unlikely(folio_try_dup_anon_rmap_pmd(src_folio, src_page, dst_vma, src_vma))) { 1783 /* Page maybe pinned: split and retry the fault on PTEs. */ 1784 folio_put(src_folio); 1785 pte_free(dst_mm, pgtable); 1786 spin_unlock(src_ptl); 1787 spin_unlock(dst_ptl); 1788 __split_huge_pmd(src_vma, src_pmd, addr, false, NULL); 1789 return -EAGAIN; 1790 } 1791 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1792 out_zero_page: 1793 mm_inc_nr_ptes(dst_mm); 1794 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1795 pmdp_set_wrprotect(src_mm, addr, src_pmd); 1796 if (!userfaultfd_wp(dst_vma)) 1797 pmd = pmd_clear_uffd_wp(pmd); 1798 pmd = pmd_wrprotect(pmd); 1799 set_pmd: 1800 pmd = pmd_mkold(pmd); 1801 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1802 1803 ret = 0; 1804 out_unlock: 1805 spin_unlock(src_ptl); 1806 spin_unlock(dst_ptl); 1807 out: 1808 return ret; 1809 } 1810 1811 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1812 void touch_pud(struct vm_area_struct *vma, unsigned long addr, 1813 pud_t *pud, bool write) 1814 { 1815 pud_t _pud; 1816 1817 _pud = pud_mkyoung(*pud); 1818 if (write) 1819 _pud = pud_mkdirty(_pud); 1820 if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK, 1821 pud, _pud, write)) 1822 update_mmu_cache_pud(vma, addr, pud); 1823 } 1824 1825 int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1826 pud_t *dst_pud, pud_t *src_pud, unsigned long addr, 1827 struct vm_area_struct *vma) 1828 { 1829 spinlock_t *dst_ptl, *src_ptl; 1830 pud_t pud; 1831 int ret; 1832 1833 dst_ptl = pud_lock(dst_mm, dst_pud); 1834 src_ptl = pud_lockptr(src_mm, src_pud); 1835 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1836 1837 ret = -EAGAIN; 1838 pud = *src_pud; 1839 if (unlikely(!pud_trans_huge(pud) && !pud_devmap(pud))) 1840 goto out_unlock; 1841 1842 /* 1843 * TODO: once we support anonymous pages, use 1844 * folio_try_dup_anon_rmap_*() and split if duplicating fails. 1845 */ 1846 if (is_cow_mapping(vma->vm_flags) && pud_write(pud)) { 1847 pudp_set_wrprotect(src_mm, addr, src_pud); 1848 pud = pud_wrprotect(pud); 1849 } 1850 pud = pud_mkold(pud); 1851 set_pud_at(dst_mm, addr, dst_pud, pud); 1852 1853 ret = 0; 1854 out_unlock: 1855 spin_unlock(src_ptl); 1856 spin_unlock(dst_ptl); 1857 return ret; 1858 } 1859 1860 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud) 1861 { 1862 bool write = vmf->flags & FAULT_FLAG_WRITE; 1863 1864 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); 1865 if (unlikely(!pud_same(*vmf->pud, orig_pud))) 1866 goto unlock; 1867 1868 touch_pud(vmf->vma, vmf->address, vmf->pud, write); 1869 unlock: 1870 spin_unlock(vmf->ptl); 1871 } 1872 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1873 1874 void huge_pmd_set_accessed(struct vm_fault *vmf) 1875 { 1876 bool write = vmf->flags & FAULT_FLAG_WRITE; 1877 1878 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); 1879 if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd))) 1880 goto unlock; 1881 1882 touch_pmd(vmf->vma, vmf->address, vmf->pmd, write); 1883 1884 unlock: 1885 spin_unlock(vmf->ptl); 1886 } 1887 1888 static vm_fault_t do_huge_zero_wp_pmd(struct vm_fault *vmf) 1889 { 1890 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1891 struct vm_area_struct *vma = vmf->vma; 1892 struct mmu_notifier_range range; 1893 struct folio *folio; 1894 vm_fault_t ret = 0; 1895 1896 folio = vma_alloc_anon_folio_pmd(vma, vmf->address); 1897 if (unlikely(!folio)) 1898 return VM_FAULT_FALLBACK; 1899 1900 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, haddr, 1901 haddr + HPAGE_PMD_SIZE); 1902 mmu_notifier_invalidate_range_start(&range); 1903 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1904 if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) 1905 goto release; 1906 ret = check_stable_address_space(vma->vm_mm); 1907 if (ret) 1908 goto release; 1909 (void)pmdp_huge_clear_flush(vma, haddr, vmf->pmd); 1910 map_anon_folio_pmd(folio, vmf->pmd, vma, haddr); 1911 goto unlock; 1912 release: 1913 folio_put(folio); 1914 unlock: 1915 spin_unlock(vmf->ptl); 1916 mmu_notifier_invalidate_range_end(&range); 1917 return ret; 1918 } 1919 1920 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) 1921 { 1922 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; 1923 struct vm_area_struct *vma = vmf->vma; 1924 struct folio *folio; 1925 struct page *page; 1926 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1927 pmd_t orig_pmd = vmf->orig_pmd; 1928 1929 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); 1930 VM_BUG_ON_VMA(!vma->anon_vma, vma); 1931 1932 if (is_huge_zero_pmd(orig_pmd)) { 1933 vm_fault_t ret = do_huge_zero_wp_pmd(vmf); 1934 1935 if (!(ret & VM_FAULT_FALLBACK)) 1936 return ret; 1937 1938 /* Fallback to splitting PMD if THP cannot be allocated */ 1939 goto fallback; 1940 } 1941 1942 spin_lock(vmf->ptl); 1943 1944 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1945 spin_unlock(vmf->ptl); 1946 return 0; 1947 } 1948 1949 page = pmd_page(orig_pmd); 1950 folio = page_folio(page); 1951 VM_BUG_ON_PAGE(!PageHead(page), page); 1952 1953 /* Early check when only holding the PT lock. */ 1954 if (PageAnonExclusive(page)) 1955 goto reuse; 1956 1957 if (!folio_trylock(folio)) { 1958 folio_get(folio); 1959 spin_unlock(vmf->ptl); 1960 folio_lock(folio); 1961 spin_lock(vmf->ptl); 1962 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1963 spin_unlock(vmf->ptl); 1964 folio_unlock(folio); 1965 folio_put(folio); 1966 return 0; 1967 } 1968 folio_put(folio); 1969 } 1970 1971 /* Recheck after temporarily dropping the PT lock. */ 1972 if (PageAnonExclusive(page)) { 1973 folio_unlock(folio); 1974 goto reuse; 1975 } 1976 1977 /* 1978 * See do_wp_page(): we can only reuse the folio exclusively if 1979 * there are no additional references. Note that we always drain 1980 * the LRU cache immediately after adding a THP. 1981 */ 1982 if (folio_ref_count(folio) > 1983 1 + folio_test_swapcache(folio) * folio_nr_pages(folio)) 1984 goto unlock_fallback; 1985 if (folio_test_swapcache(folio)) 1986 folio_free_swap(folio); 1987 if (folio_ref_count(folio) == 1) { 1988 pmd_t entry; 1989 1990 folio_move_anon_rmap(folio, vma); 1991 SetPageAnonExclusive(page); 1992 folio_unlock(folio); 1993 reuse: 1994 if (unlikely(unshare)) { 1995 spin_unlock(vmf->ptl); 1996 return 0; 1997 } 1998 entry = pmd_mkyoung(orig_pmd); 1999 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 2000 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) 2001 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 2002 spin_unlock(vmf->ptl); 2003 return 0; 2004 } 2005 2006 unlock_fallback: 2007 folio_unlock(folio); 2008 spin_unlock(vmf->ptl); 2009 fallback: 2010 __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL); 2011 return VM_FAULT_FALLBACK; 2012 } 2013 2014 static inline bool can_change_pmd_writable(struct vm_area_struct *vma, 2015 unsigned long addr, pmd_t pmd) 2016 { 2017 struct page *page; 2018 2019 if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) 2020 return false; 2021 2022 /* Don't touch entries that are not even readable (NUMA hinting). */ 2023 if (pmd_protnone(pmd)) 2024 return false; 2025 2026 /* Do we need write faults for softdirty tracking? */ 2027 if (pmd_needs_soft_dirty_wp(vma, pmd)) 2028 return false; 2029 2030 /* Do we need write faults for uffd-wp tracking? */ 2031 if (userfaultfd_huge_pmd_wp(vma, pmd)) 2032 return false; 2033 2034 if (!(vma->vm_flags & VM_SHARED)) { 2035 /* See can_change_pte_writable(). */ 2036 page = vm_normal_page_pmd(vma, addr, pmd); 2037 return page && PageAnon(page) && PageAnonExclusive(page); 2038 } 2039 2040 /* See can_change_pte_writable(). */ 2041 return pmd_dirty(pmd); 2042 } 2043 2044 /* NUMA hinting page fault entry point for trans huge pmds */ 2045 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) 2046 { 2047 struct vm_area_struct *vma = vmf->vma; 2048 struct folio *folio; 2049 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 2050 int nid = NUMA_NO_NODE; 2051 int target_nid, last_cpupid; 2052 pmd_t pmd, old_pmd; 2053 bool writable = false; 2054 int flags = 0; 2055 2056 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 2057 old_pmd = pmdp_get(vmf->pmd); 2058 2059 if (unlikely(!pmd_same(old_pmd, vmf->orig_pmd))) { 2060 spin_unlock(vmf->ptl); 2061 return 0; 2062 } 2063 2064 pmd = pmd_modify(old_pmd, vma->vm_page_prot); 2065 2066 /* 2067 * Detect now whether the PMD could be writable; this information 2068 * is only valid while holding the PT lock. 2069 */ 2070 writable = pmd_write(pmd); 2071 if (!writable && vma_wants_manual_pte_write_upgrade(vma) && 2072 can_change_pmd_writable(vma, vmf->address, pmd)) 2073 writable = true; 2074 2075 folio = vm_normal_folio_pmd(vma, haddr, pmd); 2076 if (!folio) 2077 goto out_map; 2078 2079 nid = folio_nid(folio); 2080 2081 target_nid = numa_migrate_check(folio, vmf, haddr, &flags, writable, 2082 &last_cpupid); 2083 if (target_nid == NUMA_NO_NODE) 2084 goto out_map; 2085 if (migrate_misplaced_folio_prepare(folio, vma, target_nid)) { 2086 flags |= TNF_MIGRATE_FAIL; 2087 goto out_map; 2088 } 2089 /* The folio is isolated and isolation code holds a folio reference. */ 2090 spin_unlock(vmf->ptl); 2091 writable = false; 2092 2093 if (!migrate_misplaced_folio(folio, target_nid)) { 2094 flags |= TNF_MIGRATED; 2095 nid = target_nid; 2096 task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags); 2097 return 0; 2098 } 2099 2100 flags |= TNF_MIGRATE_FAIL; 2101 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 2102 if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) { 2103 spin_unlock(vmf->ptl); 2104 return 0; 2105 } 2106 out_map: 2107 /* Restore the PMD */ 2108 pmd = pmd_modify(pmdp_get(vmf->pmd), vma->vm_page_prot); 2109 pmd = pmd_mkyoung(pmd); 2110 if (writable) 2111 pmd = pmd_mkwrite(pmd, vma); 2112 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); 2113 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 2114 spin_unlock(vmf->ptl); 2115 2116 if (nid != NUMA_NO_NODE) 2117 task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags); 2118 return 0; 2119 } 2120 2121 /* 2122 * Return true if we do MADV_FREE successfully on entire pmd page. 2123 * Otherwise, return false. 2124 */ 2125 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2126 pmd_t *pmd, unsigned long addr, unsigned long next) 2127 { 2128 spinlock_t *ptl; 2129 pmd_t orig_pmd; 2130 struct folio *folio; 2131 struct mm_struct *mm = tlb->mm; 2132 bool ret = false; 2133 2134 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2135 2136 ptl = pmd_trans_huge_lock(pmd, vma); 2137 if (!ptl) 2138 goto out_unlocked; 2139 2140 orig_pmd = *pmd; 2141 if (is_huge_zero_pmd(orig_pmd)) 2142 goto out; 2143 2144 if (unlikely(!pmd_present(orig_pmd))) { 2145 VM_BUG_ON(thp_migration_supported() && 2146 !is_pmd_migration_entry(orig_pmd)); 2147 goto out; 2148 } 2149 2150 folio = pmd_folio(orig_pmd); 2151 /* 2152 * If other processes are mapping this folio, we couldn't discard 2153 * the folio unless they all do MADV_FREE so let's skip the folio. 2154 */ 2155 if (folio_maybe_mapped_shared(folio)) 2156 goto out; 2157 2158 if (!folio_trylock(folio)) 2159 goto out; 2160 2161 /* 2162 * If user want to discard part-pages of THP, split it so MADV_FREE 2163 * will deactivate only them. 2164 */ 2165 if (next - addr != HPAGE_PMD_SIZE) { 2166 folio_get(folio); 2167 spin_unlock(ptl); 2168 split_folio(folio); 2169 folio_unlock(folio); 2170 folio_put(folio); 2171 goto out_unlocked; 2172 } 2173 2174 if (folio_test_dirty(folio)) 2175 folio_clear_dirty(folio); 2176 folio_unlock(folio); 2177 2178 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) { 2179 pmdp_invalidate(vma, addr, pmd); 2180 orig_pmd = pmd_mkold(orig_pmd); 2181 orig_pmd = pmd_mkclean(orig_pmd); 2182 2183 set_pmd_at(mm, addr, pmd, orig_pmd); 2184 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 2185 } 2186 2187 folio_mark_lazyfree(folio); 2188 ret = true; 2189 out: 2190 spin_unlock(ptl); 2191 out_unlocked: 2192 return ret; 2193 } 2194 2195 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd) 2196 { 2197 pgtable_t pgtable; 2198 2199 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2200 pte_free(mm, pgtable); 2201 mm_dec_nr_ptes(mm); 2202 } 2203 2204 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2205 pmd_t *pmd, unsigned long addr) 2206 { 2207 pmd_t orig_pmd; 2208 spinlock_t *ptl; 2209 2210 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2211 2212 ptl = __pmd_trans_huge_lock(pmd, vma); 2213 if (!ptl) 2214 return 0; 2215 /* 2216 * For architectures like ppc64 we look at deposited pgtable 2217 * when calling pmdp_huge_get_and_clear. So do the 2218 * pgtable_trans_huge_withdraw after finishing pmdp related 2219 * operations. 2220 */ 2221 orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd, 2222 tlb->fullmm); 2223 arch_check_zapped_pmd(vma, orig_pmd); 2224 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 2225 if (!vma_is_dax(vma) && vma_is_special_huge(vma)) { 2226 if (arch_needs_pgtable_deposit()) 2227 zap_deposited_table(tlb->mm, pmd); 2228 spin_unlock(ptl); 2229 } else if (is_huge_zero_pmd(orig_pmd)) { 2230 if (!vma_is_dax(vma) || arch_needs_pgtable_deposit()) 2231 zap_deposited_table(tlb->mm, pmd); 2232 spin_unlock(ptl); 2233 } else { 2234 struct folio *folio = NULL; 2235 int flush_needed = 1; 2236 2237 if (pmd_present(orig_pmd)) { 2238 struct page *page = pmd_page(orig_pmd); 2239 2240 folio = page_folio(page); 2241 folio_remove_rmap_pmd(folio, page, vma); 2242 WARN_ON_ONCE(folio_mapcount(folio) < 0); 2243 VM_BUG_ON_PAGE(!PageHead(page), page); 2244 } else if (thp_migration_supported()) { 2245 swp_entry_t entry; 2246 2247 VM_BUG_ON(!is_pmd_migration_entry(orig_pmd)); 2248 entry = pmd_to_swp_entry(orig_pmd); 2249 folio = pfn_swap_entry_folio(entry); 2250 flush_needed = 0; 2251 } else 2252 WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!"); 2253 2254 if (folio_test_anon(folio)) { 2255 zap_deposited_table(tlb->mm, pmd); 2256 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); 2257 } else { 2258 if (arch_needs_pgtable_deposit()) 2259 zap_deposited_table(tlb->mm, pmd); 2260 add_mm_counter(tlb->mm, mm_counter_file(folio), 2261 -HPAGE_PMD_NR); 2262 2263 /* 2264 * Use flush_needed to indicate whether the PMD entry 2265 * is present, instead of checking pmd_present() again. 2266 */ 2267 if (flush_needed && pmd_young(orig_pmd) && 2268 likely(vma_has_recency(vma))) 2269 folio_mark_accessed(folio); 2270 } 2271 2272 spin_unlock(ptl); 2273 if (flush_needed) 2274 tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE); 2275 } 2276 return 1; 2277 } 2278 2279 #ifndef pmd_move_must_withdraw 2280 static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl, 2281 spinlock_t *old_pmd_ptl, 2282 struct vm_area_struct *vma) 2283 { 2284 /* 2285 * With split pmd lock we also need to move preallocated 2286 * PTE page table if new_pmd is on different PMD page table. 2287 * 2288 * We also don't deposit and withdraw tables for file pages. 2289 */ 2290 return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma); 2291 } 2292 #endif 2293 2294 static pmd_t move_soft_dirty_pmd(pmd_t pmd) 2295 { 2296 #ifdef CONFIG_MEM_SOFT_DIRTY 2297 if (unlikely(is_pmd_migration_entry(pmd))) 2298 pmd = pmd_swp_mksoft_dirty(pmd); 2299 else if (pmd_present(pmd)) 2300 pmd = pmd_mksoft_dirty(pmd); 2301 #endif 2302 return pmd; 2303 } 2304 2305 static pmd_t clear_uffd_wp_pmd(pmd_t pmd) 2306 { 2307 if (pmd_present(pmd)) 2308 pmd = pmd_clear_uffd_wp(pmd); 2309 else if (is_swap_pmd(pmd)) 2310 pmd = pmd_swp_clear_uffd_wp(pmd); 2311 2312 return pmd; 2313 } 2314 2315 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, 2316 unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd) 2317 { 2318 spinlock_t *old_ptl, *new_ptl; 2319 pmd_t pmd; 2320 struct mm_struct *mm = vma->vm_mm; 2321 bool force_flush = false; 2322 2323 /* 2324 * The destination pmd shouldn't be established, free_pgtables() 2325 * should have released it; but move_page_tables() might have already 2326 * inserted a page table, if racing against shmem/file collapse. 2327 */ 2328 if (!pmd_none(*new_pmd)) { 2329 VM_BUG_ON(pmd_trans_huge(*new_pmd)); 2330 return false; 2331 } 2332 2333 /* 2334 * We don't have to worry about the ordering of src and dst 2335 * ptlocks because exclusive mmap_lock prevents deadlock. 2336 */ 2337 old_ptl = __pmd_trans_huge_lock(old_pmd, vma); 2338 if (old_ptl) { 2339 new_ptl = pmd_lockptr(mm, new_pmd); 2340 if (new_ptl != old_ptl) 2341 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 2342 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); 2343 if (pmd_present(pmd)) 2344 force_flush = true; 2345 VM_BUG_ON(!pmd_none(*new_pmd)); 2346 2347 if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) { 2348 pgtable_t pgtable; 2349 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); 2350 pgtable_trans_huge_deposit(mm, new_pmd, pgtable); 2351 } 2352 pmd = move_soft_dirty_pmd(pmd); 2353 if (vma_has_uffd_without_event_remap(vma)) 2354 pmd = clear_uffd_wp_pmd(pmd); 2355 set_pmd_at(mm, new_addr, new_pmd, pmd); 2356 if (force_flush) 2357 flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE); 2358 if (new_ptl != old_ptl) 2359 spin_unlock(new_ptl); 2360 spin_unlock(old_ptl); 2361 return true; 2362 } 2363 return false; 2364 } 2365 2366 /* 2367 * Returns 2368 * - 0 if PMD could not be locked 2369 * - 1 if PMD was locked but protections unchanged and TLB flush unnecessary 2370 * or if prot_numa but THP migration is not supported 2371 * - HPAGE_PMD_NR if protections changed and TLB flush necessary 2372 */ 2373 int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2374 pmd_t *pmd, unsigned long addr, pgprot_t newprot, 2375 unsigned long cp_flags) 2376 { 2377 struct mm_struct *mm = vma->vm_mm; 2378 spinlock_t *ptl; 2379 pmd_t oldpmd, entry; 2380 bool prot_numa = cp_flags & MM_CP_PROT_NUMA; 2381 bool uffd_wp = cp_flags & MM_CP_UFFD_WP; 2382 bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; 2383 int ret = 1; 2384 2385 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2386 2387 if (prot_numa && !thp_migration_supported()) 2388 return 1; 2389 2390 ptl = __pmd_trans_huge_lock(pmd, vma); 2391 if (!ptl) 2392 return 0; 2393 2394 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 2395 if (is_swap_pmd(*pmd)) { 2396 swp_entry_t entry = pmd_to_swp_entry(*pmd); 2397 struct folio *folio = pfn_swap_entry_folio(entry); 2398 pmd_t newpmd; 2399 2400 VM_BUG_ON(!is_pmd_migration_entry(*pmd)); 2401 if (is_writable_migration_entry(entry)) { 2402 /* 2403 * A protection check is difficult so 2404 * just be safe and disable write 2405 */ 2406 if (folio_test_anon(folio)) 2407 entry = make_readable_exclusive_migration_entry(swp_offset(entry)); 2408 else 2409 entry = make_readable_migration_entry(swp_offset(entry)); 2410 newpmd = swp_entry_to_pmd(entry); 2411 if (pmd_swp_soft_dirty(*pmd)) 2412 newpmd = pmd_swp_mksoft_dirty(newpmd); 2413 } else { 2414 newpmd = *pmd; 2415 } 2416 2417 if (uffd_wp) 2418 newpmd = pmd_swp_mkuffd_wp(newpmd); 2419 else if (uffd_wp_resolve) 2420 newpmd = pmd_swp_clear_uffd_wp(newpmd); 2421 if (!pmd_same(*pmd, newpmd)) 2422 set_pmd_at(mm, addr, pmd, newpmd); 2423 goto unlock; 2424 } 2425 #endif 2426 2427 if (prot_numa) { 2428 struct folio *folio; 2429 bool toptier; 2430 /* 2431 * Avoid trapping faults against the zero page. The read-only 2432 * data is likely to be read-cached on the local CPU and 2433 * local/remote hits to the zero page are not interesting. 2434 */ 2435 if (is_huge_zero_pmd(*pmd)) 2436 goto unlock; 2437 2438 if (pmd_protnone(*pmd)) 2439 goto unlock; 2440 2441 folio = pmd_folio(*pmd); 2442 toptier = node_is_toptier(folio_nid(folio)); 2443 /* 2444 * Skip scanning top tier node if normal numa 2445 * balancing is disabled 2446 */ 2447 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) && 2448 toptier) 2449 goto unlock; 2450 2451 if (folio_use_access_time(folio)) 2452 folio_xchg_access_time(folio, 2453 jiffies_to_msecs(jiffies)); 2454 } 2455 /* 2456 * In case prot_numa, we are under mmap_read_lock(mm). It's critical 2457 * to not clear pmd intermittently to avoid race with MADV_DONTNEED 2458 * which is also under mmap_read_lock(mm): 2459 * 2460 * CPU0: CPU1: 2461 * change_huge_pmd(prot_numa=1) 2462 * pmdp_huge_get_and_clear_notify() 2463 * madvise_dontneed() 2464 * zap_pmd_range() 2465 * pmd_trans_huge(*pmd) == 0 (without ptl) 2466 * // skip the pmd 2467 * set_pmd_at(); 2468 * // pmd is re-established 2469 * 2470 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it 2471 * which may break userspace. 2472 * 2473 * pmdp_invalidate_ad() is required to make sure we don't miss 2474 * dirty/young flags set by hardware. 2475 */ 2476 oldpmd = pmdp_invalidate_ad(vma, addr, pmd); 2477 2478 entry = pmd_modify(oldpmd, newprot); 2479 if (uffd_wp) 2480 entry = pmd_mkuffd_wp(entry); 2481 else if (uffd_wp_resolve) 2482 /* 2483 * Leave the write bit to be handled by PF interrupt 2484 * handler, then things like COW could be properly 2485 * handled. 2486 */ 2487 entry = pmd_clear_uffd_wp(entry); 2488 2489 /* See change_pte_range(). */ 2490 if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) && 2491 can_change_pmd_writable(vma, addr, entry)) 2492 entry = pmd_mkwrite(entry, vma); 2493 2494 ret = HPAGE_PMD_NR; 2495 set_pmd_at(mm, addr, pmd, entry); 2496 2497 if (huge_pmd_needs_flush(oldpmd, entry)) 2498 tlb_flush_pmd_range(tlb, addr, HPAGE_PMD_SIZE); 2499 unlock: 2500 spin_unlock(ptl); 2501 return ret; 2502 } 2503 2504 /* 2505 * Returns: 2506 * 2507 * - 0: if pud leaf changed from under us 2508 * - 1: if pud can be skipped 2509 * - HPAGE_PUD_NR: if pud was successfully processed 2510 */ 2511 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2512 int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 2513 pud_t *pudp, unsigned long addr, pgprot_t newprot, 2514 unsigned long cp_flags) 2515 { 2516 struct mm_struct *mm = vma->vm_mm; 2517 pud_t oldpud, entry; 2518 spinlock_t *ptl; 2519 2520 tlb_change_page_size(tlb, HPAGE_PUD_SIZE); 2521 2522 /* NUMA balancing doesn't apply to dax */ 2523 if (cp_flags & MM_CP_PROT_NUMA) 2524 return 1; 2525 2526 /* 2527 * Huge entries on userfault-wp only works with anonymous, while we 2528 * don't have anonymous PUDs yet. 2529 */ 2530 if (WARN_ON_ONCE(cp_flags & MM_CP_UFFD_WP_ALL)) 2531 return 1; 2532 2533 ptl = __pud_trans_huge_lock(pudp, vma); 2534 if (!ptl) 2535 return 0; 2536 2537 /* 2538 * Can't clear PUD or it can race with concurrent zapping. See 2539 * change_huge_pmd(). 2540 */ 2541 oldpud = pudp_invalidate(vma, addr, pudp); 2542 entry = pud_modify(oldpud, newprot); 2543 set_pud_at(mm, addr, pudp, entry); 2544 tlb_flush_pud_range(tlb, addr, HPAGE_PUD_SIZE); 2545 2546 spin_unlock(ptl); 2547 return HPAGE_PUD_NR; 2548 } 2549 #endif 2550 2551 #ifdef CONFIG_USERFAULTFD 2552 /* 2553 * The PT lock for src_pmd and dst_vma/src_vma (for reading) are locked by 2554 * the caller, but it must return after releasing the page_table_lock. 2555 * Just move the page from src_pmd to dst_pmd if possible. 2556 * Return zero if succeeded in moving the page, -EAGAIN if it needs to be 2557 * repeated by the caller, or other errors in case of failure. 2558 */ 2559 int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval, 2560 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, 2561 unsigned long dst_addr, unsigned long src_addr) 2562 { 2563 pmd_t _dst_pmd, src_pmdval; 2564 struct page *src_page; 2565 struct folio *src_folio; 2566 struct anon_vma *src_anon_vma; 2567 spinlock_t *src_ptl, *dst_ptl; 2568 pgtable_t src_pgtable; 2569 struct mmu_notifier_range range; 2570 int err = 0; 2571 2572 src_pmdval = *src_pmd; 2573 src_ptl = pmd_lockptr(mm, src_pmd); 2574 2575 lockdep_assert_held(src_ptl); 2576 vma_assert_locked(src_vma); 2577 vma_assert_locked(dst_vma); 2578 2579 /* Sanity checks before the operation */ 2580 if (WARN_ON_ONCE(!pmd_none(dst_pmdval)) || WARN_ON_ONCE(src_addr & ~HPAGE_PMD_MASK) || 2581 WARN_ON_ONCE(dst_addr & ~HPAGE_PMD_MASK)) { 2582 spin_unlock(src_ptl); 2583 return -EINVAL; 2584 } 2585 2586 if (!pmd_trans_huge(src_pmdval)) { 2587 spin_unlock(src_ptl); 2588 if (is_pmd_migration_entry(src_pmdval)) { 2589 pmd_migration_entry_wait(mm, &src_pmdval); 2590 return -EAGAIN; 2591 } 2592 return -ENOENT; 2593 } 2594 2595 src_page = pmd_page(src_pmdval); 2596 2597 if (!is_huge_zero_pmd(src_pmdval)) { 2598 if (unlikely(!PageAnonExclusive(src_page))) { 2599 spin_unlock(src_ptl); 2600 return -EBUSY; 2601 } 2602 2603 src_folio = page_folio(src_page); 2604 folio_get(src_folio); 2605 } else 2606 src_folio = NULL; 2607 2608 spin_unlock(src_ptl); 2609 2610 flush_cache_range(src_vma, src_addr, src_addr + HPAGE_PMD_SIZE); 2611 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, src_addr, 2612 src_addr + HPAGE_PMD_SIZE); 2613 mmu_notifier_invalidate_range_start(&range); 2614 2615 if (src_folio) { 2616 folio_lock(src_folio); 2617 2618 /* 2619 * split_huge_page walks the anon_vma chain without the page 2620 * lock. Serialize against it with the anon_vma lock, the page 2621 * lock is not enough. 2622 */ 2623 src_anon_vma = folio_get_anon_vma(src_folio); 2624 if (!src_anon_vma) { 2625 err = -EAGAIN; 2626 goto unlock_folio; 2627 } 2628 anon_vma_lock_write(src_anon_vma); 2629 } else 2630 src_anon_vma = NULL; 2631 2632 dst_ptl = pmd_lockptr(mm, dst_pmd); 2633 double_pt_lock(src_ptl, dst_ptl); 2634 if (unlikely(!pmd_same(*src_pmd, src_pmdval) || 2635 !pmd_same(*dst_pmd, dst_pmdval))) { 2636 err = -EAGAIN; 2637 goto unlock_ptls; 2638 } 2639 if (src_folio) { 2640 if (folio_maybe_dma_pinned(src_folio) || 2641 !PageAnonExclusive(&src_folio->page)) { 2642 err = -EBUSY; 2643 goto unlock_ptls; 2644 } 2645 2646 if (WARN_ON_ONCE(!folio_test_head(src_folio)) || 2647 WARN_ON_ONCE(!folio_test_anon(src_folio))) { 2648 err = -EBUSY; 2649 goto unlock_ptls; 2650 } 2651 2652 src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); 2653 /* Folio got pinned from under us. Put it back and fail the move. */ 2654 if (folio_maybe_dma_pinned(src_folio)) { 2655 set_pmd_at(mm, src_addr, src_pmd, src_pmdval); 2656 err = -EBUSY; 2657 goto unlock_ptls; 2658 } 2659 2660 folio_move_anon_rmap(src_folio, dst_vma); 2661 src_folio->index = linear_page_index(dst_vma, dst_addr); 2662 2663 _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); 2664 /* Follow mremap() behavior and treat the entry dirty after the move */ 2665 _dst_pmd = pmd_mkwrite(pmd_mkdirty(_dst_pmd), dst_vma); 2666 } else { 2667 src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); 2668 _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); 2669 } 2670 set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd); 2671 2672 src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd); 2673 pgtable_trans_huge_deposit(mm, dst_pmd, src_pgtable); 2674 unlock_ptls: 2675 double_pt_unlock(src_ptl, dst_ptl); 2676 if (src_anon_vma) { 2677 anon_vma_unlock_write(src_anon_vma); 2678 put_anon_vma(src_anon_vma); 2679 } 2680 unlock_folio: 2681 /* unblock rmap walks */ 2682 if (src_folio) 2683 folio_unlock(src_folio); 2684 mmu_notifier_invalidate_range_end(&range); 2685 if (src_folio) 2686 folio_put(src_folio); 2687 return err; 2688 } 2689 #endif /* CONFIG_USERFAULTFD */ 2690 2691 /* 2692 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise. 2693 * 2694 * Note that if it returns page table lock pointer, this routine returns without 2695 * unlocking page table lock. So callers must unlock it. 2696 */ 2697 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) 2698 { 2699 spinlock_t *ptl; 2700 ptl = pmd_lock(vma->vm_mm, pmd); 2701 if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || 2702 pmd_devmap(*pmd))) 2703 return ptl; 2704 spin_unlock(ptl); 2705 return NULL; 2706 } 2707 2708 /* 2709 * Returns page table lock pointer if a given pud maps a thp, NULL otherwise. 2710 * 2711 * Note that if it returns page table lock pointer, this routine returns without 2712 * unlocking page table lock. So callers must unlock it. 2713 */ 2714 spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma) 2715 { 2716 spinlock_t *ptl; 2717 2718 ptl = pud_lock(vma->vm_mm, pud); 2719 if (likely(pud_trans_huge(*pud) || pud_devmap(*pud))) 2720 return ptl; 2721 spin_unlock(ptl); 2722 return NULL; 2723 } 2724 2725 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2726 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 2727 pud_t *pud, unsigned long addr) 2728 { 2729 spinlock_t *ptl; 2730 pud_t orig_pud; 2731 2732 ptl = __pud_trans_huge_lock(pud, vma); 2733 if (!ptl) 2734 return 0; 2735 2736 orig_pud = pudp_huge_get_and_clear_full(vma, addr, pud, tlb->fullmm); 2737 arch_check_zapped_pud(vma, orig_pud); 2738 tlb_remove_pud_tlb_entry(tlb, pud, addr); 2739 if (!vma_is_dax(vma) && vma_is_special_huge(vma)) { 2740 spin_unlock(ptl); 2741 /* No zero page support yet */ 2742 } else { 2743 struct page *page = NULL; 2744 struct folio *folio; 2745 2746 /* No support for anonymous PUD pages or migration yet */ 2747 VM_WARN_ON_ONCE(vma_is_anonymous(vma) || 2748 !pud_present(orig_pud)); 2749 2750 page = pud_page(orig_pud); 2751 folio = page_folio(page); 2752 folio_remove_rmap_pud(folio, page, vma); 2753 add_mm_counter(tlb->mm, mm_counter_file(folio), -HPAGE_PUD_NR); 2754 2755 spin_unlock(ptl); 2756 tlb_remove_page_size(tlb, page, HPAGE_PUD_SIZE); 2757 } 2758 return 1; 2759 } 2760 2761 static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud, 2762 unsigned long haddr) 2763 { 2764 struct folio *folio; 2765 struct page *page; 2766 pud_t old_pud; 2767 2768 VM_BUG_ON(haddr & ~HPAGE_PUD_MASK); 2769 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2770 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma); 2771 VM_BUG_ON(!pud_trans_huge(*pud) && !pud_devmap(*pud)); 2772 2773 count_vm_event(THP_SPLIT_PUD); 2774 2775 old_pud = pudp_huge_clear_flush(vma, haddr, pud); 2776 2777 if (!vma_is_dax(vma)) 2778 return; 2779 2780 page = pud_page(old_pud); 2781 folio = page_folio(page); 2782 2783 if (!folio_test_dirty(folio) && pud_dirty(old_pud)) 2784 folio_mark_dirty(folio); 2785 if (!folio_test_referenced(folio) && pud_young(old_pud)) 2786 folio_set_referenced(folio); 2787 folio_remove_rmap_pud(folio, page, vma); 2788 folio_put(folio); 2789 add_mm_counter(vma->vm_mm, mm_counter_file(folio), 2790 -HPAGE_PUD_NR); 2791 } 2792 2793 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 2794 unsigned long address) 2795 { 2796 spinlock_t *ptl; 2797 struct mmu_notifier_range range; 2798 2799 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 2800 address & HPAGE_PUD_MASK, 2801 (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE); 2802 mmu_notifier_invalidate_range_start(&range); 2803 ptl = pud_lock(vma->vm_mm, pud); 2804 if (unlikely(!pud_trans_huge(*pud) && !pud_devmap(*pud))) 2805 goto out; 2806 __split_huge_pud_locked(vma, pud, range.start); 2807 2808 out: 2809 spin_unlock(ptl); 2810 mmu_notifier_invalidate_range_end(&range); 2811 } 2812 #else 2813 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 2814 unsigned long address) 2815 { 2816 } 2817 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 2818 2819 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 2820 unsigned long haddr, pmd_t *pmd) 2821 { 2822 struct mm_struct *mm = vma->vm_mm; 2823 pgtable_t pgtable; 2824 pmd_t _pmd, old_pmd; 2825 unsigned long addr; 2826 pte_t *pte; 2827 int i; 2828 2829 /* 2830 * Leave pmd empty until pte is filled note that it is fine to delay 2831 * notification until mmu_notifier_invalidate_range_end() as we are 2832 * replacing a zero pmd write protected page with a zero pte write 2833 * protected page. 2834 * 2835 * See Documentation/mm/mmu_notifier.rst 2836 */ 2837 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); 2838 2839 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2840 pmd_populate(mm, &_pmd, pgtable); 2841 2842 pte = pte_offset_map(&_pmd, haddr); 2843 VM_BUG_ON(!pte); 2844 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2845 pte_t entry; 2846 2847 entry = pfn_pte(my_zero_pfn(addr), vma->vm_page_prot); 2848 entry = pte_mkspecial(entry); 2849 if (pmd_uffd_wp(old_pmd)) 2850 entry = pte_mkuffd_wp(entry); 2851 VM_BUG_ON(!pte_none(ptep_get(pte))); 2852 set_pte_at(mm, addr, pte, entry); 2853 pte++; 2854 } 2855 pte_unmap(pte - 1); 2856 smp_wmb(); /* make pte visible before pmd */ 2857 pmd_populate(mm, pmd, pgtable); 2858 } 2859 2860 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 2861 unsigned long haddr, bool freeze) 2862 { 2863 struct mm_struct *mm = vma->vm_mm; 2864 struct folio *folio; 2865 struct page *page; 2866 pgtable_t pgtable; 2867 pmd_t old_pmd, _pmd; 2868 bool young, write, soft_dirty, pmd_migration = false, uffd_wp = false; 2869 bool anon_exclusive = false, dirty = false; 2870 unsigned long addr; 2871 pte_t *pte; 2872 int i; 2873 2874 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 2875 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2876 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 2877 VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd) 2878 && !pmd_devmap(*pmd)); 2879 2880 count_vm_event(THP_SPLIT_PMD); 2881 2882 if (!vma_is_anonymous(vma)) { 2883 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); 2884 /* 2885 * We are going to unmap this huge page. So 2886 * just go ahead and zap it 2887 */ 2888 if (arch_needs_pgtable_deposit()) 2889 zap_deposited_table(mm, pmd); 2890 if (!vma_is_dax(vma) && vma_is_special_huge(vma)) 2891 return; 2892 if (unlikely(is_pmd_migration_entry(old_pmd))) { 2893 swp_entry_t entry; 2894 2895 entry = pmd_to_swp_entry(old_pmd); 2896 folio = pfn_swap_entry_folio(entry); 2897 } else if (is_huge_zero_pmd(old_pmd)) { 2898 return; 2899 } else { 2900 page = pmd_page(old_pmd); 2901 folio = page_folio(page); 2902 if (!folio_test_dirty(folio) && pmd_dirty(old_pmd)) 2903 folio_mark_dirty(folio); 2904 if (!folio_test_referenced(folio) && pmd_young(old_pmd)) 2905 folio_set_referenced(folio); 2906 folio_remove_rmap_pmd(folio, page, vma); 2907 folio_put(folio); 2908 } 2909 add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR); 2910 return; 2911 } 2912 2913 if (is_huge_zero_pmd(*pmd)) { 2914 /* 2915 * FIXME: Do we want to invalidate secondary mmu by calling 2916 * mmu_notifier_arch_invalidate_secondary_tlbs() see comments below 2917 * inside __split_huge_pmd() ? 2918 * 2919 * We are going from a zero huge page write protected to zero 2920 * small page also write protected so it does not seems useful 2921 * to invalidate secondary mmu at this time. 2922 */ 2923 return __split_huge_zero_page_pmd(vma, haddr, pmd); 2924 } 2925 2926 pmd_migration = is_pmd_migration_entry(*pmd); 2927 if (unlikely(pmd_migration)) { 2928 swp_entry_t entry; 2929 2930 old_pmd = *pmd; 2931 entry = pmd_to_swp_entry(old_pmd); 2932 page = pfn_swap_entry_to_page(entry); 2933 write = is_writable_migration_entry(entry); 2934 if (PageAnon(page)) 2935 anon_exclusive = is_readable_exclusive_migration_entry(entry); 2936 young = is_migration_entry_young(entry); 2937 dirty = is_migration_entry_dirty(entry); 2938 soft_dirty = pmd_swp_soft_dirty(old_pmd); 2939 uffd_wp = pmd_swp_uffd_wp(old_pmd); 2940 } else { 2941 /* 2942 * Up to this point the pmd is present and huge and userland has 2943 * the whole access to the hugepage during the split (which 2944 * happens in place). If we overwrite the pmd with the not-huge 2945 * version pointing to the pte here (which of course we could if 2946 * all CPUs were bug free), userland could trigger a small page 2947 * size TLB miss on the small sized TLB while the hugepage TLB 2948 * entry is still established in the huge TLB. Some CPU doesn't 2949 * like that. See 2950 * http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum 2951 * 383 on page 105. Intel should be safe but is also warns that 2952 * it's only safe if the permission and cache attributes of the 2953 * two entries loaded in the two TLB is identical (which should 2954 * be the case here). But it is generally safer to never allow 2955 * small and huge TLB entries for the same virtual address to be 2956 * loaded simultaneously. So instead of doing "pmd_populate(); 2957 * flush_pmd_tlb_range();" we first mark the current pmd 2958 * notpresent (atomically because here the pmd_trans_huge must 2959 * remain set at all times on the pmd until the split is 2960 * complete for this pmd), then we flush the SMP TLB and finally 2961 * we write the non-huge version of the pmd entry with 2962 * pmd_populate. 2963 */ 2964 old_pmd = pmdp_invalidate(vma, haddr, pmd); 2965 page = pmd_page(old_pmd); 2966 folio = page_folio(page); 2967 if (pmd_dirty(old_pmd)) { 2968 dirty = true; 2969 folio_set_dirty(folio); 2970 } 2971 write = pmd_write(old_pmd); 2972 young = pmd_young(old_pmd); 2973 soft_dirty = pmd_soft_dirty(old_pmd); 2974 uffd_wp = pmd_uffd_wp(old_pmd); 2975 2976 VM_WARN_ON_FOLIO(!folio_ref_count(folio), folio); 2977 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); 2978 2979 /* 2980 * Without "freeze", we'll simply split the PMD, propagating the 2981 * PageAnonExclusive() flag for each PTE by setting it for 2982 * each subpage -- no need to (temporarily) clear. 2983 * 2984 * With "freeze" we want to replace mapped pages by 2985 * migration entries right away. This is only possible if we 2986 * managed to clear PageAnonExclusive() -- see 2987 * set_pmd_migration_entry(). 2988 * 2989 * In case we cannot clear PageAnonExclusive(), split the PMD 2990 * only and let try_to_migrate_one() fail later. 2991 * 2992 * See folio_try_share_anon_rmap_pmd(): invalidate PMD first. 2993 */ 2994 anon_exclusive = PageAnonExclusive(page); 2995 if (freeze && anon_exclusive && 2996 folio_try_share_anon_rmap_pmd(folio, page)) 2997 freeze = false; 2998 if (!freeze) { 2999 rmap_t rmap_flags = RMAP_NONE; 3000 3001 folio_ref_add(folio, HPAGE_PMD_NR - 1); 3002 if (anon_exclusive) 3003 rmap_flags |= RMAP_EXCLUSIVE; 3004 folio_add_anon_rmap_ptes(folio, page, HPAGE_PMD_NR, 3005 vma, haddr, rmap_flags); 3006 } 3007 } 3008 3009 /* 3010 * Withdraw the table only after we mark the pmd entry invalid. 3011 * This's critical for some architectures (Power). 3012 */ 3013 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 3014 pmd_populate(mm, &_pmd, pgtable); 3015 3016 pte = pte_offset_map(&_pmd, haddr); 3017 VM_BUG_ON(!pte); 3018 3019 /* 3020 * Note that NUMA hinting access restrictions are not transferred to 3021 * avoid any possibility of altering permissions across VMAs. 3022 */ 3023 if (freeze || pmd_migration) { 3024 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 3025 pte_t entry; 3026 swp_entry_t swp_entry; 3027 3028 if (write) 3029 swp_entry = make_writable_migration_entry( 3030 page_to_pfn(page + i)); 3031 else if (anon_exclusive) 3032 swp_entry = make_readable_exclusive_migration_entry( 3033 page_to_pfn(page + i)); 3034 else 3035 swp_entry = make_readable_migration_entry( 3036 page_to_pfn(page + i)); 3037 if (young) 3038 swp_entry = make_migration_entry_young(swp_entry); 3039 if (dirty) 3040 swp_entry = make_migration_entry_dirty(swp_entry); 3041 entry = swp_entry_to_pte(swp_entry); 3042 if (soft_dirty) 3043 entry = pte_swp_mksoft_dirty(entry); 3044 if (uffd_wp) 3045 entry = pte_swp_mkuffd_wp(entry); 3046 3047 VM_WARN_ON(!pte_none(ptep_get(pte + i))); 3048 set_pte_at(mm, addr, pte + i, entry); 3049 } 3050 } else { 3051 pte_t entry; 3052 3053 entry = mk_pte(page, READ_ONCE(vma->vm_page_prot)); 3054 if (write) 3055 entry = pte_mkwrite(entry, vma); 3056 if (!young) 3057 entry = pte_mkold(entry); 3058 /* NOTE: this may set soft-dirty too on some archs */ 3059 if (dirty) 3060 entry = pte_mkdirty(entry); 3061 if (soft_dirty) 3062 entry = pte_mksoft_dirty(entry); 3063 if (uffd_wp) 3064 entry = pte_mkuffd_wp(entry); 3065 3066 for (i = 0; i < HPAGE_PMD_NR; i++) 3067 VM_WARN_ON(!pte_none(ptep_get(pte + i))); 3068 3069 set_ptes(mm, haddr, pte, entry, HPAGE_PMD_NR); 3070 } 3071 pte_unmap(pte); 3072 3073 if (!pmd_migration) 3074 folio_remove_rmap_pmd(folio, page, vma); 3075 if (freeze) 3076 put_page(page); 3077 3078 smp_wmb(); /* make pte visible before pmd */ 3079 pmd_populate(mm, pmd, pgtable); 3080 } 3081 3082 void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, 3083 pmd_t *pmd, bool freeze, struct folio *folio) 3084 { 3085 bool pmd_migration = is_pmd_migration_entry(*pmd); 3086 3087 VM_WARN_ON_ONCE(folio && !folio_test_pmd_mappable(folio)); 3088 VM_WARN_ON_ONCE(!IS_ALIGNED(address, HPAGE_PMD_SIZE)); 3089 VM_WARN_ON_ONCE(folio && !folio_test_locked(folio)); 3090 VM_BUG_ON(freeze && !folio); 3091 3092 /* 3093 * When the caller requests to set up a migration entry, we 3094 * require a folio to check the PMD against. Otherwise, there 3095 * is a risk of replacing the wrong folio. 3096 */ 3097 if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd) || pmd_migration) { 3098 /* 3099 * Do not apply pmd_folio() to a migration entry; and folio lock 3100 * guarantees that it must be of the wrong folio anyway. 3101 */ 3102 if (folio && (pmd_migration || folio != pmd_folio(*pmd))) 3103 return; 3104 __split_huge_pmd_locked(vma, pmd, address, freeze); 3105 } 3106 } 3107 3108 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 3109 unsigned long address, bool freeze, struct folio *folio) 3110 { 3111 spinlock_t *ptl; 3112 struct mmu_notifier_range range; 3113 3114 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 3115 address & HPAGE_PMD_MASK, 3116 (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE); 3117 mmu_notifier_invalidate_range_start(&range); 3118 ptl = pmd_lock(vma->vm_mm, pmd); 3119 split_huge_pmd_locked(vma, range.start, pmd, freeze, folio); 3120 spin_unlock(ptl); 3121 mmu_notifier_invalidate_range_end(&range); 3122 } 3123 3124 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, 3125 bool freeze, struct folio *folio) 3126 { 3127 pmd_t *pmd = mm_find_pmd(vma->vm_mm, address); 3128 3129 if (!pmd) 3130 return; 3131 3132 __split_huge_pmd(vma, pmd, address, freeze, folio); 3133 } 3134 3135 static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address) 3136 { 3137 /* 3138 * If the new address isn't hpage aligned and it could previously 3139 * contain an hugepage: check if we need to split an huge pmd. 3140 */ 3141 if (!IS_ALIGNED(address, HPAGE_PMD_SIZE) && 3142 range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE), 3143 ALIGN(address, HPAGE_PMD_SIZE))) 3144 split_huge_pmd_address(vma, address, false, NULL); 3145 } 3146 3147 void vma_adjust_trans_huge(struct vm_area_struct *vma, 3148 unsigned long start, 3149 unsigned long end, 3150 struct vm_area_struct *next) 3151 { 3152 /* Check if we need to split start first. */ 3153 split_huge_pmd_if_needed(vma, start); 3154 3155 /* Check if we need to split end next. */ 3156 split_huge_pmd_if_needed(vma, end); 3157 3158 /* If we're incrementing next->vm_start, we might need to split it. */ 3159 if (next) 3160 split_huge_pmd_if_needed(next, end); 3161 } 3162 3163 static void unmap_folio(struct folio *folio) 3164 { 3165 enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SYNC | 3166 TTU_BATCH_FLUSH; 3167 3168 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 3169 3170 if (folio_test_pmd_mappable(folio)) 3171 ttu_flags |= TTU_SPLIT_HUGE_PMD; 3172 3173 /* 3174 * Anon pages need migration entries to preserve them, but file 3175 * pages can simply be left unmapped, then faulted back on demand. 3176 * If that is ever changed (perhaps for mlock), update remap_page(). 3177 */ 3178 if (folio_test_anon(folio)) 3179 try_to_migrate(folio, ttu_flags); 3180 else 3181 try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK); 3182 3183 try_to_unmap_flush(); 3184 } 3185 3186 static bool __discard_anon_folio_pmd_locked(struct vm_area_struct *vma, 3187 unsigned long addr, pmd_t *pmdp, 3188 struct folio *folio) 3189 { 3190 struct mm_struct *mm = vma->vm_mm; 3191 int ref_count, map_count; 3192 pmd_t orig_pmd = *pmdp; 3193 3194 if (pmd_dirty(orig_pmd)) 3195 folio_set_dirty(folio); 3196 if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { 3197 folio_set_swapbacked(folio); 3198 return false; 3199 } 3200 3201 orig_pmd = pmdp_huge_clear_flush(vma, addr, pmdp); 3202 3203 /* 3204 * Syncing against concurrent GUP-fast: 3205 * - clear PMD; barrier; read refcount 3206 * - inc refcount; barrier; read PMD 3207 */ 3208 smp_mb(); 3209 3210 ref_count = folio_ref_count(folio); 3211 map_count = folio_mapcount(folio); 3212 3213 /* 3214 * Order reads for folio refcount and dirty flag 3215 * (see comments in __remove_mapping()). 3216 */ 3217 smp_rmb(); 3218 3219 /* 3220 * If the folio or its PMD is redirtied at this point, or if there 3221 * are unexpected references, we will give up to discard this folio 3222 * and remap it. 3223 * 3224 * The only folio refs must be one from isolation plus the rmap(s). 3225 */ 3226 if (pmd_dirty(orig_pmd)) 3227 folio_set_dirty(folio); 3228 if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { 3229 folio_set_swapbacked(folio); 3230 set_pmd_at(mm, addr, pmdp, orig_pmd); 3231 return false; 3232 } 3233 3234 if (ref_count != map_count + 1) { 3235 set_pmd_at(mm, addr, pmdp, orig_pmd); 3236 return false; 3237 } 3238 3239 folio_remove_rmap_pmd(folio, pmd_page(orig_pmd), vma); 3240 zap_deposited_table(mm, pmdp); 3241 add_mm_counter(mm, MM_ANONPAGES, -HPAGE_PMD_NR); 3242 if (vma->vm_flags & VM_LOCKED) 3243 mlock_drain_local(); 3244 folio_put(folio); 3245 3246 return true; 3247 } 3248 3249 bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr, 3250 pmd_t *pmdp, struct folio *folio) 3251 { 3252 VM_WARN_ON_FOLIO(!folio_test_pmd_mappable(folio), folio); 3253 VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); 3254 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); 3255 VM_WARN_ON_FOLIO(folio_test_swapbacked(folio), folio); 3256 VM_WARN_ON_ONCE(!IS_ALIGNED(addr, HPAGE_PMD_SIZE)); 3257 3258 return __discard_anon_folio_pmd_locked(vma, addr, pmdp, folio); 3259 } 3260 3261 static void remap_page(struct folio *folio, unsigned long nr, int flags) 3262 { 3263 int i = 0; 3264 3265 /* If unmap_folio() uses try_to_migrate() on file, remove this check */ 3266 if (!folio_test_anon(folio)) 3267 return; 3268 for (;;) { 3269 remove_migration_ptes(folio, folio, RMP_LOCKED | flags); 3270 i += folio_nr_pages(folio); 3271 if (i >= nr) 3272 break; 3273 folio = folio_next(folio); 3274 } 3275 } 3276 3277 static void lru_add_split_folio(struct folio *folio, struct folio *new_folio, 3278 struct lruvec *lruvec, struct list_head *list) 3279 { 3280 VM_BUG_ON_FOLIO(folio_test_lru(new_folio), folio); 3281 lockdep_assert_held(&lruvec->lru_lock); 3282 3283 if (list) { 3284 /* page reclaim is reclaiming a huge page */ 3285 VM_WARN_ON(folio_test_lru(folio)); 3286 folio_get(new_folio); 3287 list_add_tail(&new_folio->lru, list); 3288 } else { 3289 /* head is still on lru (and we have it frozen) */ 3290 VM_WARN_ON(!folio_test_lru(folio)); 3291 if (folio_test_unevictable(folio)) 3292 new_folio->mlock_count = 0; 3293 else 3294 list_add_tail(&new_folio->lru, &folio->lru); 3295 folio_set_lru(new_folio); 3296 } 3297 } 3298 3299 /* Racy check whether the huge page can be split */ 3300 bool can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins) 3301 { 3302 int extra_pins; 3303 3304 /* Additional pins from page cache */ 3305 if (folio_test_anon(folio)) 3306 extra_pins = folio_test_swapcache(folio) ? 3307 folio_nr_pages(folio) : 0; 3308 else 3309 extra_pins = folio_nr_pages(folio); 3310 if (pextra_pins) 3311 *pextra_pins = extra_pins; 3312 return folio_mapcount(folio) == folio_ref_count(folio) - extra_pins - 3313 caller_pins; 3314 } 3315 3316 /* 3317 * It splits @folio into @new_order folios and copies the @folio metadata to 3318 * all the resulting folios. 3319 */ 3320 static void __split_folio_to_order(struct folio *folio, int old_order, 3321 int new_order) 3322 { 3323 long new_nr_pages = 1 << new_order; 3324 long nr_pages = 1 << old_order; 3325 long i; 3326 3327 /* 3328 * Skip the first new_nr_pages, since the new folio from them have all 3329 * the flags from the original folio. 3330 */ 3331 for (i = new_nr_pages; i < nr_pages; i += new_nr_pages) { 3332 struct page *new_head = &folio->page + i; 3333 3334 /* 3335 * Careful: new_folio is not a "real" folio before we cleared PageTail. 3336 * Don't pass it around before clear_compound_head(). 3337 */ 3338 struct folio *new_folio = (struct folio *)new_head; 3339 3340 VM_BUG_ON_PAGE(atomic_read(&new_folio->_mapcount) != -1, new_head); 3341 3342 /* 3343 * Clone page flags before unfreezing refcount. 3344 * 3345 * After successful get_page_unless_zero() might follow flags change, 3346 * for example lock_page() which set PG_waiters. 3347 * 3348 * Note that for mapped sub-pages of an anonymous THP, 3349 * PG_anon_exclusive has been cleared in unmap_folio() and is stored in 3350 * the migration entry instead from where remap_page() will restore it. 3351 * We can still have PG_anon_exclusive set on effectively unmapped and 3352 * unreferenced sub-pages of an anonymous THP: we can simply drop 3353 * PG_anon_exclusive (-> PG_mappedtodisk) for these here. 3354 */ 3355 new_folio->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 3356 new_folio->flags |= (folio->flags & 3357 ((1L << PG_referenced) | 3358 (1L << PG_swapbacked) | 3359 (1L << PG_swapcache) | 3360 (1L << PG_mlocked) | 3361 (1L << PG_uptodate) | 3362 (1L << PG_active) | 3363 (1L << PG_workingset) | 3364 (1L << PG_locked) | 3365 (1L << PG_unevictable) | 3366 #ifdef CONFIG_ARCH_USES_PG_ARCH_2 3367 (1L << PG_arch_2) | 3368 #endif 3369 #ifdef CONFIG_ARCH_USES_PG_ARCH_3 3370 (1L << PG_arch_3) | 3371 #endif 3372 (1L << PG_dirty) | 3373 LRU_GEN_MASK | LRU_REFS_MASK)); 3374 3375 new_folio->mapping = folio->mapping; 3376 new_folio->index = folio->index + i; 3377 3378 /* 3379 * page->private should not be set in tail pages. Fix up and warn once 3380 * if private is unexpectedly set. 3381 */ 3382 if (unlikely(new_folio->private)) { 3383 VM_WARN_ON_ONCE_PAGE(true, new_head); 3384 new_folio->private = NULL; 3385 } 3386 3387 if (folio_test_swapcache(folio)) 3388 new_folio->swap.val = folio->swap.val + i; 3389 3390 /* Page flags must be visible before we make the page non-compound. */ 3391 smp_wmb(); 3392 3393 /* 3394 * Clear PageTail before unfreezing page refcount. 3395 * 3396 * After successful get_page_unless_zero() might follow put_page() 3397 * which needs correct compound_head(). 3398 */ 3399 clear_compound_head(new_head); 3400 if (new_order) { 3401 prep_compound_page(new_head, new_order); 3402 folio_set_large_rmappable(new_folio); 3403 } 3404 3405 if (folio_test_young(folio)) 3406 folio_set_young(new_folio); 3407 if (folio_test_idle(folio)) 3408 folio_set_idle(new_folio); 3409 #ifdef CONFIG_MEMCG 3410 new_folio->memcg_data = folio->memcg_data; 3411 #endif 3412 3413 folio_xchg_last_cpupid(new_folio, folio_last_cpupid(folio)); 3414 } 3415 3416 if (new_order) 3417 folio_set_order(folio, new_order); 3418 else 3419 ClearPageCompound(&folio->page); 3420 } 3421 3422 /* 3423 * It splits an unmapped @folio to lower order smaller folios in two ways. 3424 * @folio: the to-be-split folio 3425 * @new_order: the smallest order of the after split folios (since buddy 3426 * allocator like split generates folios with orders from @folio's 3427 * order - 1 to new_order). 3428 * @split_at: in buddy allocator like split, the folio containing @split_at 3429 * will be split until its order becomes @new_order. 3430 * @lock_at: the folio containing @lock_at is left locked for caller. 3431 * @list: the after split folios will be added to @list if it is not NULL, 3432 * otherwise to LRU lists. 3433 * @end: the end of the file @folio maps to. -1 if @folio is anonymous memory. 3434 * @xas: xa_state pointing to folio->mapping->i_pages and locked by caller 3435 * @mapping: @folio->mapping 3436 * @uniform_split: if the split is uniform or not (buddy allocator like split) 3437 * 3438 * 3439 * 1. uniform split: the given @folio into multiple @new_order small folios, 3440 * where all small folios have the same order. This is done when 3441 * uniform_split is true. 3442 * 2. buddy allocator like (non-uniform) split: the given @folio is split into 3443 * half and one of the half (containing the given page) is split into half 3444 * until the given @page's order becomes @new_order. This is done when 3445 * uniform_split is false. 3446 * 3447 * The high level flow for these two methods are: 3448 * 1. uniform split: a single __split_folio_to_order() is called to split the 3449 * @folio into @new_order, then we traverse all the resulting folios one by 3450 * one in PFN ascending order and perform stats, unfreeze, adding to list, 3451 * and file mapping index operations. 3452 * 2. non-uniform split: in general, folio_order - @new_order calls to 3453 * __split_folio_to_order() are made in a for loop to split the @folio 3454 * to one lower order at a time. The resulting small folios are processed 3455 * like what is done during the traversal in 1, except the one containing 3456 * @page, which is split in next for loop. 3457 * 3458 * After splitting, the caller's folio reference will be transferred to the 3459 * folio containing @page. The other folios may be freed if they are not mapped. 3460 * 3461 * In terms of locking, after splitting, 3462 * 1. uniform split leaves @page (or the folio contains it) locked; 3463 * 2. buddy allocator like (non-uniform) split leaves @folio locked. 3464 * 3465 * 3466 * For !uniform_split, when -ENOMEM is returned, the original folio might be 3467 * split. The caller needs to check the input folio. 3468 */ 3469 static int __split_unmapped_folio(struct folio *folio, int new_order, 3470 struct page *split_at, struct page *lock_at, 3471 struct list_head *list, pgoff_t end, 3472 struct xa_state *xas, struct address_space *mapping, 3473 bool uniform_split) 3474 { 3475 struct lruvec *lruvec; 3476 struct address_space *swap_cache = NULL; 3477 struct folio *origin_folio = folio; 3478 struct folio *next_folio = folio_next(folio); 3479 struct folio *new_folio; 3480 struct folio *next; 3481 int order = folio_order(folio); 3482 int split_order; 3483 int start_order = uniform_split ? new_order : order - 1; 3484 int nr_dropped = 0; 3485 int ret = 0; 3486 bool stop_split = false; 3487 3488 if (folio_test_swapcache(folio)) { 3489 VM_BUG_ON(mapping); 3490 3491 /* a swapcache folio can only be uniformly split to order-0 */ 3492 if (!uniform_split || new_order != 0) 3493 return -EINVAL; 3494 3495 swap_cache = swap_address_space(folio->swap); 3496 xa_lock(&swap_cache->i_pages); 3497 } 3498 3499 if (folio_test_anon(folio)) 3500 mod_mthp_stat(order, MTHP_STAT_NR_ANON, -1); 3501 3502 /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ 3503 lruvec = folio_lruvec_lock(folio); 3504 3505 folio_clear_has_hwpoisoned(folio); 3506 3507 /* 3508 * split to new_order one order at a time. For uniform split, 3509 * folio is split to new_order directly. 3510 */ 3511 for (split_order = start_order; 3512 split_order >= new_order && !stop_split; 3513 split_order--) { 3514 int old_order = folio_order(folio); 3515 struct folio *release; 3516 struct folio *end_folio = folio_next(folio); 3517 3518 /* order-1 anonymous folio is not supported */ 3519 if (folio_test_anon(folio) && split_order == 1) 3520 continue; 3521 if (uniform_split && split_order != new_order) 3522 continue; 3523 3524 if (mapping) { 3525 /* 3526 * uniform split has xas_split_alloc() called before 3527 * irq is disabled to allocate enough memory, whereas 3528 * non-uniform split can handle ENOMEM. 3529 */ 3530 if (uniform_split) 3531 xas_split(xas, folio, old_order); 3532 else { 3533 xas_set_order(xas, folio->index, split_order); 3534 xas_try_split(xas, folio, old_order); 3535 if (xas_error(xas)) { 3536 ret = xas_error(xas); 3537 stop_split = true; 3538 goto after_split; 3539 } 3540 } 3541 } 3542 3543 folio_split_memcg_refs(folio, old_order, split_order); 3544 split_page_owner(&folio->page, old_order, split_order); 3545 pgalloc_tag_split(folio, old_order, split_order); 3546 3547 __split_folio_to_order(folio, old_order, split_order); 3548 3549 after_split: 3550 /* 3551 * Iterate through after-split folios and perform related 3552 * operations. But in buddy allocator like split, the folio 3553 * containing the specified page is skipped until its order 3554 * is new_order, since the folio will be worked on in next 3555 * iteration. 3556 */ 3557 for (release = folio; release != end_folio; release = next) { 3558 next = folio_next(release); 3559 /* 3560 * for buddy allocator like split, the folio containing 3561 * page will be split next and should not be released, 3562 * until the folio's order is new_order or stop_split 3563 * is set to true by the above xas_split() failure. 3564 */ 3565 if (release == page_folio(split_at)) { 3566 folio = release; 3567 if (split_order != new_order && !stop_split) 3568 continue; 3569 } 3570 if (folio_test_anon(release)) { 3571 mod_mthp_stat(folio_order(release), 3572 MTHP_STAT_NR_ANON, 1); 3573 } 3574 3575 /* 3576 * origin_folio should be kept frozon until page cache 3577 * entries are updated with all the other after-split 3578 * folios to prevent others seeing stale page cache 3579 * entries. 3580 */ 3581 if (release == origin_folio) 3582 continue; 3583 3584 folio_ref_unfreeze(release, 1 + 3585 ((mapping || swap_cache) ? 3586 folio_nr_pages(release) : 0)); 3587 3588 lru_add_split_folio(origin_folio, release, lruvec, 3589 list); 3590 3591 /* Some pages can be beyond EOF: drop them from cache */ 3592 if (release->index >= end) { 3593 if (shmem_mapping(mapping)) 3594 nr_dropped += folio_nr_pages(release); 3595 else if (folio_test_clear_dirty(release)) 3596 folio_account_cleaned(release, 3597 inode_to_wb(mapping->host)); 3598 __filemap_remove_folio(release, NULL); 3599 folio_put_refs(release, folio_nr_pages(release)); 3600 } else if (mapping) { 3601 __xa_store(&mapping->i_pages, 3602 release->index, release, 0); 3603 } else if (swap_cache) { 3604 __xa_store(&swap_cache->i_pages, 3605 swap_cache_index(release->swap), 3606 release, 0); 3607 } 3608 } 3609 } 3610 3611 /* 3612 * Unfreeze origin_folio only after all page cache entries, which used 3613 * to point to it, have been updated with new folios. Otherwise, 3614 * a parallel folio_try_get() can grab origin_folio and its caller can 3615 * see stale page cache entries. 3616 */ 3617 folio_ref_unfreeze(origin_folio, 1 + 3618 ((mapping || swap_cache) ? folio_nr_pages(origin_folio) : 0)); 3619 3620 unlock_page_lruvec(lruvec); 3621 3622 if (swap_cache) 3623 xa_unlock(&swap_cache->i_pages); 3624 if (mapping) 3625 xa_unlock(&mapping->i_pages); 3626 3627 /* Caller disabled irqs, so they are still disabled here */ 3628 local_irq_enable(); 3629 3630 if (nr_dropped) 3631 shmem_uncharge(mapping->host, nr_dropped); 3632 3633 remap_page(origin_folio, 1 << order, 3634 folio_test_anon(origin_folio) ? 3635 RMP_USE_SHARED_ZEROPAGE : 0); 3636 3637 /* 3638 * At this point, folio should contain the specified page. 3639 * For uniform split, it is left for caller to unlock. 3640 * For buddy allocator like split, the first after-split folio is left 3641 * for caller to unlock. 3642 */ 3643 for (new_folio = origin_folio; new_folio != next_folio; new_folio = next) { 3644 next = folio_next(new_folio); 3645 if (new_folio == page_folio(lock_at)) 3646 continue; 3647 3648 folio_unlock(new_folio); 3649 /* 3650 * Subpages may be freed if there wasn't any mapping 3651 * like if add_to_swap() is running on a lru page that 3652 * had its mapping zapped. And freeing these pages 3653 * requires taking the lru_lock so we do the put_page 3654 * of the tail pages after the split is complete. 3655 */ 3656 free_folio_and_swap_cache(new_folio); 3657 } 3658 return ret; 3659 } 3660 3661 bool non_uniform_split_supported(struct folio *folio, unsigned int new_order, 3662 bool warns) 3663 { 3664 if (folio_test_anon(folio)) { 3665 /* order-1 is not supported for anonymous THP. */ 3666 VM_WARN_ONCE(warns && new_order == 1, 3667 "Cannot split to order-1 folio"); 3668 return new_order != 1; 3669 } else if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && 3670 !mapping_large_folio_support(folio->mapping)) { 3671 /* 3672 * No split if the file system does not support large folio. 3673 * Note that we might still have THPs in such mappings due to 3674 * CONFIG_READ_ONLY_THP_FOR_FS. But in that case, the mapping 3675 * does not actually support large folios properly. 3676 */ 3677 VM_WARN_ONCE(warns, 3678 "Cannot split file folio to non-0 order"); 3679 return false; 3680 } 3681 3682 /* Only swapping a whole PMD-mapped folio is supported */ 3683 if (folio_test_swapcache(folio)) { 3684 VM_WARN_ONCE(warns, 3685 "Cannot split swapcache folio to non-0 order"); 3686 return false; 3687 } 3688 3689 return true; 3690 } 3691 3692 /* See comments in non_uniform_split_supported() */ 3693 bool uniform_split_supported(struct folio *folio, unsigned int new_order, 3694 bool warns) 3695 { 3696 if (folio_test_anon(folio)) { 3697 VM_WARN_ONCE(warns && new_order == 1, 3698 "Cannot split to order-1 folio"); 3699 return new_order != 1; 3700 } else if (new_order) { 3701 if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && 3702 !mapping_large_folio_support(folio->mapping)) { 3703 VM_WARN_ONCE(warns, 3704 "Cannot split file folio to non-0 order"); 3705 return false; 3706 } 3707 } 3708 3709 if (new_order && folio_test_swapcache(folio)) { 3710 VM_WARN_ONCE(warns, 3711 "Cannot split swapcache folio to non-0 order"); 3712 return false; 3713 } 3714 3715 return true; 3716 } 3717 3718 /* 3719 * __folio_split: split a folio at @split_at to a @new_order folio 3720 * @folio: folio to split 3721 * @new_order: the order of the new folio 3722 * @split_at: a page within the new folio 3723 * @lock_at: a page within @folio to be left locked to caller 3724 * @list: after-split folios will be put on it if non NULL 3725 * @uniform_split: perform uniform split or not (non-uniform split) 3726 * 3727 * It calls __split_unmapped_folio() to perform uniform and non-uniform split. 3728 * It is in charge of checking whether the split is supported or not and 3729 * preparing @folio for __split_unmapped_folio(). 3730 * 3731 * return: 0: successful, <0 failed (if -ENOMEM is returned, @folio might be 3732 * split but not to @new_order, the caller needs to check) 3733 */ 3734 static int __folio_split(struct folio *folio, unsigned int new_order, 3735 struct page *split_at, struct page *lock_at, 3736 struct list_head *list, bool uniform_split) 3737 { 3738 struct deferred_split *ds_queue = get_deferred_split_queue(folio); 3739 XA_STATE(xas, &folio->mapping->i_pages, folio->index); 3740 bool is_anon = folio_test_anon(folio); 3741 struct address_space *mapping = NULL; 3742 struct anon_vma *anon_vma = NULL; 3743 int order = folio_order(folio); 3744 int extra_pins, ret; 3745 pgoff_t end; 3746 bool is_hzp; 3747 3748 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 3749 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 3750 3751 if (folio != page_folio(split_at) || folio != page_folio(lock_at)) 3752 return -EINVAL; 3753 3754 if (new_order >= folio_order(folio)) 3755 return -EINVAL; 3756 3757 if (uniform_split && !uniform_split_supported(folio, new_order, true)) 3758 return -EINVAL; 3759 3760 if (!uniform_split && 3761 !non_uniform_split_supported(folio, new_order, true)) 3762 return -EINVAL; 3763 3764 is_hzp = is_huge_zero_folio(folio); 3765 if (is_hzp) { 3766 pr_warn_ratelimited("Called split_huge_page for huge zero page\n"); 3767 return -EBUSY; 3768 } 3769 3770 if (folio_test_writeback(folio)) 3771 return -EBUSY; 3772 3773 if (is_anon) { 3774 /* 3775 * The caller does not necessarily hold an mmap_lock that would 3776 * prevent the anon_vma disappearing so we first we take a 3777 * reference to it and then lock the anon_vma for write. This 3778 * is similar to folio_lock_anon_vma_read except the write lock 3779 * is taken to serialise against parallel split or collapse 3780 * operations. 3781 */ 3782 anon_vma = folio_get_anon_vma(folio); 3783 if (!anon_vma) { 3784 ret = -EBUSY; 3785 goto out; 3786 } 3787 end = -1; 3788 mapping = NULL; 3789 anon_vma_lock_write(anon_vma); 3790 } else { 3791 unsigned int min_order; 3792 gfp_t gfp; 3793 3794 mapping = folio->mapping; 3795 3796 /* Truncated ? */ 3797 /* 3798 * TODO: add support for large shmem folio in swap cache. 3799 * When shmem is in swap cache, mapping is NULL and 3800 * folio_test_swapcache() is true. 3801 */ 3802 if (!mapping) { 3803 ret = -EBUSY; 3804 goto out; 3805 } 3806 3807 min_order = mapping_min_folio_order(folio->mapping); 3808 if (new_order < min_order) { 3809 VM_WARN_ONCE(1, "Cannot split mapped folio below min-order: %u", 3810 min_order); 3811 ret = -EINVAL; 3812 goto out; 3813 } 3814 3815 gfp = current_gfp_context(mapping_gfp_mask(mapping) & 3816 GFP_RECLAIM_MASK); 3817 3818 if (!filemap_release_folio(folio, gfp)) { 3819 ret = -EBUSY; 3820 goto out; 3821 } 3822 3823 if (uniform_split) { 3824 xas_set_order(&xas, folio->index, new_order); 3825 xas_split_alloc(&xas, folio, folio_order(folio), gfp); 3826 if (xas_error(&xas)) { 3827 ret = xas_error(&xas); 3828 goto out; 3829 } 3830 } 3831 3832 anon_vma = NULL; 3833 i_mmap_lock_read(mapping); 3834 3835 /* 3836 *__split_unmapped_folio() may need to trim off pages beyond 3837 * EOF: but on 32-bit, i_size_read() takes an irq-unsafe 3838 * seqlock, which cannot be nested inside the page tree lock. 3839 * So note end now: i_size itself may be changed at any moment, 3840 * but folio lock is good enough to serialize the trimming. 3841 */ 3842 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); 3843 if (shmem_mapping(mapping)) 3844 end = shmem_fallocend(mapping->host, end); 3845 } 3846 3847 /* 3848 * Racy check if we can split the page, before unmap_folio() will 3849 * split PMDs 3850 */ 3851 if (!can_split_folio(folio, 1, &extra_pins)) { 3852 ret = -EAGAIN; 3853 goto out_unlock; 3854 } 3855 3856 unmap_folio(folio); 3857 3858 /* block interrupt reentry in xa_lock and spinlock */ 3859 local_irq_disable(); 3860 if (mapping) { 3861 /* 3862 * Check if the folio is present in page cache. 3863 * We assume all tail are present too, if folio is there. 3864 */ 3865 xas_lock(&xas); 3866 xas_reset(&xas); 3867 if (xas_load(&xas) != folio) 3868 goto fail; 3869 } 3870 3871 /* Prevent deferred_split_scan() touching ->_refcount */ 3872 spin_lock(&ds_queue->split_queue_lock); 3873 if (folio_ref_freeze(folio, 1 + extra_pins)) { 3874 if (folio_order(folio) > 1 && 3875 !list_empty(&folio->_deferred_list)) { 3876 ds_queue->split_queue_len--; 3877 if (folio_test_partially_mapped(folio)) { 3878 folio_clear_partially_mapped(folio); 3879 mod_mthp_stat(folio_order(folio), 3880 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 3881 } 3882 /* 3883 * Reinitialize page_deferred_list after removing the 3884 * page from the split_queue, otherwise a subsequent 3885 * split will see list corruption when checking the 3886 * page_deferred_list. 3887 */ 3888 list_del_init(&folio->_deferred_list); 3889 } 3890 spin_unlock(&ds_queue->split_queue_lock); 3891 if (mapping) { 3892 int nr = folio_nr_pages(folio); 3893 3894 if (folio_test_pmd_mappable(folio) && 3895 new_order < HPAGE_PMD_ORDER) { 3896 if (folio_test_swapbacked(folio)) { 3897 __lruvec_stat_mod_folio(folio, 3898 NR_SHMEM_THPS, -nr); 3899 } else { 3900 __lruvec_stat_mod_folio(folio, 3901 NR_FILE_THPS, -nr); 3902 filemap_nr_thps_dec(mapping); 3903 } 3904 } 3905 } 3906 3907 ret = __split_unmapped_folio(folio, new_order, 3908 split_at, lock_at, list, end, &xas, mapping, 3909 uniform_split); 3910 } else { 3911 spin_unlock(&ds_queue->split_queue_lock); 3912 fail: 3913 if (mapping) 3914 xas_unlock(&xas); 3915 local_irq_enable(); 3916 remap_page(folio, folio_nr_pages(folio), 0); 3917 ret = -EAGAIN; 3918 } 3919 3920 out_unlock: 3921 if (anon_vma) { 3922 anon_vma_unlock_write(anon_vma); 3923 put_anon_vma(anon_vma); 3924 } 3925 if (mapping) 3926 i_mmap_unlock_read(mapping); 3927 out: 3928 xas_destroy(&xas); 3929 if (order == HPAGE_PMD_ORDER) 3930 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); 3931 count_mthp_stat(order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED); 3932 return ret; 3933 } 3934 3935 /* 3936 * This function splits a large folio into smaller folios of order @new_order. 3937 * @page can point to any page of the large folio to split. The split operation 3938 * does not change the position of @page. 3939 * 3940 * Prerequisites: 3941 * 3942 * 1) The caller must hold a reference on the @page's owning folio, also known 3943 * as the large folio. 3944 * 3945 * 2) The large folio must be locked. 3946 * 3947 * 3) The folio must not be pinned. Any unexpected folio references, including 3948 * GUP pins, will result in the folio not getting split; instead, the caller 3949 * will receive an -EAGAIN. 3950 * 3951 * 4) @new_order > 1, usually. Splitting to order-1 anonymous folios is not 3952 * supported for non-file-backed folios, because folio->_deferred_list, which 3953 * is used by partially mapped folios, is stored in subpage 2, but an order-1 3954 * folio only has subpages 0 and 1. File-backed order-1 folios are supported, 3955 * since they do not use _deferred_list. 3956 * 3957 * After splitting, the caller's folio reference will be transferred to @page, 3958 * resulting in a raised refcount of @page after this call. The other pages may 3959 * be freed if they are not mapped. 3960 * 3961 * If @list is null, tail pages will be added to LRU list, otherwise, to @list. 3962 * 3963 * Pages in @new_order will inherit the mapping, flags, and so on from the 3964 * huge page. 3965 * 3966 * Returns 0 if the huge page was split successfully. 3967 * 3968 * Returns -EAGAIN if the folio has unexpected reference (e.g., GUP) or if 3969 * the folio was concurrently removed from the page cache. 3970 * 3971 * Returns -EBUSY when trying to split the huge zeropage, if the folio is 3972 * under writeback, if fs-specific folio metadata cannot currently be 3973 * released, or if some unexpected race happened (e.g., anon VMA disappeared, 3974 * truncation). 3975 * 3976 * Callers should ensure that the order respects the address space mapping 3977 * min-order if one is set for non-anonymous folios. 3978 * 3979 * Returns -EINVAL when trying to split to an order that is incompatible 3980 * with the folio. Splitting to order 0 is compatible with all folios. 3981 */ 3982 int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, 3983 unsigned int new_order) 3984 { 3985 struct folio *folio = page_folio(page); 3986 3987 return __folio_split(folio, new_order, &folio->page, page, list, true); 3988 } 3989 3990 /* 3991 * folio_split: split a folio at @split_at to a @new_order folio 3992 * @folio: folio to split 3993 * @new_order: the order of the new folio 3994 * @split_at: a page within the new folio 3995 * 3996 * return: 0: successful, <0 failed (if -ENOMEM is returned, @folio might be 3997 * split but not to @new_order, the caller needs to check) 3998 * 3999 * It has the same prerequisites and returns as 4000 * split_huge_page_to_list_to_order(). 4001 * 4002 * Split a folio at @split_at to a new_order folio, leave the 4003 * remaining subpages of the original folio as large as possible. For example, 4004 * in the case of splitting an order-9 folio at its third order-3 subpages to 4005 * an order-3 folio, there are 2^(9-3)=64 order-3 subpages in the order-9 folio. 4006 * After the split, there will be a group of folios with different orders and 4007 * the new folio containing @split_at is marked in bracket: 4008 * [order-4, {order-3}, order-3, order-5, order-6, order-7, order-8]. 4009 * 4010 * After split, folio is left locked for caller. 4011 */ 4012 int folio_split(struct folio *folio, unsigned int new_order, 4013 struct page *split_at, struct list_head *list) 4014 { 4015 return __folio_split(folio, new_order, split_at, &folio->page, list, 4016 false); 4017 } 4018 4019 int min_order_for_split(struct folio *folio) 4020 { 4021 if (folio_test_anon(folio)) 4022 return 0; 4023 4024 if (!folio->mapping) { 4025 if (folio_test_pmd_mappable(folio)) 4026 count_vm_event(THP_SPLIT_PAGE_FAILED); 4027 return -EBUSY; 4028 } 4029 4030 return mapping_min_folio_order(folio->mapping); 4031 } 4032 4033 int split_folio_to_list(struct folio *folio, struct list_head *list) 4034 { 4035 int ret = min_order_for_split(folio); 4036 4037 if (ret < 0) 4038 return ret; 4039 4040 return split_huge_page_to_list_to_order(&folio->page, list, ret); 4041 } 4042 4043 /* 4044 * __folio_unqueue_deferred_split() is not to be called directly: 4045 * the folio_unqueue_deferred_split() inline wrapper in mm/internal.h 4046 * limits its calls to those folios which may have a _deferred_list for 4047 * queueing THP splits, and that list is (racily observed to be) non-empty. 4048 * 4049 * It is unsafe to call folio_unqueue_deferred_split() until folio refcount is 4050 * zero: because even when split_queue_lock is held, a non-empty _deferred_list 4051 * might be in use on deferred_split_scan()'s unlocked on-stack list. 4052 * 4053 * If memory cgroups are enabled, split_queue_lock is in the mem_cgroup: it is 4054 * therefore important to unqueue deferred split before changing folio memcg. 4055 */ 4056 bool __folio_unqueue_deferred_split(struct folio *folio) 4057 { 4058 struct deferred_split *ds_queue; 4059 unsigned long flags; 4060 bool unqueued = false; 4061 4062 WARN_ON_ONCE(folio_ref_count(folio)); 4063 WARN_ON_ONCE(!mem_cgroup_disabled() && !folio_memcg(folio)); 4064 4065 ds_queue = get_deferred_split_queue(folio); 4066 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 4067 if (!list_empty(&folio->_deferred_list)) { 4068 ds_queue->split_queue_len--; 4069 if (folio_test_partially_mapped(folio)) { 4070 folio_clear_partially_mapped(folio); 4071 mod_mthp_stat(folio_order(folio), 4072 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 4073 } 4074 list_del_init(&folio->_deferred_list); 4075 unqueued = true; 4076 } 4077 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 4078 4079 return unqueued; /* useful for debug warnings */ 4080 } 4081 4082 /* partially_mapped=false won't clear PG_partially_mapped folio flag */ 4083 void deferred_split_folio(struct folio *folio, bool partially_mapped) 4084 { 4085 struct deferred_split *ds_queue = get_deferred_split_queue(folio); 4086 #ifdef CONFIG_MEMCG 4087 struct mem_cgroup *memcg = folio_memcg(folio); 4088 #endif 4089 unsigned long flags; 4090 4091 /* 4092 * Order 1 folios have no space for a deferred list, but we also 4093 * won't waste much memory by not adding them to the deferred list. 4094 */ 4095 if (folio_order(folio) <= 1) 4096 return; 4097 4098 if (!partially_mapped && !split_underused_thp) 4099 return; 4100 4101 /* 4102 * Exclude swapcache: originally to avoid a corrupt deferred split 4103 * queue. Nowadays that is fully prevented by memcg1_swapout(); 4104 * but if page reclaim is already handling the same folio, it is 4105 * unnecessary to handle it again in the shrinker, so excluding 4106 * swapcache here may still be a useful optimization. 4107 */ 4108 if (folio_test_swapcache(folio)) 4109 return; 4110 4111 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 4112 if (partially_mapped) { 4113 if (!folio_test_partially_mapped(folio)) { 4114 folio_set_partially_mapped(folio); 4115 if (folio_test_pmd_mappable(folio)) 4116 count_vm_event(THP_DEFERRED_SPLIT_PAGE); 4117 count_mthp_stat(folio_order(folio), MTHP_STAT_SPLIT_DEFERRED); 4118 mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, 1); 4119 4120 } 4121 } else { 4122 /* partially mapped folios cannot become non-partially mapped */ 4123 VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio); 4124 } 4125 if (list_empty(&folio->_deferred_list)) { 4126 list_add_tail(&folio->_deferred_list, &ds_queue->split_queue); 4127 ds_queue->split_queue_len++; 4128 #ifdef CONFIG_MEMCG 4129 if (memcg) 4130 set_shrinker_bit(memcg, folio_nid(folio), 4131 deferred_split_shrinker->id); 4132 #endif 4133 } 4134 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 4135 } 4136 4137 static unsigned long deferred_split_count(struct shrinker *shrink, 4138 struct shrink_control *sc) 4139 { 4140 struct pglist_data *pgdata = NODE_DATA(sc->nid); 4141 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 4142 4143 #ifdef CONFIG_MEMCG 4144 if (sc->memcg) 4145 ds_queue = &sc->memcg->deferred_split_queue; 4146 #endif 4147 return READ_ONCE(ds_queue->split_queue_len); 4148 } 4149 4150 static bool thp_underused(struct folio *folio) 4151 { 4152 int num_zero_pages = 0, num_filled_pages = 0; 4153 void *kaddr; 4154 int i; 4155 4156 if (khugepaged_max_ptes_none == HPAGE_PMD_NR - 1) 4157 return false; 4158 4159 for (i = 0; i < folio_nr_pages(folio); i++) { 4160 kaddr = kmap_local_folio(folio, i * PAGE_SIZE); 4161 if (!memchr_inv(kaddr, 0, PAGE_SIZE)) { 4162 num_zero_pages++; 4163 if (num_zero_pages > khugepaged_max_ptes_none) { 4164 kunmap_local(kaddr); 4165 return true; 4166 } 4167 } else { 4168 /* 4169 * Another path for early exit once the number 4170 * of non-zero filled pages exceeds threshold. 4171 */ 4172 num_filled_pages++; 4173 if (num_filled_pages >= HPAGE_PMD_NR - khugepaged_max_ptes_none) { 4174 kunmap_local(kaddr); 4175 return false; 4176 } 4177 } 4178 kunmap_local(kaddr); 4179 } 4180 return false; 4181 } 4182 4183 static unsigned long deferred_split_scan(struct shrinker *shrink, 4184 struct shrink_control *sc) 4185 { 4186 struct pglist_data *pgdata = NODE_DATA(sc->nid); 4187 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 4188 unsigned long flags; 4189 LIST_HEAD(list); 4190 struct folio *folio, *next, *prev = NULL; 4191 int split = 0, removed = 0; 4192 4193 #ifdef CONFIG_MEMCG 4194 if (sc->memcg) 4195 ds_queue = &sc->memcg->deferred_split_queue; 4196 #endif 4197 4198 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 4199 /* Take pin on all head pages to avoid freeing them under us */ 4200 list_for_each_entry_safe(folio, next, &ds_queue->split_queue, 4201 _deferred_list) { 4202 if (folio_try_get(folio)) { 4203 list_move(&folio->_deferred_list, &list); 4204 } else { 4205 /* We lost race with folio_put() */ 4206 if (folio_test_partially_mapped(folio)) { 4207 folio_clear_partially_mapped(folio); 4208 mod_mthp_stat(folio_order(folio), 4209 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 4210 } 4211 list_del_init(&folio->_deferred_list); 4212 ds_queue->split_queue_len--; 4213 } 4214 if (!--sc->nr_to_scan) 4215 break; 4216 } 4217 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 4218 4219 list_for_each_entry_safe(folio, next, &list, _deferred_list) { 4220 bool did_split = false; 4221 bool underused = false; 4222 4223 if (!folio_test_partially_mapped(folio)) { 4224 underused = thp_underused(folio); 4225 if (!underused) 4226 goto next; 4227 } 4228 if (!folio_trylock(folio)) 4229 goto next; 4230 if (!split_folio(folio)) { 4231 did_split = true; 4232 if (underused) 4233 count_vm_event(THP_UNDERUSED_SPLIT_PAGE); 4234 split++; 4235 } 4236 folio_unlock(folio); 4237 next: 4238 /* 4239 * split_folio() removes folio from list on success. 4240 * Only add back to the queue if folio is partially mapped. 4241 * If thp_underused returns false, or if split_folio fails 4242 * in the case it was underused, then consider it used and 4243 * don't add it back to split_queue. 4244 */ 4245 if (did_split) { 4246 ; /* folio already removed from list */ 4247 } else if (!folio_test_partially_mapped(folio)) { 4248 list_del_init(&folio->_deferred_list); 4249 removed++; 4250 } else { 4251 /* 4252 * That unlocked list_del_init() above would be unsafe, 4253 * unless its folio is separated from any earlier folios 4254 * left on the list (which may be concurrently unqueued) 4255 * by one safe folio with refcount still raised. 4256 */ 4257 swap(folio, prev); 4258 } 4259 if (folio) 4260 folio_put(folio); 4261 } 4262 4263 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 4264 list_splice_tail(&list, &ds_queue->split_queue); 4265 ds_queue->split_queue_len -= removed; 4266 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 4267 4268 if (prev) 4269 folio_put(prev); 4270 4271 /* 4272 * Stop shrinker if we didn't split any page, but the queue is empty. 4273 * This can happen if pages were freed under us. 4274 */ 4275 if (!split && list_empty(&ds_queue->split_queue)) 4276 return SHRINK_STOP; 4277 return split; 4278 } 4279 4280 #ifdef CONFIG_DEBUG_FS 4281 static void split_huge_pages_all(void) 4282 { 4283 struct zone *zone; 4284 struct page *page; 4285 struct folio *folio; 4286 unsigned long pfn, max_zone_pfn; 4287 unsigned long total = 0, split = 0; 4288 4289 pr_debug("Split all THPs\n"); 4290 for_each_zone(zone) { 4291 if (!managed_zone(zone)) 4292 continue; 4293 max_zone_pfn = zone_end_pfn(zone); 4294 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 4295 int nr_pages; 4296 4297 page = pfn_to_online_page(pfn); 4298 if (!page || PageTail(page)) 4299 continue; 4300 folio = page_folio(page); 4301 if (!folio_try_get(folio)) 4302 continue; 4303 4304 if (unlikely(page_folio(page) != folio)) 4305 goto next; 4306 4307 if (zone != folio_zone(folio)) 4308 goto next; 4309 4310 if (!folio_test_large(folio) 4311 || folio_test_hugetlb(folio) 4312 || !folio_test_lru(folio)) 4313 goto next; 4314 4315 total++; 4316 folio_lock(folio); 4317 nr_pages = folio_nr_pages(folio); 4318 if (!split_folio(folio)) 4319 split++; 4320 pfn += nr_pages - 1; 4321 folio_unlock(folio); 4322 next: 4323 folio_put(folio); 4324 cond_resched(); 4325 } 4326 } 4327 4328 pr_debug("%lu of %lu THP split\n", split, total); 4329 } 4330 4331 static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma) 4332 { 4333 return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) || 4334 is_vm_hugetlb_page(vma); 4335 } 4336 4337 static int split_huge_pages_pid(int pid, unsigned long vaddr_start, 4338 unsigned long vaddr_end, unsigned int new_order, 4339 long in_folio_offset) 4340 { 4341 int ret = 0; 4342 struct task_struct *task; 4343 struct mm_struct *mm; 4344 unsigned long total = 0, split = 0; 4345 unsigned long addr; 4346 4347 vaddr_start &= PAGE_MASK; 4348 vaddr_end &= PAGE_MASK; 4349 4350 task = find_get_task_by_vpid(pid); 4351 if (!task) { 4352 ret = -ESRCH; 4353 goto out; 4354 } 4355 4356 /* Find the mm_struct */ 4357 mm = get_task_mm(task); 4358 put_task_struct(task); 4359 4360 if (!mm) { 4361 ret = -EINVAL; 4362 goto out; 4363 } 4364 4365 pr_debug("Split huge pages in pid: %d, vaddr: [0x%lx - 0x%lx]\n", 4366 pid, vaddr_start, vaddr_end); 4367 4368 mmap_read_lock(mm); 4369 /* 4370 * always increase addr by PAGE_SIZE, since we could have a PTE page 4371 * table filled with PTE-mapped THPs, each of which is distinct. 4372 */ 4373 for (addr = vaddr_start; addr < vaddr_end; addr += PAGE_SIZE) { 4374 struct vm_area_struct *vma = vma_lookup(mm, addr); 4375 struct folio_walk fw; 4376 struct folio *folio; 4377 struct address_space *mapping; 4378 unsigned int target_order = new_order; 4379 4380 if (!vma) 4381 break; 4382 4383 /* skip special VMA and hugetlb VMA */ 4384 if (vma_not_suitable_for_thp_split(vma)) { 4385 addr = vma->vm_end; 4386 continue; 4387 } 4388 4389 folio = folio_walk_start(&fw, vma, addr, 0); 4390 if (!folio) 4391 continue; 4392 4393 if (!is_transparent_hugepage(folio)) 4394 goto next; 4395 4396 if (!folio_test_anon(folio)) { 4397 mapping = folio->mapping; 4398 target_order = max(new_order, 4399 mapping_min_folio_order(mapping)); 4400 } 4401 4402 if (target_order >= folio_order(folio)) 4403 goto next; 4404 4405 total++; 4406 /* 4407 * For folios with private, split_huge_page_to_list_to_order() 4408 * will try to drop it before split and then check if the folio 4409 * can be split or not. So skip the check here. 4410 */ 4411 if (!folio_test_private(folio) && 4412 !can_split_folio(folio, 0, NULL)) 4413 goto next; 4414 4415 if (!folio_trylock(folio)) 4416 goto next; 4417 folio_get(folio); 4418 folio_walk_end(&fw, vma); 4419 4420 if (!folio_test_anon(folio) && folio->mapping != mapping) 4421 goto unlock; 4422 4423 if (in_folio_offset < 0 || 4424 in_folio_offset >= folio_nr_pages(folio)) { 4425 if (!split_folio_to_order(folio, target_order)) 4426 split++; 4427 } else { 4428 struct page *split_at = folio_page(folio, 4429 in_folio_offset); 4430 if (!folio_split(folio, target_order, split_at, NULL)) 4431 split++; 4432 } 4433 4434 unlock: 4435 4436 folio_unlock(folio); 4437 folio_put(folio); 4438 4439 cond_resched(); 4440 continue; 4441 next: 4442 folio_walk_end(&fw, vma); 4443 cond_resched(); 4444 } 4445 mmap_read_unlock(mm); 4446 mmput(mm); 4447 4448 pr_debug("%lu of %lu THP split\n", split, total); 4449 4450 out: 4451 return ret; 4452 } 4453 4454 static int split_huge_pages_in_file(const char *file_path, pgoff_t off_start, 4455 pgoff_t off_end, unsigned int new_order, 4456 long in_folio_offset) 4457 { 4458 struct filename *file; 4459 struct file *candidate; 4460 struct address_space *mapping; 4461 int ret = -EINVAL; 4462 pgoff_t index; 4463 int nr_pages = 1; 4464 unsigned long total = 0, split = 0; 4465 unsigned int min_order; 4466 unsigned int target_order; 4467 4468 file = getname_kernel(file_path); 4469 if (IS_ERR(file)) 4470 return ret; 4471 4472 candidate = file_open_name(file, O_RDONLY, 0); 4473 if (IS_ERR(candidate)) 4474 goto out; 4475 4476 pr_debug("split file-backed THPs in file: %s, page offset: [0x%lx - 0x%lx]\n", 4477 file_path, off_start, off_end); 4478 4479 mapping = candidate->f_mapping; 4480 min_order = mapping_min_folio_order(mapping); 4481 target_order = max(new_order, min_order); 4482 4483 for (index = off_start; index < off_end; index += nr_pages) { 4484 struct folio *folio = filemap_get_folio(mapping, index); 4485 4486 nr_pages = 1; 4487 if (IS_ERR(folio)) 4488 continue; 4489 4490 if (!folio_test_large(folio)) 4491 goto next; 4492 4493 total++; 4494 nr_pages = folio_nr_pages(folio); 4495 4496 if (target_order >= folio_order(folio)) 4497 goto next; 4498 4499 if (!folio_trylock(folio)) 4500 goto next; 4501 4502 if (folio->mapping != mapping) 4503 goto unlock; 4504 4505 if (in_folio_offset < 0 || in_folio_offset >= nr_pages) { 4506 if (!split_folio_to_order(folio, target_order)) 4507 split++; 4508 } else { 4509 struct page *split_at = folio_page(folio, 4510 in_folio_offset); 4511 if (!folio_split(folio, target_order, split_at, NULL)) 4512 split++; 4513 } 4514 4515 unlock: 4516 folio_unlock(folio); 4517 next: 4518 folio_put(folio); 4519 cond_resched(); 4520 } 4521 4522 filp_close(candidate, NULL); 4523 ret = 0; 4524 4525 pr_debug("%lu of %lu file-backed THP split\n", split, total); 4526 out: 4527 putname(file); 4528 return ret; 4529 } 4530 4531 #define MAX_INPUT_BUF_SZ 255 4532 4533 static ssize_t split_huge_pages_write(struct file *file, const char __user *buf, 4534 size_t count, loff_t *ppops) 4535 { 4536 static DEFINE_MUTEX(split_debug_mutex); 4537 ssize_t ret; 4538 /* 4539 * hold pid, start_vaddr, end_vaddr, new_order or 4540 * file_path, off_start, off_end, new_order 4541 */ 4542 char input_buf[MAX_INPUT_BUF_SZ]; 4543 int pid; 4544 unsigned long vaddr_start, vaddr_end; 4545 unsigned int new_order = 0; 4546 long in_folio_offset = -1; 4547 4548 ret = mutex_lock_interruptible(&split_debug_mutex); 4549 if (ret) 4550 return ret; 4551 4552 ret = -EFAULT; 4553 4554 memset(input_buf, 0, MAX_INPUT_BUF_SZ); 4555 if (copy_from_user(input_buf, buf, min_t(size_t, count, MAX_INPUT_BUF_SZ))) 4556 goto out; 4557 4558 input_buf[MAX_INPUT_BUF_SZ - 1] = '\0'; 4559 4560 if (input_buf[0] == '/') { 4561 char *tok; 4562 char *tok_buf = input_buf; 4563 char file_path[MAX_INPUT_BUF_SZ]; 4564 pgoff_t off_start = 0, off_end = 0; 4565 size_t input_len = strlen(input_buf); 4566 4567 tok = strsep(&tok_buf, ","); 4568 if (tok && tok_buf) { 4569 strscpy(file_path, tok); 4570 } else { 4571 ret = -EINVAL; 4572 goto out; 4573 } 4574 4575 ret = sscanf(tok_buf, "0x%lx,0x%lx,%d,%ld", &off_start, &off_end, 4576 &new_order, &in_folio_offset); 4577 if (ret != 2 && ret != 3 && ret != 4) { 4578 ret = -EINVAL; 4579 goto out; 4580 } 4581 ret = split_huge_pages_in_file(file_path, off_start, off_end, 4582 new_order, in_folio_offset); 4583 if (!ret) 4584 ret = input_len; 4585 4586 goto out; 4587 } 4588 4589 ret = sscanf(input_buf, "%d,0x%lx,0x%lx,%d,%ld", &pid, &vaddr_start, 4590 &vaddr_end, &new_order, &in_folio_offset); 4591 if (ret == 1 && pid == 1) { 4592 split_huge_pages_all(); 4593 ret = strlen(input_buf); 4594 goto out; 4595 } else if (ret != 3 && ret != 4 && ret != 5) { 4596 ret = -EINVAL; 4597 goto out; 4598 } 4599 4600 ret = split_huge_pages_pid(pid, vaddr_start, vaddr_end, new_order, 4601 in_folio_offset); 4602 if (!ret) 4603 ret = strlen(input_buf); 4604 out: 4605 mutex_unlock(&split_debug_mutex); 4606 return ret; 4607 4608 } 4609 4610 static const struct file_operations split_huge_pages_fops = { 4611 .owner = THIS_MODULE, 4612 .write = split_huge_pages_write, 4613 }; 4614 4615 static int __init split_huge_pages_debugfs(void) 4616 { 4617 debugfs_create_file("split_huge_pages", 0200, NULL, NULL, 4618 &split_huge_pages_fops); 4619 return 0; 4620 } 4621 late_initcall(split_huge_pages_debugfs); 4622 #endif 4623 4624 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 4625 int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 4626 struct page *page) 4627 { 4628 struct folio *folio = page_folio(page); 4629 struct vm_area_struct *vma = pvmw->vma; 4630 struct mm_struct *mm = vma->vm_mm; 4631 unsigned long address = pvmw->address; 4632 bool anon_exclusive; 4633 pmd_t pmdval; 4634 swp_entry_t entry; 4635 pmd_t pmdswp; 4636 4637 if (!(pvmw->pmd && !pvmw->pte)) 4638 return 0; 4639 4640 flush_cache_range(vma, address, address + HPAGE_PMD_SIZE); 4641 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); 4642 4643 /* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. */ 4644 anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(page); 4645 if (anon_exclusive && folio_try_share_anon_rmap_pmd(folio, page)) { 4646 set_pmd_at(mm, address, pvmw->pmd, pmdval); 4647 return -EBUSY; 4648 } 4649 4650 if (pmd_dirty(pmdval)) 4651 folio_mark_dirty(folio); 4652 if (pmd_write(pmdval)) 4653 entry = make_writable_migration_entry(page_to_pfn(page)); 4654 else if (anon_exclusive) 4655 entry = make_readable_exclusive_migration_entry(page_to_pfn(page)); 4656 else 4657 entry = make_readable_migration_entry(page_to_pfn(page)); 4658 if (pmd_young(pmdval)) 4659 entry = make_migration_entry_young(entry); 4660 if (pmd_dirty(pmdval)) 4661 entry = make_migration_entry_dirty(entry); 4662 pmdswp = swp_entry_to_pmd(entry); 4663 if (pmd_soft_dirty(pmdval)) 4664 pmdswp = pmd_swp_mksoft_dirty(pmdswp); 4665 if (pmd_uffd_wp(pmdval)) 4666 pmdswp = pmd_swp_mkuffd_wp(pmdswp); 4667 set_pmd_at(mm, address, pvmw->pmd, pmdswp); 4668 folio_remove_rmap_pmd(folio, page, vma); 4669 folio_put(folio); 4670 trace_set_migration_pmd(address, pmd_val(pmdswp)); 4671 4672 return 0; 4673 } 4674 4675 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) 4676 { 4677 struct folio *folio = page_folio(new); 4678 struct vm_area_struct *vma = pvmw->vma; 4679 struct mm_struct *mm = vma->vm_mm; 4680 unsigned long address = pvmw->address; 4681 unsigned long haddr = address & HPAGE_PMD_MASK; 4682 pmd_t pmde; 4683 swp_entry_t entry; 4684 4685 if (!(pvmw->pmd && !pvmw->pte)) 4686 return; 4687 4688 entry = pmd_to_swp_entry(*pvmw->pmd); 4689 folio_get(folio); 4690 pmde = folio_mk_pmd(folio, READ_ONCE(vma->vm_page_prot)); 4691 if (pmd_swp_soft_dirty(*pvmw->pmd)) 4692 pmde = pmd_mksoft_dirty(pmde); 4693 if (is_writable_migration_entry(entry)) 4694 pmde = pmd_mkwrite(pmde, vma); 4695 if (pmd_swp_uffd_wp(*pvmw->pmd)) 4696 pmde = pmd_mkuffd_wp(pmde); 4697 if (!is_migration_entry_young(entry)) 4698 pmde = pmd_mkold(pmde); 4699 /* NOTE: this may contain setting soft-dirty on some archs */ 4700 if (folio_test_dirty(folio) && is_migration_entry_dirty(entry)) 4701 pmde = pmd_mkdirty(pmde); 4702 4703 if (folio_test_anon(folio)) { 4704 rmap_t rmap_flags = RMAP_NONE; 4705 4706 if (!is_readable_migration_entry(entry)) 4707 rmap_flags |= RMAP_EXCLUSIVE; 4708 4709 folio_add_anon_rmap_pmd(folio, new, vma, haddr, rmap_flags); 4710 } else { 4711 folio_add_file_rmap_pmd(folio, new, vma); 4712 } 4713 VM_BUG_ON(pmd_write(pmde) && folio_test_anon(folio) && !PageAnonExclusive(new)); 4714 set_pmd_at(mm, haddr, pvmw->pmd, pmde); 4715 4716 /* No need to invalidate - it was non-present before */ 4717 update_mmu_cache_pmd(vma, address, pvmw->pmd); 4718 trace_remove_migration_pmd(address, pmd_val(pmde)); 4719 } 4720 #endif 4721