1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2009 Red Hat, Inc. 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/mm.h> 9 #include <linux/sched.h> 10 #include <linux/sched/mm.h> 11 #include <linux/sched/numa_balancing.h> 12 #include <linux/highmem.h> 13 #include <linux/hugetlb.h> 14 #include <linux/mmu_notifier.h> 15 #include <linux/rmap.h> 16 #include <linux/swap.h> 17 #include <linux/shrinker.h> 18 #include <linux/mm_inline.h> 19 #include <linux/swapops.h> 20 #include <linux/backing-dev.h> 21 #include <linux/dax.h> 22 #include <linux/mm_types.h> 23 #include <linux/khugepaged.h> 24 #include <linux/freezer.h> 25 #include <linux/mman.h> 26 #include <linux/memremap.h> 27 #include <linux/pagemap.h> 28 #include <linux/debugfs.h> 29 #include <linux/migrate.h> 30 #include <linux/hashtable.h> 31 #include <linux/userfaultfd_k.h> 32 #include <linux/page_idle.h> 33 #include <linux/shmem_fs.h> 34 #include <linux/oom.h> 35 #include <linux/numa.h> 36 #include <linux/page_owner.h> 37 #include <linux/sched/sysctl.h> 38 #include <linux/memory-tiers.h> 39 #include <linux/compat.h> 40 #include <linux/pgalloc_tag.h> 41 #include <linux/pagewalk.h> 42 43 #include <asm/tlb.h> 44 #include <asm/pgalloc.h> 45 #include "internal.h" 46 #include "swap.h" 47 48 #define CREATE_TRACE_POINTS 49 #include <trace/events/thp.h> 50 51 /* 52 * By default, transparent hugepage support is disabled in order to avoid 53 * risking an increased memory footprint for applications that are not 54 * guaranteed to benefit from it. When transparent hugepage support is 55 * enabled, it is for all mappings, and khugepaged scans all mappings. 56 * Defrag is invoked by khugepaged hugepage allocations and by page faults 57 * for all hugepage allocations. 58 */ 59 unsigned long transparent_hugepage_flags __read_mostly = 60 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS 61 (1<<TRANSPARENT_HUGEPAGE_FLAG)| 62 #endif 63 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE 64 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| 65 #endif 66 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)| 67 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| 68 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 69 70 static struct shrinker *deferred_split_shrinker; 71 static unsigned long deferred_split_count(struct shrinker *shrink, 72 struct shrink_control *sc); 73 static unsigned long deferred_split_scan(struct shrinker *shrink, 74 struct shrink_control *sc); 75 static bool split_underused_thp = true; 76 77 static atomic_t huge_zero_refcount; 78 struct folio *huge_zero_folio __read_mostly; 79 unsigned long huge_zero_pfn __read_mostly = ~0UL; 80 unsigned long huge_anon_orders_always __read_mostly; 81 unsigned long huge_anon_orders_madvise __read_mostly; 82 unsigned long huge_anon_orders_inherit __read_mostly; 83 static bool anon_orders_configured __initdata; 84 85 static inline bool file_thp_enabled(struct vm_area_struct *vma) 86 { 87 struct inode *inode; 88 89 if (!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS)) 90 return false; 91 92 if (!vma->vm_file) 93 return false; 94 95 inode = file_inode(vma->vm_file); 96 97 return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); 98 } 99 100 unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, 101 vm_flags_t vm_flags, 102 enum tva_type type, 103 unsigned long orders) 104 { 105 const bool smaps = type == TVA_SMAPS; 106 const bool in_pf = type == TVA_PAGEFAULT; 107 const bool forced_collapse = type == TVA_FORCED_COLLAPSE; 108 unsigned long supported_orders; 109 110 /* Check the intersection of requested and supported orders. */ 111 if (vma_is_anonymous(vma)) 112 supported_orders = THP_ORDERS_ALL_ANON; 113 else if (vma_is_special_huge(vma)) 114 supported_orders = THP_ORDERS_ALL_SPECIAL; 115 else 116 supported_orders = THP_ORDERS_ALL_FILE_DEFAULT; 117 118 orders &= supported_orders; 119 if (!orders) 120 return 0; 121 122 if (!vma->vm_mm) /* vdso */ 123 return 0; 124 125 if (thp_disabled_by_hw() || vma_thp_disabled(vma, vm_flags, forced_collapse)) 126 return 0; 127 128 /* khugepaged doesn't collapse DAX vma, but page fault is fine. */ 129 if (vma_is_dax(vma)) 130 return in_pf ? orders : 0; 131 132 /* 133 * khugepaged special VMA and hugetlb VMA. 134 * Must be checked after dax since some dax mappings may have 135 * VM_MIXEDMAP set. 136 */ 137 if (!in_pf && !smaps && (vm_flags & VM_NO_KHUGEPAGED)) 138 return 0; 139 140 /* 141 * Check alignment for file vma and size for both file and anon vma by 142 * filtering out the unsuitable orders. 143 * 144 * Skip the check for page fault. Huge fault does the check in fault 145 * handlers. 146 */ 147 if (!in_pf) { 148 int order = highest_order(orders); 149 unsigned long addr; 150 151 while (orders) { 152 addr = vma->vm_end - (PAGE_SIZE << order); 153 if (thp_vma_suitable_order(vma, addr, order)) 154 break; 155 order = next_order(&orders, order); 156 } 157 158 if (!orders) 159 return 0; 160 } 161 162 /* 163 * Enabled via shmem mount options or sysfs settings. 164 * Must be done before hugepage flags check since shmem has its 165 * own flags. 166 */ 167 if (!in_pf && shmem_file(vma->vm_file)) 168 return orders & shmem_allowable_huge_orders(file_inode(vma->vm_file), 169 vma, vma->vm_pgoff, 0, 170 forced_collapse); 171 172 if (!vma_is_anonymous(vma)) { 173 /* 174 * Enforce THP collapse requirements as necessary. Anonymous vmas 175 * were already handled in thp_vma_allowable_orders(). 176 */ 177 if (!forced_collapse && 178 (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) && 179 !hugepage_global_always()))) 180 return 0; 181 182 /* 183 * Trust that ->huge_fault() handlers know what they are doing 184 * in fault path. 185 */ 186 if (((in_pf || smaps)) && vma->vm_ops->huge_fault) 187 return orders; 188 /* Only regular file is valid in collapse path */ 189 if (((!in_pf || smaps)) && file_thp_enabled(vma)) 190 return orders; 191 return 0; 192 } 193 194 if (vma_is_temporary_stack(vma)) 195 return 0; 196 197 /* 198 * THPeligible bit of smaps should show 1 for proper VMAs even 199 * though anon_vma is not initialized yet. 200 * 201 * Allow page fault since anon_vma may be not initialized until 202 * the first page fault. 203 */ 204 if (!vma->anon_vma) 205 return (smaps || in_pf) ? orders : 0; 206 207 return orders; 208 } 209 210 static bool get_huge_zero_folio(void) 211 { 212 struct folio *zero_folio; 213 retry: 214 if (likely(atomic_inc_not_zero(&huge_zero_refcount))) 215 return true; 216 217 zero_folio = folio_alloc((GFP_TRANSHUGE | __GFP_ZERO | __GFP_ZEROTAGS) & 218 ~__GFP_MOVABLE, 219 HPAGE_PMD_ORDER); 220 if (!zero_folio) { 221 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); 222 return false; 223 } 224 /* Ensure zero folio won't have large_rmappable flag set. */ 225 folio_clear_large_rmappable(zero_folio); 226 preempt_disable(); 227 if (cmpxchg(&huge_zero_folio, NULL, zero_folio)) { 228 preempt_enable(); 229 folio_put(zero_folio); 230 goto retry; 231 } 232 WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio)); 233 234 /* We take additional reference here. It will be put back by shrinker */ 235 atomic_set(&huge_zero_refcount, 2); 236 preempt_enable(); 237 count_vm_event(THP_ZERO_PAGE_ALLOC); 238 return true; 239 } 240 241 static void put_huge_zero_folio(void) 242 { 243 /* 244 * Counter should never go to zero here. Only shrinker can put 245 * last reference. 246 */ 247 BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); 248 } 249 250 struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) 251 { 252 if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) 253 return huge_zero_folio; 254 255 if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm)) 256 return READ_ONCE(huge_zero_folio); 257 258 if (!get_huge_zero_folio()) 259 return NULL; 260 261 if (mm_flags_test_and_set(MMF_HUGE_ZERO_FOLIO, mm)) 262 put_huge_zero_folio(); 263 264 return READ_ONCE(huge_zero_folio); 265 } 266 267 void mm_put_huge_zero_folio(struct mm_struct *mm) 268 { 269 if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) 270 return; 271 272 if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm)) 273 put_huge_zero_folio(); 274 } 275 276 static unsigned long shrink_huge_zero_folio_count(struct shrinker *shrink, 277 struct shrink_control *sc) 278 { 279 /* we can free zero page only if last reference remains */ 280 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; 281 } 282 283 static unsigned long shrink_huge_zero_folio_scan(struct shrinker *shrink, 284 struct shrink_control *sc) 285 { 286 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { 287 struct folio *zero_folio = xchg(&huge_zero_folio, NULL); 288 BUG_ON(zero_folio == NULL); 289 WRITE_ONCE(huge_zero_pfn, ~0UL); 290 folio_put(zero_folio); 291 return HPAGE_PMD_NR; 292 } 293 294 return 0; 295 } 296 297 static struct shrinker *huge_zero_folio_shrinker; 298 299 #ifdef CONFIG_SYSFS 300 static ssize_t enabled_show(struct kobject *kobj, 301 struct kobj_attribute *attr, char *buf) 302 { 303 const char *output; 304 305 if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags)) 306 output = "[always] madvise never"; 307 else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 308 &transparent_hugepage_flags)) 309 output = "always [madvise] never"; 310 else 311 output = "always madvise [never]"; 312 313 return sysfs_emit(buf, "%s\n", output); 314 } 315 316 static ssize_t enabled_store(struct kobject *kobj, 317 struct kobj_attribute *attr, 318 const char *buf, size_t count) 319 { 320 ssize_t ret = count; 321 322 if (sysfs_streq(buf, "always")) { 323 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 324 set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 325 } else if (sysfs_streq(buf, "madvise")) { 326 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 327 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 328 } else if (sysfs_streq(buf, "never")) { 329 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 330 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 331 } else 332 ret = -EINVAL; 333 334 if (ret > 0) { 335 int err = start_stop_khugepaged(); 336 if (err) 337 ret = err; 338 } 339 return ret; 340 } 341 342 static struct kobj_attribute enabled_attr = __ATTR_RW(enabled); 343 344 ssize_t single_hugepage_flag_show(struct kobject *kobj, 345 struct kobj_attribute *attr, char *buf, 346 enum transparent_hugepage_flag flag) 347 { 348 return sysfs_emit(buf, "%d\n", 349 !!test_bit(flag, &transparent_hugepage_flags)); 350 } 351 352 ssize_t single_hugepage_flag_store(struct kobject *kobj, 353 struct kobj_attribute *attr, 354 const char *buf, size_t count, 355 enum transparent_hugepage_flag flag) 356 { 357 unsigned long value; 358 int ret; 359 360 ret = kstrtoul(buf, 10, &value); 361 if (ret < 0) 362 return ret; 363 if (value > 1) 364 return -EINVAL; 365 366 if (value) 367 set_bit(flag, &transparent_hugepage_flags); 368 else 369 clear_bit(flag, &transparent_hugepage_flags); 370 371 return count; 372 } 373 374 static ssize_t defrag_show(struct kobject *kobj, 375 struct kobj_attribute *attr, char *buf) 376 { 377 const char *output; 378 379 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, 380 &transparent_hugepage_flags)) 381 output = "[always] defer defer+madvise madvise never"; 382 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, 383 &transparent_hugepage_flags)) 384 output = "always [defer] defer+madvise madvise never"; 385 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, 386 &transparent_hugepage_flags)) 387 output = "always defer [defer+madvise] madvise never"; 388 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, 389 &transparent_hugepage_flags)) 390 output = "always defer defer+madvise [madvise] never"; 391 else 392 output = "always defer defer+madvise madvise [never]"; 393 394 return sysfs_emit(buf, "%s\n", output); 395 } 396 397 static ssize_t defrag_store(struct kobject *kobj, 398 struct kobj_attribute *attr, 399 const char *buf, size_t count) 400 { 401 if (sysfs_streq(buf, "always")) { 402 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 403 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 404 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 405 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 406 } else if (sysfs_streq(buf, "defer+madvise")) { 407 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 408 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 409 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 410 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 411 } else if (sysfs_streq(buf, "defer")) { 412 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 413 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 414 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 415 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 416 } else if (sysfs_streq(buf, "madvise")) { 417 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 418 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 419 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 420 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 421 } else if (sysfs_streq(buf, "never")) { 422 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 423 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 424 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 425 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 426 } else 427 return -EINVAL; 428 429 return count; 430 } 431 static struct kobj_attribute defrag_attr = __ATTR_RW(defrag); 432 433 static ssize_t use_zero_page_show(struct kobject *kobj, 434 struct kobj_attribute *attr, char *buf) 435 { 436 return single_hugepage_flag_show(kobj, attr, buf, 437 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 438 } 439 static ssize_t use_zero_page_store(struct kobject *kobj, 440 struct kobj_attribute *attr, const char *buf, size_t count) 441 { 442 return single_hugepage_flag_store(kobj, attr, buf, count, 443 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 444 } 445 static struct kobj_attribute use_zero_page_attr = __ATTR_RW(use_zero_page); 446 447 static ssize_t hpage_pmd_size_show(struct kobject *kobj, 448 struct kobj_attribute *attr, char *buf) 449 { 450 return sysfs_emit(buf, "%lu\n", HPAGE_PMD_SIZE); 451 } 452 static struct kobj_attribute hpage_pmd_size_attr = 453 __ATTR_RO(hpage_pmd_size); 454 455 static ssize_t split_underused_thp_show(struct kobject *kobj, 456 struct kobj_attribute *attr, char *buf) 457 { 458 return sysfs_emit(buf, "%d\n", split_underused_thp); 459 } 460 461 static ssize_t split_underused_thp_store(struct kobject *kobj, 462 struct kobj_attribute *attr, 463 const char *buf, size_t count) 464 { 465 int err = kstrtobool(buf, &split_underused_thp); 466 467 if (err < 0) 468 return err; 469 470 return count; 471 } 472 473 static struct kobj_attribute split_underused_thp_attr = __ATTR( 474 shrink_underused, 0644, split_underused_thp_show, split_underused_thp_store); 475 476 static struct attribute *hugepage_attr[] = { 477 &enabled_attr.attr, 478 &defrag_attr.attr, 479 &use_zero_page_attr.attr, 480 &hpage_pmd_size_attr.attr, 481 #ifdef CONFIG_SHMEM 482 &shmem_enabled_attr.attr, 483 #endif 484 &split_underused_thp_attr.attr, 485 NULL, 486 }; 487 488 static const struct attribute_group hugepage_attr_group = { 489 .attrs = hugepage_attr, 490 }; 491 492 static void hugepage_exit_sysfs(struct kobject *hugepage_kobj); 493 static void thpsize_release(struct kobject *kobj); 494 static DEFINE_SPINLOCK(huge_anon_orders_lock); 495 static LIST_HEAD(thpsize_list); 496 497 static ssize_t anon_enabled_show(struct kobject *kobj, 498 struct kobj_attribute *attr, char *buf) 499 { 500 int order = to_thpsize(kobj)->order; 501 const char *output; 502 503 if (test_bit(order, &huge_anon_orders_always)) 504 output = "[always] inherit madvise never"; 505 else if (test_bit(order, &huge_anon_orders_inherit)) 506 output = "always [inherit] madvise never"; 507 else if (test_bit(order, &huge_anon_orders_madvise)) 508 output = "always inherit [madvise] never"; 509 else 510 output = "always inherit madvise [never]"; 511 512 return sysfs_emit(buf, "%s\n", output); 513 } 514 515 static ssize_t anon_enabled_store(struct kobject *kobj, 516 struct kobj_attribute *attr, 517 const char *buf, size_t count) 518 { 519 int order = to_thpsize(kobj)->order; 520 ssize_t ret = count; 521 522 if (sysfs_streq(buf, "always")) { 523 spin_lock(&huge_anon_orders_lock); 524 clear_bit(order, &huge_anon_orders_inherit); 525 clear_bit(order, &huge_anon_orders_madvise); 526 set_bit(order, &huge_anon_orders_always); 527 spin_unlock(&huge_anon_orders_lock); 528 } else if (sysfs_streq(buf, "inherit")) { 529 spin_lock(&huge_anon_orders_lock); 530 clear_bit(order, &huge_anon_orders_always); 531 clear_bit(order, &huge_anon_orders_madvise); 532 set_bit(order, &huge_anon_orders_inherit); 533 spin_unlock(&huge_anon_orders_lock); 534 } else if (sysfs_streq(buf, "madvise")) { 535 spin_lock(&huge_anon_orders_lock); 536 clear_bit(order, &huge_anon_orders_always); 537 clear_bit(order, &huge_anon_orders_inherit); 538 set_bit(order, &huge_anon_orders_madvise); 539 spin_unlock(&huge_anon_orders_lock); 540 } else if (sysfs_streq(buf, "never")) { 541 spin_lock(&huge_anon_orders_lock); 542 clear_bit(order, &huge_anon_orders_always); 543 clear_bit(order, &huge_anon_orders_inherit); 544 clear_bit(order, &huge_anon_orders_madvise); 545 spin_unlock(&huge_anon_orders_lock); 546 } else 547 ret = -EINVAL; 548 549 if (ret > 0) { 550 int err; 551 552 err = start_stop_khugepaged(); 553 if (err) 554 ret = err; 555 } 556 return ret; 557 } 558 559 static struct kobj_attribute anon_enabled_attr = 560 __ATTR(enabled, 0644, anon_enabled_show, anon_enabled_store); 561 562 static struct attribute *anon_ctrl_attrs[] = { 563 &anon_enabled_attr.attr, 564 NULL, 565 }; 566 567 static const struct attribute_group anon_ctrl_attr_grp = { 568 .attrs = anon_ctrl_attrs, 569 }; 570 571 static struct attribute *file_ctrl_attrs[] = { 572 #ifdef CONFIG_SHMEM 573 &thpsize_shmem_enabled_attr.attr, 574 #endif 575 NULL, 576 }; 577 578 static const struct attribute_group file_ctrl_attr_grp = { 579 .attrs = file_ctrl_attrs, 580 }; 581 582 static struct attribute *any_ctrl_attrs[] = { 583 NULL, 584 }; 585 586 static const struct attribute_group any_ctrl_attr_grp = { 587 .attrs = any_ctrl_attrs, 588 }; 589 590 static const struct kobj_type thpsize_ktype = { 591 .release = &thpsize_release, 592 .sysfs_ops = &kobj_sysfs_ops, 593 }; 594 595 DEFINE_PER_CPU(struct mthp_stat, mthp_stats) = {{{0}}}; 596 597 static unsigned long sum_mthp_stat(int order, enum mthp_stat_item item) 598 { 599 unsigned long sum = 0; 600 int cpu; 601 602 for_each_possible_cpu(cpu) { 603 struct mthp_stat *this = &per_cpu(mthp_stats, cpu); 604 605 sum += this->stats[order][item]; 606 } 607 608 return sum; 609 } 610 611 #define DEFINE_MTHP_STAT_ATTR(_name, _index) \ 612 static ssize_t _name##_show(struct kobject *kobj, \ 613 struct kobj_attribute *attr, char *buf) \ 614 { \ 615 int order = to_thpsize(kobj)->order; \ 616 \ 617 return sysfs_emit(buf, "%lu\n", sum_mthp_stat(order, _index)); \ 618 } \ 619 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 620 621 DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC); 622 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK); 623 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); 624 DEFINE_MTHP_STAT_ATTR(zswpout, MTHP_STAT_ZSWPOUT); 625 DEFINE_MTHP_STAT_ATTR(swpin, MTHP_STAT_SWPIN); 626 DEFINE_MTHP_STAT_ATTR(swpin_fallback, MTHP_STAT_SWPIN_FALLBACK); 627 DEFINE_MTHP_STAT_ATTR(swpin_fallback_charge, MTHP_STAT_SWPIN_FALLBACK_CHARGE); 628 DEFINE_MTHP_STAT_ATTR(swpout, MTHP_STAT_SWPOUT); 629 DEFINE_MTHP_STAT_ATTR(swpout_fallback, MTHP_STAT_SWPOUT_FALLBACK); 630 #ifdef CONFIG_SHMEM 631 DEFINE_MTHP_STAT_ATTR(shmem_alloc, MTHP_STAT_SHMEM_ALLOC); 632 DEFINE_MTHP_STAT_ATTR(shmem_fallback, MTHP_STAT_SHMEM_FALLBACK); 633 DEFINE_MTHP_STAT_ATTR(shmem_fallback_charge, MTHP_STAT_SHMEM_FALLBACK_CHARGE); 634 #endif 635 DEFINE_MTHP_STAT_ATTR(split, MTHP_STAT_SPLIT); 636 DEFINE_MTHP_STAT_ATTR(split_failed, MTHP_STAT_SPLIT_FAILED); 637 DEFINE_MTHP_STAT_ATTR(split_deferred, MTHP_STAT_SPLIT_DEFERRED); 638 DEFINE_MTHP_STAT_ATTR(nr_anon, MTHP_STAT_NR_ANON); 639 DEFINE_MTHP_STAT_ATTR(nr_anon_partially_mapped, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED); 640 641 static struct attribute *anon_stats_attrs[] = { 642 &anon_fault_alloc_attr.attr, 643 &anon_fault_fallback_attr.attr, 644 &anon_fault_fallback_charge_attr.attr, 645 #ifndef CONFIG_SHMEM 646 &zswpout_attr.attr, 647 &swpin_attr.attr, 648 &swpin_fallback_attr.attr, 649 &swpin_fallback_charge_attr.attr, 650 &swpout_attr.attr, 651 &swpout_fallback_attr.attr, 652 #endif 653 &split_deferred_attr.attr, 654 &nr_anon_attr.attr, 655 &nr_anon_partially_mapped_attr.attr, 656 NULL, 657 }; 658 659 static struct attribute_group anon_stats_attr_grp = { 660 .name = "stats", 661 .attrs = anon_stats_attrs, 662 }; 663 664 static struct attribute *file_stats_attrs[] = { 665 #ifdef CONFIG_SHMEM 666 &shmem_alloc_attr.attr, 667 &shmem_fallback_attr.attr, 668 &shmem_fallback_charge_attr.attr, 669 #endif 670 NULL, 671 }; 672 673 static struct attribute_group file_stats_attr_grp = { 674 .name = "stats", 675 .attrs = file_stats_attrs, 676 }; 677 678 static struct attribute *any_stats_attrs[] = { 679 #ifdef CONFIG_SHMEM 680 &zswpout_attr.attr, 681 &swpin_attr.attr, 682 &swpin_fallback_attr.attr, 683 &swpin_fallback_charge_attr.attr, 684 &swpout_attr.attr, 685 &swpout_fallback_attr.attr, 686 #endif 687 &split_attr.attr, 688 &split_failed_attr.attr, 689 NULL, 690 }; 691 692 static struct attribute_group any_stats_attr_grp = { 693 .name = "stats", 694 .attrs = any_stats_attrs, 695 }; 696 697 static int sysfs_add_group(struct kobject *kobj, 698 const struct attribute_group *grp) 699 { 700 int ret = -ENOENT; 701 702 /* 703 * If the group is named, try to merge first, assuming the subdirectory 704 * was already created. This avoids the warning emitted by 705 * sysfs_create_group() if the directory already exists. 706 */ 707 if (grp->name) 708 ret = sysfs_merge_group(kobj, grp); 709 if (ret) 710 ret = sysfs_create_group(kobj, grp); 711 712 return ret; 713 } 714 715 static struct thpsize *thpsize_create(int order, struct kobject *parent) 716 { 717 unsigned long size = (PAGE_SIZE << order) / SZ_1K; 718 struct thpsize *thpsize; 719 int ret = -ENOMEM; 720 721 thpsize = kzalloc(sizeof(*thpsize), GFP_KERNEL); 722 if (!thpsize) 723 goto err; 724 725 thpsize->order = order; 726 727 ret = kobject_init_and_add(&thpsize->kobj, &thpsize_ktype, parent, 728 "hugepages-%lukB", size); 729 if (ret) { 730 kfree(thpsize); 731 goto err; 732 } 733 734 735 ret = sysfs_add_group(&thpsize->kobj, &any_ctrl_attr_grp); 736 if (ret) 737 goto err_put; 738 739 ret = sysfs_add_group(&thpsize->kobj, &any_stats_attr_grp); 740 if (ret) 741 goto err_put; 742 743 if (BIT(order) & THP_ORDERS_ALL_ANON) { 744 ret = sysfs_add_group(&thpsize->kobj, &anon_ctrl_attr_grp); 745 if (ret) 746 goto err_put; 747 748 ret = sysfs_add_group(&thpsize->kobj, &anon_stats_attr_grp); 749 if (ret) 750 goto err_put; 751 } 752 753 if (BIT(order) & THP_ORDERS_ALL_FILE_DEFAULT) { 754 ret = sysfs_add_group(&thpsize->kobj, &file_ctrl_attr_grp); 755 if (ret) 756 goto err_put; 757 758 ret = sysfs_add_group(&thpsize->kobj, &file_stats_attr_grp); 759 if (ret) 760 goto err_put; 761 } 762 763 return thpsize; 764 err_put: 765 kobject_put(&thpsize->kobj); 766 err: 767 return ERR_PTR(ret); 768 } 769 770 static void thpsize_release(struct kobject *kobj) 771 { 772 kfree(to_thpsize(kobj)); 773 } 774 775 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj) 776 { 777 int err; 778 struct thpsize *thpsize; 779 unsigned long orders; 780 int order; 781 782 /* 783 * Default to setting PMD-sized THP to inherit the global setting and 784 * disable all other sizes. powerpc's PMD_ORDER isn't a compile-time 785 * constant so we have to do this here. 786 */ 787 if (!anon_orders_configured) 788 huge_anon_orders_inherit = BIT(PMD_ORDER); 789 790 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj); 791 if (unlikely(!*hugepage_kobj)) { 792 pr_err("failed to create transparent hugepage kobject\n"); 793 return -ENOMEM; 794 } 795 796 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group); 797 if (err) { 798 pr_err("failed to register transparent hugepage group\n"); 799 goto delete_obj; 800 } 801 802 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group); 803 if (err) { 804 pr_err("failed to register transparent hugepage group\n"); 805 goto remove_hp_group; 806 } 807 808 orders = THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_FILE_DEFAULT; 809 order = highest_order(orders); 810 while (orders) { 811 thpsize = thpsize_create(order, *hugepage_kobj); 812 if (IS_ERR(thpsize)) { 813 pr_err("failed to create thpsize for order %d\n", order); 814 err = PTR_ERR(thpsize); 815 goto remove_all; 816 } 817 list_add(&thpsize->node, &thpsize_list); 818 order = next_order(&orders, order); 819 } 820 821 return 0; 822 823 remove_all: 824 hugepage_exit_sysfs(*hugepage_kobj); 825 return err; 826 remove_hp_group: 827 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group); 828 delete_obj: 829 kobject_put(*hugepage_kobj); 830 return err; 831 } 832 833 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj) 834 { 835 struct thpsize *thpsize, *tmp; 836 837 list_for_each_entry_safe(thpsize, tmp, &thpsize_list, node) { 838 list_del(&thpsize->node); 839 kobject_put(&thpsize->kobj); 840 } 841 842 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group); 843 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group); 844 kobject_put(hugepage_kobj); 845 } 846 #else 847 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj) 848 { 849 return 0; 850 } 851 852 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) 853 { 854 } 855 #endif /* CONFIG_SYSFS */ 856 857 static int __init thp_shrinker_init(void) 858 { 859 deferred_split_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE | 860 SHRINKER_MEMCG_AWARE | 861 SHRINKER_NONSLAB, 862 "thp-deferred_split"); 863 if (!deferred_split_shrinker) 864 return -ENOMEM; 865 866 deferred_split_shrinker->count_objects = deferred_split_count; 867 deferred_split_shrinker->scan_objects = deferred_split_scan; 868 shrinker_register(deferred_split_shrinker); 869 870 if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) { 871 /* 872 * Bump the reference of the huge_zero_folio and do not 873 * initialize the shrinker. 874 * 875 * huge_zero_folio will always be NULL on failure. We assume 876 * that get_huge_zero_folio() will most likely not fail as 877 * thp_shrinker_init() is invoked early on during boot. 878 */ 879 if (!get_huge_zero_folio()) 880 pr_warn("Allocating persistent huge zero folio failed\n"); 881 return 0; 882 } 883 884 huge_zero_folio_shrinker = shrinker_alloc(0, "thp-zero"); 885 if (!huge_zero_folio_shrinker) { 886 shrinker_free(deferred_split_shrinker); 887 return -ENOMEM; 888 } 889 890 huge_zero_folio_shrinker->count_objects = shrink_huge_zero_folio_count; 891 huge_zero_folio_shrinker->scan_objects = shrink_huge_zero_folio_scan; 892 shrinker_register(huge_zero_folio_shrinker); 893 894 return 0; 895 } 896 897 static void __init thp_shrinker_exit(void) 898 { 899 shrinker_free(huge_zero_folio_shrinker); 900 shrinker_free(deferred_split_shrinker); 901 } 902 903 static int __init hugepage_init(void) 904 { 905 int err; 906 struct kobject *hugepage_kobj; 907 908 if (!has_transparent_hugepage()) { 909 transparent_hugepage_flags = 1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED; 910 return -EINVAL; 911 } 912 913 /* 914 * hugepages can't be allocated by the buddy allocator 915 */ 916 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER > MAX_PAGE_ORDER); 917 918 err = hugepage_init_sysfs(&hugepage_kobj); 919 if (err) 920 goto err_sysfs; 921 922 err = khugepaged_init(); 923 if (err) 924 goto err_slab; 925 926 err = thp_shrinker_init(); 927 if (err) 928 goto err_shrinker; 929 930 /* 931 * By default disable transparent hugepages on smaller systems, 932 * where the extra memory used could hurt more than TLB overhead 933 * is likely to save. The admin can still enable it through /sys. 934 */ 935 if (totalram_pages() < MB_TO_PAGES(512)) { 936 transparent_hugepage_flags = 0; 937 return 0; 938 } 939 940 err = start_stop_khugepaged(); 941 if (err) 942 goto err_khugepaged; 943 944 return 0; 945 err_khugepaged: 946 thp_shrinker_exit(); 947 err_shrinker: 948 khugepaged_destroy(); 949 err_slab: 950 hugepage_exit_sysfs(hugepage_kobj); 951 err_sysfs: 952 return err; 953 } 954 subsys_initcall(hugepage_init); 955 956 static int __init setup_transparent_hugepage(char *str) 957 { 958 int ret = 0; 959 if (!str) 960 goto out; 961 if (!strcmp(str, "always")) { 962 set_bit(TRANSPARENT_HUGEPAGE_FLAG, 963 &transparent_hugepage_flags); 964 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 965 &transparent_hugepage_flags); 966 ret = 1; 967 } else if (!strcmp(str, "madvise")) { 968 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 969 &transparent_hugepage_flags); 970 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 971 &transparent_hugepage_flags); 972 ret = 1; 973 } else if (!strcmp(str, "never")) { 974 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 975 &transparent_hugepage_flags); 976 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 977 &transparent_hugepage_flags); 978 ret = 1; 979 } 980 out: 981 if (!ret) 982 pr_warn("transparent_hugepage= cannot parse, ignored\n"); 983 return ret; 984 } 985 __setup("transparent_hugepage=", setup_transparent_hugepage); 986 987 static char str_dup[PAGE_SIZE] __initdata; 988 static int __init setup_thp_anon(char *str) 989 { 990 char *token, *range, *policy, *subtoken; 991 unsigned long always, inherit, madvise; 992 char *start_size, *end_size; 993 int start, end, nr; 994 char *p; 995 996 if (!str || strlen(str) + 1 > PAGE_SIZE) 997 goto err; 998 strscpy(str_dup, str); 999 1000 always = huge_anon_orders_always; 1001 madvise = huge_anon_orders_madvise; 1002 inherit = huge_anon_orders_inherit; 1003 p = str_dup; 1004 while ((token = strsep(&p, ";")) != NULL) { 1005 range = strsep(&token, ":"); 1006 policy = token; 1007 1008 if (!policy) 1009 goto err; 1010 1011 while ((subtoken = strsep(&range, ",")) != NULL) { 1012 if (strchr(subtoken, '-')) { 1013 start_size = strsep(&subtoken, "-"); 1014 end_size = subtoken; 1015 1016 start = get_order_from_str(start_size, THP_ORDERS_ALL_ANON); 1017 end = get_order_from_str(end_size, THP_ORDERS_ALL_ANON); 1018 } else { 1019 start_size = end_size = subtoken; 1020 start = end = get_order_from_str(subtoken, 1021 THP_ORDERS_ALL_ANON); 1022 } 1023 1024 if (start == -EINVAL) { 1025 pr_err("invalid size %s in thp_anon boot parameter\n", start_size); 1026 goto err; 1027 } 1028 1029 if (end == -EINVAL) { 1030 pr_err("invalid size %s in thp_anon boot parameter\n", end_size); 1031 goto err; 1032 } 1033 1034 if (start < 0 || end < 0 || start > end) 1035 goto err; 1036 1037 nr = end - start + 1; 1038 if (!strcmp(policy, "always")) { 1039 bitmap_set(&always, start, nr); 1040 bitmap_clear(&inherit, start, nr); 1041 bitmap_clear(&madvise, start, nr); 1042 } else if (!strcmp(policy, "madvise")) { 1043 bitmap_set(&madvise, start, nr); 1044 bitmap_clear(&inherit, start, nr); 1045 bitmap_clear(&always, start, nr); 1046 } else if (!strcmp(policy, "inherit")) { 1047 bitmap_set(&inherit, start, nr); 1048 bitmap_clear(&madvise, start, nr); 1049 bitmap_clear(&always, start, nr); 1050 } else if (!strcmp(policy, "never")) { 1051 bitmap_clear(&inherit, start, nr); 1052 bitmap_clear(&madvise, start, nr); 1053 bitmap_clear(&always, start, nr); 1054 } else { 1055 pr_err("invalid policy %s in thp_anon boot parameter\n", policy); 1056 goto err; 1057 } 1058 } 1059 } 1060 1061 huge_anon_orders_always = always; 1062 huge_anon_orders_madvise = madvise; 1063 huge_anon_orders_inherit = inherit; 1064 anon_orders_configured = true; 1065 return 1; 1066 1067 err: 1068 pr_warn("thp_anon=%s: error parsing string, ignoring setting\n", str); 1069 return 0; 1070 } 1071 __setup("thp_anon=", setup_thp_anon); 1072 1073 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 1074 { 1075 if (likely(vma->vm_flags & VM_WRITE)) 1076 pmd = pmd_mkwrite(pmd, vma); 1077 return pmd; 1078 } 1079 1080 #ifdef CONFIG_MEMCG 1081 static inline 1082 struct deferred_split *get_deferred_split_queue(struct folio *folio) 1083 { 1084 struct mem_cgroup *memcg = folio_memcg(folio); 1085 struct pglist_data *pgdat = NODE_DATA(folio_nid(folio)); 1086 1087 if (memcg) 1088 return &memcg->deferred_split_queue; 1089 else 1090 return &pgdat->deferred_split_queue; 1091 } 1092 #else 1093 static inline 1094 struct deferred_split *get_deferred_split_queue(struct folio *folio) 1095 { 1096 struct pglist_data *pgdat = NODE_DATA(folio_nid(folio)); 1097 1098 return &pgdat->deferred_split_queue; 1099 } 1100 #endif 1101 1102 static inline bool is_transparent_hugepage(const struct folio *folio) 1103 { 1104 if (!folio_test_large(folio)) 1105 return false; 1106 1107 return is_huge_zero_folio(folio) || 1108 folio_test_large_rmappable(folio); 1109 } 1110 1111 static unsigned long __thp_get_unmapped_area(struct file *filp, 1112 unsigned long addr, unsigned long len, 1113 loff_t off, unsigned long flags, unsigned long size, 1114 vm_flags_t vm_flags) 1115 { 1116 loff_t off_end = off + len; 1117 loff_t off_align = round_up(off, size); 1118 unsigned long len_pad, ret, off_sub; 1119 1120 if (!IS_ENABLED(CONFIG_64BIT) || in_compat_syscall()) 1121 return 0; 1122 1123 if (off_end <= off_align || (off_end - off_align) < size) 1124 return 0; 1125 1126 len_pad = len + size; 1127 if (len_pad < len || (off + len_pad) < off) 1128 return 0; 1129 1130 ret = mm_get_unmapped_area_vmflags(current->mm, filp, addr, len_pad, 1131 off >> PAGE_SHIFT, flags, vm_flags); 1132 1133 /* 1134 * The failure might be due to length padding. The caller will retry 1135 * without the padding. 1136 */ 1137 if (IS_ERR_VALUE(ret)) 1138 return 0; 1139 1140 /* 1141 * Do not try to align to THP boundary if allocation at the address 1142 * hint succeeds. 1143 */ 1144 if (ret == addr) 1145 return addr; 1146 1147 off_sub = (off - ret) & (size - 1); 1148 1149 if (mm_flags_test(MMF_TOPDOWN, current->mm) && !off_sub) 1150 return ret + size; 1151 1152 ret += off_sub; 1153 return ret; 1154 } 1155 1156 unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, 1157 unsigned long len, unsigned long pgoff, unsigned long flags, 1158 vm_flags_t vm_flags) 1159 { 1160 unsigned long ret; 1161 loff_t off = (loff_t)pgoff << PAGE_SHIFT; 1162 1163 ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, vm_flags); 1164 if (ret) 1165 return ret; 1166 1167 return mm_get_unmapped_area_vmflags(current->mm, filp, addr, len, pgoff, flags, 1168 vm_flags); 1169 } 1170 1171 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, 1172 unsigned long len, unsigned long pgoff, unsigned long flags) 1173 { 1174 return thp_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0); 1175 } 1176 EXPORT_SYMBOL_GPL(thp_get_unmapped_area); 1177 1178 static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma, 1179 unsigned long addr) 1180 { 1181 gfp_t gfp = vma_thp_gfp_mask(vma); 1182 const int order = HPAGE_PMD_ORDER; 1183 struct folio *folio; 1184 1185 folio = vma_alloc_folio(gfp, order, vma, addr & HPAGE_PMD_MASK); 1186 1187 if (unlikely(!folio)) { 1188 count_vm_event(THP_FAULT_FALLBACK); 1189 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK); 1190 return NULL; 1191 } 1192 1193 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 1194 if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) { 1195 folio_put(folio); 1196 count_vm_event(THP_FAULT_FALLBACK); 1197 count_vm_event(THP_FAULT_FALLBACK_CHARGE); 1198 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK); 1199 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); 1200 return NULL; 1201 } 1202 folio_throttle_swaprate(folio, gfp); 1203 1204 /* 1205 * When a folio is not zeroed during allocation (__GFP_ZERO not used) 1206 * or user folios require special handling, folio_zero_user() is used to 1207 * make sure that the page corresponding to the faulting address will be 1208 * hot in the cache after zeroing. 1209 */ 1210 if (user_alloc_needs_zeroing()) 1211 folio_zero_user(folio, addr); 1212 /* 1213 * The memory barrier inside __folio_mark_uptodate makes sure that 1214 * folio_zero_user writes become visible before the set_pmd_at() 1215 * write. 1216 */ 1217 __folio_mark_uptodate(folio); 1218 return folio; 1219 } 1220 1221 static void map_anon_folio_pmd(struct folio *folio, pmd_t *pmd, 1222 struct vm_area_struct *vma, unsigned long haddr) 1223 { 1224 pmd_t entry; 1225 1226 entry = folio_mk_pmd(folio, vma->vm_page_prot); 1227 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1228 folio_add_new_anon_rmap(folio, vma, haddr, RMAP_EXCLUSIVE); 1229 folio_add_lru_vma(folio, vma); 1230 set_pmd_at(vma->vm_mm, haddr, pmd, entry); 1231 update_mmu_cache_pmd(vma, haddr, pmd); 1232 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1233 count_vm_event(THP_FAULT_ALLOC); 1234 count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_ALLOC); 1235 count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); 1236 } 1237 1238 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf) 1239 { 1240 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1241 struct vm_area_struct *vma = vmf->vma; 1242 struct folio *folio; 1243 pgtable_t pgtable; 1244 vm_fault_t ret = 0; 1245 1246 folio = vma_alloc_anon_folio_pmd(vma, vmf->address); 1247 if (unlikely(!folio)) 1248 return VM_FAULT_FALLBACK; 1249 1250 pgtable = pte_alloc_one(vma->vm_mm); 1251 if (unlikely(!pgtable)) { 1252 ret = VM_FAULT_OOM; 1253 goto release; 1254 } 1255 1256 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1257 if (unlikely(!pmd_none(*vmf->pmd))) { 1258 goto unlock_release; 1259 } else { 1260 ret = check_stable_address_space(vma->vm_mm); 1261 if (ret) 1262 goto unlock_release; 1263 1264 /* Deliver the page fault to userland */ 1265 if (userfaultfd_missing(vma)) { 1266 spin_unlock(vmf->ptl); 1267 folio_put(folio); 1268 pte_free(vma->vm_mm, pgtable); 1269 ret = handle_userfault(vmf, VM_UFFD_MISSING); 1270 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 1271 return ret; 1272 } 1273 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); 1274 map_anon_folio_pmd(folio, vmf->pmd, vma, haddr); 1275 mm_inc_nr_ptes(vma->vm_mm); 1276 deferred_split_folio(folio, false); 1277 spin_unlock(vmf->ptl); 1278 } 1279 1280 return 0; 1281 unlock_release: 1282 spin_unlock(vmf->ptl); 1283 release: 1284 if (pgtable) 1285 pte_free(vma->vm_mm, pgtable); 1286 folio_put(folio); 1287 return ret; 1288 1289 } 1290 1291 /* 1292 * always: directly stall for all thp allocations 1293 * defer: wake kswapd and fail if not immediately available 1294 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise 1295 * fail if not immediately available 1296 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately 1297 * available 1298 * never: never stall for any thp allocation 1299 */ 1300 gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma) 1301 { 1302 const bool vma_madvised = vma && (vma->vm_flags & VM_HUGEPAGE); 1303 1304 /* Always do synchronous compaction */ 1305 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) 1306 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY); 1307 1308 /* Kick kcompactd and fail quickly */ 1309 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) 1310 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM; 1311 1312 /* Synchronous compaction if madvised, otherwise kick kcompactd */ 1313 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags)) 1314 return GFP_TRANSHUGE_LIGHT | 1315 (vma_madvised ? __GFP_DIRECT_RECLAIM : 1316 __GFP_KSWAPD_RECLAIM); 1317 1318 /* Only do synchronous compaction if madvised */ 1319 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags)) 1320 return GFP_TRANSHUGE_LIGHT | 1321 (vma_madvised ? __GFP_DIRECT_RECLAIM : 0); 1322 1323 return GFP_TRANSHUGE_LIGHT; 1324 } 1325 1326 /* Caller must hold page table lock. */ 1327 static void set_huge_zero_folio(pgtable_t pgtable, struct mm_struct *mm, 1328 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, 1329 struct folio *zero_folio) 1330 { 1331 pmd_t entry; 1332 entry = folio_mk_pmd(zero_folio, vma->vm_page_prot); 1333 entry = pmd_mkspecial(entry); 1334 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1335 set_pmd_at(mm, haddr, pmd, entry); 1336 mm_inc_nr_ptes(mm); 1337 } 1338 1339 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) 1340 { 1341 struct vm_area_struct *vma = vmf->vma; 1342 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1343 vm_fault_t ret; 1344 1345 if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER)) 1346 return VM_FAULT_FALLBACK; 1347 ret = vmf_anon_prepare(vmf); 1348 if (ret) 1349 return ret; 1350 khugepaged_enter_vma(vma, vma->vm_flags); 1351 1352 if (!(vmf->flags & FAULT_FLAG_WRITE) && 1353 !mm_forbids_zeropage(vma->vm_mm) && 1354 transparent_hugepage_use_zero_page()) { 1355 pgtable_t pgtable; 1356 struct folio *zero_folio; 1357 vm_fault_t ret; 1358 1359 pgtable = pte_alloc_one(vma->vm_mm); 1360 if (unlikely(!pgtable)) 1361 return VM_FAULT_OOM; 1362 zero_folio = mm_get_huge_zero_folio(vma->vm_mm); 1363 if (unlikely(!zero_folio)) { 1364 pte_free(vma->vm_mm, pgtable); 1365 count_vm_event(THP_FAULT_FALLBACK); 1366 return VM_FAULT_FALLBACK; 1367 } 1368 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1369 ret = 0; 1370 if (pmd_none(*vmf->pmd)) { 1371 ret = check_stable_address_space(vma->vm_mm); 1372 if (ret) { 1373 spin_unlock(vmf->ptl); 1374 pte_free(vma->vm_mm, pgtable); 1375 } else if (userfaultfd_missing(vma)) { 1376 spin_unlock(vmf->ptl); 1377 pte_free(vma->vm_mm, pgtable); 1378 ret = handle_userfault(vmf, VM_UFFD_MISSING); 1379 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 1380 } else { 1381 set_huge_zero_folio(pgtable, vma->vm_mm, vma, 1382 haddr, vmf->pmd, zero_folio); 1383 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1384 spin_unlock(vmf->ptl); 1385 } 1386 } else { 1387 spin_unlock(vmf->ptl); 1388 pte_free(vma->vm_mm, pgtable); 1389 } 1390 return ret; 1391 } 1392 1393 return __do_huge_pmd_anonymous_page(vmf); 1394 } 1395 1396 struct folio_or_pfn { 1397 union { 1398 struct folio *folio; 1399 unsigned long pfn; 1400 }; 1401 bool is_folio; 1402 }; 1403 1404 static vm_fault_t insert_pmd(struct vm_area_struct *vma, unsigned long addr, 1405 pmd_t *pmd, struct folio_or_pfn fop, pgprot_t prot, 1406 bool write) 1407 { 1408 struct mm_struct *mm = vma->vm_mm; 1409 pgtable_t pgtable = NULL; 1410 spinlock_t *ptl; 1411 pmd_t entry; 1412 1413 if (addr < vma->vm_start || addr >= vma->vm_end) 1414 return VM_FAULT_SIGBUS; 1415 1416 if (arch_needs_pgtable_deposit()) { 1417 pgtable = pte_alloc_one(vma->vm_mm); 1418 if (!pgtable) 1419 return VM_FAULT_OOM; 1420 } 1421 1422 ptl = pmd_lock(mm, pmd); 1423 if (!pmd_none(*pmd)) { 1424 const unsigned long pfn = fop.is_folio ? folio_pfn(fop.folio) : 1425 fop.pfn; 1426 1427 if (write) { 1428 if (pmd_pfn(*pmd) != pfn) { 1429 WARN_ON_ONCE(!is_huge_zero_pmd(*pmd)); 1430 goto out_unlock; 1431 } 1432 entry = pmd_mkyoung(*pmd); 1433 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1434 if (pmdp_set_access_flags(vma, addr, pmd, entry, 1)) 1435 update_mmu_cache_pmd(vma, addr, pmd); 1436 } 1437 goto out_unlock; 1438 } 1439 1440 if (fop.is_folio) { 1441 entry = folio_mk_pmd(fop.folio, vma->vm_page_prot); 1442 1443 if (is_huge_zero_folio(fop.folio)) { 1444 entry = pmd_mkspecial(entry); 1445 } else { 1446 folio_get(fop.folio); 1447 folio_add_file_rmap_pmd(fop.folio, &fop.folio->page, vma); 1448 add_mm_counter(mm, mm_counter_file(fop.folio), HPAGE_PMD_NR); 1449 } 1450 } else { 1451 entry = pmd_mkhuge(pfn_pmd(fop.pfn, prot)); 1452 entry = pmd_mkspecial(entry); 1453 } 1454 if (write) { 1455 entry = pmd_mkyoung(pmd_mkdirty(entry)); 1456 entry = maybe_pmd_mkwrite(entry, vma); 1457 } 1458 1459 if (pgtable) { 1460 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1461 mm_inc_nr_ptes(mm); 1462 pgtable = NULL; 1463 } 1464 1465 set_pmd_at(mm, addr, pmd, entry); 1466 update_mmu_cache_pmd(vma, addr, pmd); 1467 1468 out_unlock: 1469 spin_unlock(ptl); 1470 if (pgtable) 1471 pte_free(mm, pgtable); 1472 return VM_FAULT_NOPAGE; 1473 } 1474 1475 /** 1476 * vmf_insert_pfn_pmd - insert a pmd size pfn 1477 * @vmf: Structure describing the fault 1478 * @pfn: pfn to insert 1479 * @write: whether it's a write fault 1480 * 1481 * Insert a pmd size pfn. See vmf_insert_pfn() for additional info. 1482 * 1483 * Return: vm_fault_t value. 1484 */ 1485 vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, unsigned long pfn, 1486 bool write) 1487 { 1488 unsigned long addr = vmf->address & PMD_MASK; 1489 struct vm_area_struct *vma = vmf->vma; 1490 pgprot_t pgprot = vma->vm_page_prot; 1491 struct folio_or_pfn fop = { 1492 .pfn = pfn, 1493 }; 1494 1495 /* 1496 * If we had pmd_special, we could avoid all these restrictions, 1497 * but we need to be consistent with PTEs and architectures that 1498 * can't support a 'special' bit. 1499 */ 1500 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 1501 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1502 (VM_PFNMAP|VM_MIXEDMAP)); 1503 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1504 1505 pfnmap_setup_cachemode_pfn(pfn, &pgprot); 1506 1507 return insert_pmd(vma, addr, vmf->pmd, fop, pgprot, write); 1508 } 1509 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd); 1510 1511 vm_fault_t vmf_insert_folio_pmd(struct vm_fault *vmf, struct folio *folio, 1512 bool write) 1513 { 1514 struct vm_area_struct *vma = vmf->vma; 1515 unsigned long addr = vmf->address & PMD_MASK; 1516 struct folio_or_pfn fop = { 1517 .folio = folio, 1518 .is_folio = true, 1519 }; 1520 1521 if (WARN_ON_ONCE(folio_order(folio) != PMD_ORDER)) 1522 return VM_FAULT_SIGBUS; 1523 1524 return insert_pmd(vma, addr, vmf->pmd, fop, vma->vm_page_prot, write); 1525 } 1526 EXPORT_SYMBOL_GPL(vmf_insert_folio_pmd); 1527 1528 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1529 static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma) 1530 { 1531 if (likely(vma->vm_flags & VM_WRITE)) 1532 pud = pud_mkwrite(pud); 1533 return pud; 1534 } 1535 1536 static vm_fault_t insert_pud(struct vm_area_struct *vma, unsigned long addr, 1537 pud_t *pud, struct folio_or_pfn fop, pgprot_t prot, bool write) 1538 { 1539 struct mm_struct *mm = vma->vm_mm; 1540 spinlock_t *ptl; 1541 pud_t entry; 1542 1543 if (addr < vma->vm_start || addr >= vma->vm_end) 1544 return VM_FAULT_SIGBUS; 1545 1546 ptl = pud_lock(mm, pud); 1547 if (!pud_none(*pud)) { 1548 const unsigned long pfn = fop.is_folio ? folio_pfn(fop.folio) : 1549 fop.pfn; 1550 1551 if (write) { 1552 if (WARN_ON_ONCE(pud_pfn(*pud) != pfn)) 1553 goto out_unlock; 1554 entry = pud_mkyoung(*pud); 1555 entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma); 1556 if (pudp_set_access_flags(vma, addr, pud, entry, 1)) 1557 update_mmu_cache_pud(vma, addr, pud); 1558 } 1559 goto out_unlock; 1560 } 1561 1562 if (fop.is_folio) { 1563 entry = folio_mk_pud(fop.folio, vma->vm_page_prot); 1564 1565 folio_get(fop.folio); 1566 folio_add_file_rmap_pud(fop.folio, &fop.folio->page, vma); 1567 add_mm_counter(mm, mm_counter_file(fop.folio), HPAGE_PUD_NR); 1568 } else { 1569 entry = pud_mkhuge(pfn_pud(fop.pfn, prot)); 1570 entry = pud_mkspecial(entry); 1571 } 1572 if (write) { 1573 entry = pud_mkyoung(pud_mkdirty(entry)); 1574 entry = maybe_pud_mkwrite(entry, vma); 1575 } 1576 set_pud_at(mm, addr, pud, entry); 1577 update_mmu_cache_pud(vma, addr, pud); 1578 out_unlock: 1579 spin_unlock(ptl); 1580 return VM_FAULT_NOPAGE; 1581 } 1582 1583 /** 1584 * vmf_insert_pfn_pud - insert a pud size pfn 1585 * @vmf: Structure describing the fault 1586 * @pfn: pfn to insert 1587 * @write: whether it's a write fault 1588 * 1589 * Insert a pud size pfn. See vmf_insert_pfn() for additional info. 1590 * 1591 * Return: vm_fault_t value. 1592 */ 1593 vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, unsigned long pfn, 1594 bool write) 1595 { 1596 unsigned long addr = vmf->address & PUD_MASK; 1597 struct vm_area_struct *vma = vmf->vma; 1598 pgprot_t pgprot = vma->vm_page_prot; 1599 struct folio_or_pfn fop = { 1600 .pfn = pfn, 1601 }; 1602 1603 /* 1604 * If we had pud_special, we could avoid all these restrictions, 1605 * but we need to be consistent with PTEs and architectures that 1606 * can't support a 'special' bit. 1607 */ 1608 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 1609 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1610 (VM_PFNMAP|VM_MIXEDMAP)); 1611 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1612 1613 pfnmap_setup_cachemode_pfn(pfn, &pgprot); 1614 1615 return insert_pud(vma, addr, vmf->pud, fop, pgprot, write); 1616 } 1617 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud); 1618 1619 /** 1620 * vmf_insert_folio_pud - insert a pud size folio mapped by a pud entry 1621 * @vmf: Structure describing the fault 1622 * @folio: folio to insert 1623 * @write: whether it's a write fault 1624 * 1625 * Return: vm_fault_t value. 1626 */ 1627 vm_fault_t vmf_insert_folio_pud(struct vm_fault *vmf, struct folio *folio, 1628 bool write) 1629 { 1630 struct vm_area_struct *vma = vmf->vma; 1631 unsigned long addr = vmf->address & PUD_MASK; 1632 struct folio_or_pfn fop = { 1633 .folio = folio, 1634 .is_folio = true, 1635 }; 1636 1637 if (WARN_ON_ONCE(folio_order(folio) != PUD_ORDER)) 1638 return VM_FAULT_SIGBUS; 1639 1640 return insert_pud(vma, addr, vmf->pud, fop, vma->vm_page_prot, write); 1641 } 1642 EXPORT_SYMBOL_GPL(vmf_insert_folio_pud); 1643 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1644 1645 /** 1646 * touch_pmd - Mark page table pmd entry as accessed and dirty (for write) 1647 * @vma: The VMA covering @addr 1648 * @addr: The virtual address 1649 * @pmd: pmd pointer into the page table mapping @addr 1650 * @write: Whether it's a write access 1651 * 1652 * Return: whether the pmd entry is changed 1653 */ 1654 bool touch_pmd(struct vm_area_struct *vma, unsigned long addr, 1655 pmd_t *pmd, bool write) 1656 { 1657 pmd_t entry; 1658 1659 entry = pmd_mkyoung(*pmd); 1660 if (write) 1661 entry = pmd_mkdirty(entry); 1662 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, 1663 pmd, entry, write)) { 1664 update_mmu_cache_pmd(vma, addr, pmd); 1665 return true; 1666 } 1667 1668 return false; 1669 } 1670 1671 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1672 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 1673 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) 1674 { 1675 spinlock_t *dst_ptl, *src_ptl; 1676 struct page *src_page; 1677 struct folio *src_folio; 1678 pmd_t pmd; 1679 pgtable_t pgtable = NULL; 1680 int ret = -ENOMEM; 1681 1682 pmd = pmdp_get_lockless(src_pmd); 1683 if (unlikely(pmd_present(pmd) && pmd_special(pmd) && 1684 !is_huge_zero_pmd(pmd))) { 1685 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1686 src_ptl = pmd_lockptr(src_mm, src_pmd); 1687 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1688 /* 1689 * No need to recheck the pmd, it can't change with write 1690 * mmap lock held here. 1691 * 1692 * Meanwhile, making sure it's not a CoW VMA with writable 1693 * mapping, otherwise it means either the anon page wrongly 1694 * applied special bit, or we made the PRIVATE mapping be 1695 * able to wrongly write to the backend MMIO. 1696 */ 1697 VM_WARN_ON_ONCE(is_cow_mapping(src_vma->vm_flags) && pmd_write(pmd)); 1698 goto set_pmd; 1699 } 1700 1701 /* Skip if can be re-fill on fault */ 1702 if (!vma_is_anonymous(dst_vma)) 1703 return 0; 1704 1705 pgtable = pte_alloc_one(dst_mm); 1706 if (unlikely(!pgtable)) 1707 goto out; 1708 1709 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1710 src_ptl = pmd_lockptr(src_mm, src_pmd); 1711 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1712 1713 ret = -EAGAIN; 1714 pmd = *src_pmd; 1715 1716 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 1717 if (unlikely(is_swap_pmd(pmd))) { 1718 swp_entry_t entry = pmd_to_swp_entry(pmd); 1719 1720 VM_BUG_ON(!is_pmd_migration_entry(pmd)); 1721 if (!is_readable_migration_entry(entry)) { 1722 entry = make_readable_migration_entry( 1723 swp_offset(entry)); 1724 pmd = swp_entry_to_pmd(entry); 1725 if (pmd_swp_soft_dirty(*src_pmd)) 1726 pmd = pmd_swp_mksoft_dirty(pmd); 1727 if (pmd_swp_uffd_wp(*src_pmd)) 1728 pmd = pmd_swp_mkuffd_wp(pmd); 1729 set_pmd_at(src_mm, addr, src_pmd, pmd); 1730 } 1731 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1732 mm_inc_nr_ptes(dst_mm); 1733 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1734 if (!userfaultfd_wp(dst_vma)) 1735 pmd = pmd_swp_clear_uffd_wp(pmd); 1736 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1737 ret = 0; 1738 goto out_unlock; 1739 } 1740 #endif 1741 1742 if (unlikely(!pmd_trans_huge(pmd))) { 1743 pte_free(dst_mm, pgtable); 1744 goto out_unlock; 1745 } 1746 /* 1747 * When page table lock is held, the huge zero pmd should not be 1748 * under splitting since we don't split the page itself, only pmd to 1749 * a page table. 1750 */ 1751 if (is_huge_zero_pmd(pmd)) { 1752 /* 1753 * mm_get_huge_zero_folio() will never allocate a new 1754 * folio here, since we already have a zero page to 1755 * copy. It just takes a reference. 1756 */ 1757 mm_get_huge_zero_folio(dst_mm); 1758 goto out_zero_page; 1759 } 1760 1761 src_page = pmd_page(pmd); 1762 VM_BUG_ON_PAGE(!PageHead(src_page), src_page); 1763 src_folio = page_folio(src_page); 1764 1765 folio_get(src_folio); 1766 if (unlikely(folio_try_dup_anon_rmap_pmd(src_folio, src_page, dst_vma, src_vma))) { 1767 /* Page maybe pinned: split and retry the fault on PTEs. */ 1768 folio_put(src_folio); 1769 pte_free(dst_mm, pgtable); 1770 spin_unlock(src_ptl); 1771 spin_unlock(dst_ptl); 1772 __split_huge_pmd(src_vma, src_pmd, addr, false); 1773 return -EAGAIN; 1774 } 1775 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1776 out_zero_page: 1777 mm_inc_nr_ptes(dst_mm); 1778 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1779 pmdp_set_wrprotect(src_mm, addr, src_pmd); 1780 if (!userfaultfd_wp(dst_vma)) 1781 pmd = pmd_clear_uffd_wp(pmd); 1782 pmd = pmd_wrprotect(pmd); 1783 set_pmd: 1784 pmd = pmd_mkold(pmd); 1785 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1786 1787 ret = 0; 1788 out_unlock: 1789 spin_unlock(src_ptl); 1790 spin_unlock(dst_ptl); 1791 out: 1792 return ret; 1793 } 1794 1795 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1796 void touch_pud(struct vm_area_struct *vma, unsigned long addr, 1797 pud_t *pud, bool write) 1798 { 1799 pud_t _pud; 1800 1801 _pud = pud_mkyoung(*pud); 1802 if (write) 1803 _pud = pud_mkdirty(_pud); 1804 if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK, 1805 pud, _pud, write)) 1806 update_mmu_cache_pud(vma, addr, pud); 1807 } 1808 1809 int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1810 pud_t *dst_pud, pud_t *src_pud, unsigned long addr, 1811 struct vm_area_struct *vma) 1812 { 1813 spinlock_t *dst_ptl, *src_ptl; 1814 pud_t pud; 1815 int ret; 1816 1817 dst_ptl = pud_lock(dst_mm, dst_pud); 1818 src_ptl = pud_lockptr(src_mm, src_pud); 1819 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1820 1821 ret = -EAGAIN; 1822 pud = *src_pud; 1823 if (unlikely(!pud_trans_huge(pud))) 1824 goto out_unlock; 1825 1826 /* 1827 * TODO: once we support anonymous pages, use 1828 * folio_try_dup_anon_rmap_*() and split if duplicating fails. 1829 */ 1830 if (is_cow_mapping(vma->vm_flags) && pud_write(pud)) { 1831 pudp_set_wrprotect(src_mm, addr, src_pud); 1832 pud = pud_wrprotect(pud); 1833 } 1834 pud = pud_mkold(pud); 1835 set_pud_at(dst_mm, addr, dst_pud, pud); 1836 1837 ret = 0; 1838 out_unlock: 1839 spin_unlock(src_ptl); 1840 spin_unlock(dst_ptl); 1841 return ret; 1842 } 1843 1844 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud) 1845 { 1846 bool write = vmf->flags & FAULT_FLAG_WRITE; 1847 1848 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); 1849 if (unlikely(!pud_same(*vmf->pud, orig_pud))) 1850 goto unlock; 1851 1852 touch_pud(vmf->vma, vmf->address, vmf->pud, write); 1853 unlock: 1854 spin_unlock(vmf->ptl); 1855 } 1856 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1857 1858 bool huge_pmd_set_accessed(struct vm_fault *vmf) 1859 { 1860 bool write = vmf->flags & FAULT_FLAG_WRITE; 1861 1862 if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd))) 1863 return false; 1864 1865 return touch_pmd(vmf->vma, vmf->address, vmf->pmd, write); 1866 } 1867 1868 static vm_fault_t do_huge_zero_wp_pmd(struct vm_fault *vmf) 1869 { 1870 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1871 struct vm_area_struct *vma = vmf->vma; 1872 struct mmu_notifier_range range; 1873 struct folio *folio; 1874 vm_fault_t ret = 0; 1875 1876 folio = vma_alloc_anon_folio_pmd(vma, vmf->address); 1877 if (unlikely(!folio)) 1878 return VM_FAULT_FALLBACK; 1879 1880 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, haddr, 1881 haddr + HPAGE_PMD_SIZE); 1882 mmu_notifier_invalidate_range_start(&range); 1883 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1884 if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) 1885 goto release; 1886 ret = check_stable_address_space(vma->vm_mm); 1887 if (ret) 1888 goto release; 1889 (void)pmdp_huge_clear_flush(vma, haddr, vmf->pmd); 1890 map_anon_folio_pmd(folio, vmf->pmd, vma, haddr); 1891 goto unlock; 1892 release: 1893 folio_put(folio); 1894 unlock: 1895 spin_unlock(vmf->ptl); 1896 mmu_notifier_invalidate_range_end(&range); 1897 return ret; 1898 } 1899 1900 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) 1901 { 1902 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; 1903 struct vm_area_struct *vma = vmf->vma; 1904 struct folio *folio; 1905 struct page *page; 1906 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1907 pmd_t orig_pmd = vmf->orig_pmd; 1908 1909 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); 1910 VM_BUG_ON_VMA(!vma->anon_vma, vma); 1911 1912 if (is_huge_zero_pmd(orig_pmd)) { 1913 vm_fault_t ret = do_huge_zero_wp_pmd(vmf); 1914 1915 if (!(ret & VM_FAULT_FALLBACK)) 1916 return ret; 1917 1918 /* Fallback to splitting PMD if THP cannot be allocated */ 1919 goto fallback; 1920 } 1921 1922 spin_lock(vmf->ptl); 1923 1924 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1925 spin_unlock(vmf->ptl); 1926 return 0; 1927 } 1928 1929 page = pmd_page(orig_pmd); 1930 folio = page_folio(page); 1931 VM_BUG_ON_PAGE(!PageHead(page), page); 1932 1933 /* Early check when only holding the PT lock. */ 1934 if (PageAnonExclusive(page)) 1935 goto reuse; 1936 1937 if (!folio_trylock(folio)) { 1938 folio_get(folio); 1939 spin_unlock(vmf->ptl); 1940 folio_lock(folio); 1941 spin_lock(vmf->ptl); 1942 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1943 spin_unlock(vmf->ptl); 1944 folio_unlock(folio); 1945 folio_put(folio); 1946 return 0; 1947 } 1948 folio_put(folio); 1949 } 1950 1951 /* Recheck after temporarily dropping the PT lock. */ 1952 if (PageAnonExclusive(page)) { 1953 folio_unlock(folio); 1954 goto reuse; 1955 } 1956 1957 /* 1958 * See do_wp_page(): we can only reuse the folio exclusively if 1959 * there are no additional references. Note that we always drain 1960 * the LRU cache immediately after adding a THP. 1961 */ 1962 if (folio_ref_count(folio) > 1963 1 + folio_test_swapcache(folio) * folio_nr_pages(folio)) 1964 goto unlock_fallback; 1965 if (folio_test_swapcache(folio)) 1966 folio_free_swap(folio); 1967 if (folio_ref_count(folio) == 1) { 1968 pmd_t entry; 1969 1970 folio_move_anon_rmap(folio, vma); 1971 SetPageAnonExclusive(page); 1972 folio_unlock(folio); 1973 reuse: 1974 if (unlikely(unshare)) { 1975 spin_unlock(vmf->ptl); 1976 return 0; 1977 } 1978 entry = pmd_mkyoung(orig_pmd); 1979 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1980 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) 1981 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1982 spin_unlock(vmf->ptl); 1983 return 0; 1984 } 1985 1986 unlock_fallback: 1987 folio_unlock(folio); 1988 spin_unlock(vmf->ptl); 1989 fallback: 1990 __split_huge_pmd(vma, vmf->pmd, vmf->address, false); 1991 return VM_FAULT_FALLBACK; 1992 } 1993 1994 static inline bool can_change_pmd_writable(struct vm_area_struct *vma, 1995 unsigned long addr, pmd_t pmd) 1996 { 1997 struct page *page; 1998 1999 if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) 2000 return false; 2001 2002 /* Don't touch entries that are not even readable (NUMA hinting). */ 2003 if (pmd_protnone(pmd)) 2004 return false; 2005 2006 /* Do we need write faults for softdirty tracking? */ 2007 if (pmd_needs_soft_dirty_wp(vma, pmd)) 2008 return false; 2009 2010 /* Do we need write faults for uffd-wp tracking? */ 2011 if (userfaultfd_huge_pmd_wp(vma, pmd)) 2012 return false; 2013 2014 if (!(vma->vm_flags & VM_SHARED)) { 2015 /* See can_change_pte_writable(). */ 2016 page = vm_normal_page_pmd(vma, addr, pmd); 2017 return page && PageAnon(page) && PageAnonExclusive(page); 2018 } 2019 2020 /* See can_change_pte_writable(). */ 2021 return pmd_dirty(pmd); 2022 } 2023 2024 /* NUMA hinting page fault entry point for trans huge pmds */ 2025 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) 2026 { 2027 struct vm_area_struct *vma = vmf->vma; 2028 struct folio *folio; 2029 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 2030 int nid = NUMA_NO_NODE; 2031 int target_nid, last_cpupid; 2032 pmd_t pmd, old_pmd; 2033 bool writable = false; 2034 int flags = 0; 2035 2036 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 2037 old_pmd = pmdp_get(vmf->pmd); 2038 2039 if (unlikely(!pmd_same(old_pmd, vmf->orig_pmd))) { 2040 spin_unlock(vmf->ptl); 2041 return 0; 2042 } 2043 2044 pmd = pmd_modify(old_pmd, vma->vm_page_prot); 2045 2046 /* 2047 * Detect now whether the PMD could be writable; this information 2048 * is only valid while holding the PT lock. 2049 */ 2050 writable = pmd_write(pmd); 2051 if (!writable && vma_wants_manual_pte_write_upgrade(vma) && 2052 can_change_pmd_writable(vma, vmf->address, pmd)) 2053 writable = true; 2054 2055 folio = vm_normal_folio_pmd(vma, haddr, pmd); 2056 if (!folio) 2057 goto out_map; 2058 2059 nid = folio_nid(folio); 2060 2061 target_nid = numa_migrate_check(folio, vmf, haddr, &flags, writable, 2062 &last_cpupid); 2063 if (target_nid == NUMA_NO_NODE) 2064 goto out_map; 2065 if (migrate_misplaced_folio_prepare(folio, vma, target_nid)) { 2066 flags |= TNF_MIGRATE_FAIL; 2067 goto out_map; 2068 } 2069 /* The folio is isolated and isolation code holds a folio reference. */ 2070 spin_unlock(vmf->ptl); 2071 writable = false; 2072 2073 if (!migrate_misplaced_folio(folio, target_nid)) { 2074 flags |= TNF_MIGRATED; 2075 nid = target_nid; 2076 task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags); 2077 return 0; 2078 } 2079 2080 flags |= TNF_MIGRATE_FAIL; 2081 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 2082 if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) { 2083 spin_unlock(vmf->ptl); 2084 return 0; 2085 } 2086 out_map: 2087 /* Restore the PMD */ 2088 pmd = pmd_modify(pmdp_get(vmf->pmd), vma->vm_page_prot); 2089 pmd = pmd_mkyoung(pmd); 2090 if (writable) 2091 pmd = pmd_mkwrite(pmd, vma); 2092 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); 2093 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 2094 spin_unlock(vmf->ptl); 2095 2096 if (nid != NUMA_NO_NODE) 2097 task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags); 2098 return 0; 2099 } 2100 2101 /* 2102 * Return true if we do MADV_FREE successfully on entire pmd page. 2103 * Otherwise, return false. 2104 */ 2105 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2106 pmd_t *pmd, unsigned long addr, unsigned long next) 2107 { 2108 spinlock_t *ptl; 2109 pmd_t orig_pmd; 2110 struct folio *folio; 2111 struct mm_struct *mm = tlb->mm; 2112 bool ret = false; 2113 2114 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2115 2116 ptl = pmd_trans_huge_lock(pmd, vma); 2117 if (!ptl) 2118 goto out_unlocked; 2119 2120 orig_pmd = *pmd; 2121 if (is_huge_zero_pmd(orig_pmd)) 2122 goto out; 2123 2124 if (unlikely(!pmd_present(orig_pmd))) { 2125 VM_BUG_ON(thp_migration_supported() && 2126 !is_pmd_migration_entry(orig_pmd)); 2127 goto out; 2128 } 2129 2130 folio = pmd_folio(orig_pmd); 2131 /* 2132 * If other processes are mapping this folio, we couldn't discard 2133 * the folio unless they all do MADV_FREE so let's skip the folio. 2134 */ 2135 if (folio_maybe_mapped_shared(folio)) 2136 goto out; 2137 2138 if (!folio_trylock(folio)) 2139 goto out; 2140 2141 /* 2142 * If user want to discard part-pages of THP, split it so MADV_FREE 2143 * will deactivate only them. 2144 */ 2145 if (next - addr != HPAGE_PMD_SIZE) { 2146 folio_get(folio); 2147 spin_unlock(ptl); 2148 split_folio(folio); 2149 folio_unlock(folio); 2150 folio_put(folio); 2151 goto out_unlocked; 2152 } 2153 2154 if (folio_test_dirty(folio)) 2155 folio_clear_dirty(folio); 2156 folio_unlock(folio); 2157 2158 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) { 2159 pmdp_invalidate(vma, addr, pmd); 2160 orig_pmd = pmd_mkold(orig_pmd); 2161 orig_pmd = pmd_mkclean(orig_pmd); 2162 2163 set_pmd_at(mm, addr, pmd, orig_pmd); 2164 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 2165 } 2166 2167 folio_mark_lazyfree(folio); 2168 ret = true; 2169 out: 2170 spin_unlock(ptl); 2171 out_unlocked: 2172 return ret; 2173 } 2174 2175 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd) 2176 { 2177 pgtable_t pgtable; 2178 2179 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2180 pte_free(mm, pgtable); 2181 mm_dec_nr_ptes(mm); 2182 } 2183 2184 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2185 pmd_t *pmd, unsigned long addr) 2186 { 2187 pmd_t orig_pmd; 2188 spinlock_t *ptl; 2189 2190 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2191 2192 ptl = __pmd_trans_huge_lock(pmd, vma); 2193 if (!ptl) 2194 return 0; 2195 /* 2196 * For architectures like ppc64 we look at deposited pgtable 2197 * when calling pmdp_huge_get_and_clear. So do the 2198 * pgtable_trans_huge_withdraw after finishing pmdp related 2199 * operations. 2200 */ 2201 orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd, 2202 tlb->fullmm); 2203 arch_check_zapped_pmd(vma, orig_pmd); 2204 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 2205 if (!vma_is_dax(vma) && vma_is_special_huge(vma)) { 2206 if (arch_needs_pgtable_deposit()) 2207 zap_deposited_table(tlb->mm, pmd); 2208 spin_unlock(ptl); 2209 } else if (is_huge_zero_pmd(orig_pmd)) { 2210 if (!vma_is_dax(vma) || arch_needs_pgtable_deposit()) 2211 zap_deposited_table(tlb->mm, pmd); 2212 spin_unlock(ptl); 2213 } else { 2214 struct folio *folio = NULL; 2215 int flush_needed = 1; 2216 2217 if (pmd_present(orig_pmd)) { 2218 struct page *page = pmd_page(orig_pmd); 2219 2220 folio = page_folio(page); 2221 folio_remove_rmap_pmd(folio, page, vma); 2222 WARN_ON_ONCE(folio_mapcount(folio) < 0); 2223 VM_BUG_ON_PAGE(!PageHead(page), page); 2224 } else if (thp_migration_supported()) { 2225 swp_entry_t entry; 2226 2227 VM_BUG_ON(!is_pmd_migration_entry(orig_pmd)); 2228 entry = pmd_to_swp_entry(orig_pmd); 2229 folio = pfn_swap_entry_folio(entry); 2230 flush_needed = 0; 2231 } else 2232 WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!"); 2233 2234 if (folio_test_anon(folio)) { 2235 zap_deposited_table(tlb->mm, pmd); 2236 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); 2237 } else { 2238 if (arch_needs_pgtable_deposit()) 2239 zap_deposited_table(tlb->mm, pmd); 2240 add_mm_counter(tlb->mm, mm_counter_file(folio), 2241 -HPAGE_PMD_NR); 2242 2243 /* 2244 * Use flush_needed to indicate whether the PMD entry 2245 * is present, instead of checking pmd_present() again. 2246 */ 2247 if (flush_needed && pmd_young(orig_pmd) && 2248 likely(vma_has_recency(vma))) 2249 folio_mark_accessed(folio); 2250 } 2251 2252 spin_unlock(ptl); 2253 if (flush_needed) 2254 tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE); 2255 } 2256 return 1; 2257 } 2258 2259 #ifndef pmd_move_must_withdraw 2260 static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl, 2261 spinlock_t *old_pmd_ptl, 2262 struct vm_area_struct *vma) 2263 { 2264 /* 2265 * With split pmd lock we also need to move preallocated 2266 * PTE page table if new_pmd is on different PMD page table. 2267 * 2268 * We also don't deposit and withdraw tables for file pages. 2269 */ 2270 return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma); 2271 } 2272 #endif 2273 2274 static pmd_t move_soft_dirty_pmd(pmd_t pmd) 2275 { 2276 #ifdef CONFIG_MEM_SOFT_DIRTY 2277 if (unlikely(is_pmd_migration_entry(pmd))) 2278 pmd = pmd_swp_mksoft_dirty(pmd); 2279 else if (pmd_present(pmd)) 2280 pmd = pmd_mksoft_dirty(pmd); 2281 #endif 2282 return pmd; 2283 } 2284 2285 static pmd_t clear_uffd_wp_pmd(pmd_t pmd) 2286 { 2287 if (pmd_present(pmd)) 2288 pmd = pmd_clear_uffd_wp(pmd); 2289 else if (is_swap_pmd(pmd)) 2290 pmd = pmd_swp_clear_uffd_wp(pmd); 2291 2292 return pmd; 2293 } 2294 2295 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, 2296 unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd) 2297 { 2298 spinlock_t *old_ptl, *new_ptl; 2299 pmd_t pmd; 2300 struct mm_struct *mm = vma->vm_mm; 2301 bool force_flush = false; 2302 2303 /* 2304 * The destination pmd shouldn't be established, free_pgtables() 2305 * should have released it; but move_page_tables() might have already 2306 * inserted a page table, if racing against shmem/file collapse. 2307 */ 2308 if (!pmd_none(*new_pmd)) { 2309 VM_BUG_ON(pmd_trans_huge(*new_pmd)); 2310 return false; 2311 } 2312 2313 /* 2314 * We don't have to worry about the ordering of src and dst 2315 * ptlocks because exclusive mmap_lock prevents deadlock. 2316 */ 2317 old_ptl = __pmd_trans_huge_lock(old_pmd, vma); 2318 if (old_ptl) { 2319 new_ptl = pmd_lockptr(mm, new_pmd); 2320 if (new_ptl != old_ptl) 2321 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 2322 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); 2323 if (pmd_present(pmd)) 2324 force_flush = true; 2325 VM_BUG_ON(!pmd_none(*new_pmd)); 2326 2327 if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) { 2328 pgtable_t pgtable; 2329 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); 2330 pgtable_trans_huge_deposit(mm, new_pmd, pgtable); 2331 } 2332 pmd = move_soft_dirty_pmd(pmd); 2333 if (vma_has_uffd_without_event_remap(vma)) 2334 pmd = clear_uffd_wp_pmd(pmd); 2335 set_pmd_at(mm, new_addr, new_pmd, pmd); 2336 if (force_flush) 2337 flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE); 2338 if (new_ptl != old_ptl) 2339 spin_unlock(new_ptl); 2340 spin_unlock(old_ptl); 2341 return true; 2342 } 2343 return false; 2344 } 2345 2346 /* 2347 * Returns 2348 * - 0 if PMD could not be locked 2349 * - 1 if PMD was locked but protections unchanged and TLB flush unnecessary 2350 * or if prot_numa but THP migration is not supported 2351 * - HPAGE_PMD_NR if protections changed and TLB flush necessary 2352 */ 2353 int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2354 pmd_t *pmd, unsigned long addr, pgprot_t newprot, 2355 unsigned long cp_flags) 2356 { 2357 struct mm_struct *mm = vma->vm_mm; 2358 spinlock_t *ptl; 2359 pmd_t oldpmd, entry; 2360 bool prot_numa = cp_flags & MM_CP_PROT_NUMA; 2361 bool uffd_wp = cp_flags & MM_CP_UFFD_WP; 2362 bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; 2363 int ret = 1; 2364 2365 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2366 2367 if (prot_numa && !thp_migration_supported()) 2368 return 1; 2369 2370 ptl = __pmd_trans_huge_lock(pmd, vma); 2371 if (!ptl) 2372 return 0; 2373 2374 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 2375 if (is_swap_pmd(*pmd)) { 2376 swp_entry_t entry = pmd_to_swp_entry(*pmd); 2377 struct folio *folio = pfn_swap_entry_folio(entry); 2378 pmd_t newpmd; 2379 2380 VM_BUG_ON(!is_pmd_migration_entry(*pmd)); 2381 if (is_writable_migration_entry(entry)) { 2382 /* 2383 * A protection check is difficult so 2384 * just be safe and disable write 2385 */ 2386 if (folio_test_anon(folio)) 2387 entry = make_readable_exclusive_migration_entry(swp_offset(entry)); 2388 else 2389 entry = make_readable_migration_entry(swp_offset(entry)); 2390 newpmd = swp_entry_to_pmd(entry); 2391 if (pmd_swp_soft_dirty(*pmd)) 2392 newpmd = pmd_swp_mksoft_dirty(newpmd); 2393 } else { 2394 newpmd = *pmd; 2395 } 2396 2397 if (uffd_wp) 2398 newpmd = pmd_swp_mkuffd_wp(newpmd); 2399 else if (uffd_wp_resolve) 2400 newpmd = pmd_swp_clear_uffd_wp(newpmd); 2401 if (!pmd_same(*pmd, newpmd)) 2402 set_pmd_at(mm, addr, pmd, newpmd); 2403 goto unlock; 2404 } 2405 #endif 2406 2407 if (prot_numa) { 2408 struct folio *folio; 2409 bool toptier; 2410 /* 2411 * Avoid trapping faults against the zero page. The read-only 2412 * data is likely to be read-cached on the local CPU and 2413 * local/remote hits to the zero page are not interesting. 2414 */ 2415 if (is_huge_zero_pmd(*pmd)) 2416 goto unlock; 2417 2418 if (pmd_protnone(*pmd)) 2419 goto unlock; 2420 2421 folio = pmd_folio(*pmd); 2422 toptier = node_is_toptier(folio_nid(folio)); 2423 /* 2424 * Skip scanning top tier node if normal numa 2425 * balancing is disabled 2426 */ 2427 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) && 2428 toptier) 2429 goto unlock; 2430 2431 if (folio_use_access_time(folio)) 2432 folio_xchg_access_time(folio, 2433 jiffies_to_msecs(jiffies)); 2434 } 2435 /* 2436 * In case prot_numa, we are under mmap_read_lock(mm). It's critical 2437 * to not clear pmd intermittently to avoid race with MADV_DONTNEED 2438 * which is also under mmap_read_lock(mm): 2439 * 2440 * CPU0: CPU1: 2441 * change_huge_pmd(prot_numa=1) 2442 * pmdp_huge_get_and_clear_notify() 2443 * madvise_dontneed() 2444 * zap_pmd_range() 2445 * pmd_trans_huge(*pmd) == 0 (without ptl) 2446 * // skip the pmd 2447 * set_pmd_at(); 2448 * // pmd is re-established 2449 * 2450 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it 2451 * which may break userspace. 2452 * 2453 * pmdp_invalidate_ad() is required to make sure we don't miss 2454 * dirty/young flags set by hardware. 2455 */ 2456 oldpmd = pmdp_invalidate_ad(vma, addr, pmd); 2457 2458 entry = pmd_modify(oldpmd, newprot); 2459 if (uffd_wp) 2460 entry = pmd_mkuffd_wp(entry); 2461 else if (uffd_wp_resolve) 2462 /* 2463 * Leave the write bit to be handled by PF interrupt 2464 * handler, then things like COW could be properly 2465 * handled. 2466 */ 2467 entry = pmd_clear_uffd_wp(entry); 2468 2469 /* See change_pte_range(). */ 2470 if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) && 2471 can_change_pmd_writable(vma, addr, entry)) 2472 entry = pmd_mkwrite(entry, vma); 2473 2474 ret = HPAGE_PMD_NR; 2475 set_pmd_at(mm, addr, pmd, entry); 2476 2477 if (huge_pmd_needs_flush(oldpmd, entry)) 2478 tlb_flush_pmd_range(tlb, addr, HPAGE_PMD_SIZE); 2479 unlock: 2480 spin_unlock(ptl); 2481 return ret; 2482 } 2483 2484 /* 2485 * Returns: 2486 * 2487 * - 0: if pud leaf changed from under us 2488 * - 1: if pud can be skipped 2489 * - HPAGE_PUD_NR: if pud was successfully processed 2490 */ 2491 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2492 int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 2493 pud_t *pudp, unsigned long addr, pgprot_t newprot, 2494 unsigned long cp_flags) 2495 { 2496 struct mm_struct *mm = vma->vm_mm; 2497 pud_t oldpud, entry; 2498 spinlock_t *ptl; 2499 2500 tlb_change_page_size(tlb, HPAGE_PUD_SIZE); 2501 2502 /* NUMA balancing doesn't apply to dax */ 2503 if (cp_flags & MM_CP_PROT_NUMA) 2504 return 1; 2505 2506 /* 2507 * Huge entries on userfault-wp only works with anonymous, while we 2508 * don't have anonymous PUDs yet. 2509 */ 2510 if (WARN_ON_ONCE(cp_flags & MM_CP_UFFD_WP_ALL)) 2511 return 1; 2512 2513 ptl = __pud_trans_huge_lock(pudp, vma); 2514 if (!ptl) 2515 return 0; 2516 2517 /* 2518 * Can't clear PUD or it can race with concurrent zapping. See 2519 * change_huge_pmd(). 2520 */ 2521 oldpud = pudp_invalidate(vma, addr, pudp); 2522 entry = pud_modify(oldpud, newprot); 2523 set_pud_at(mm, addr, pudp, entry); 2524 tlb_flush_pud_range(tlb, addr, HPAGE_PUD_SIZE); 2525 2526 spin_unlock(ptl); 2527 return HPAGE_PUD_NR; 2528 } 2529 #endif 2530 2531 #ifdef CONFIG_USERFAULTFD 2532 /* 2533 * The PT lock for src_pmd and dst_vma/src_vma (for reading) are locked by 2534 * the caller, but it must return after releasing the page_table_lock. 2535 * Just move the page from src_pmd to dst_pmd if possible. 2536 * Return zero if succeeded in moving the page, -EAGAIN if it needs to be 2537 * repeated by the caller, or other errors in case of failure. 2538 */ 2539 int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval, 2540 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, 2541 unsigned long dst_addr, unsigned long src_addr) 2542 { 2543 pmd_t _dst_pmd, src_pmdval; 2544 struct page *src_page; 2545 struct folio *src_folio; 2546 struct anon_vma *src_anon_vma; 2547 spinlock_t *src_ptl, *dst_ptl; 2548 pgtable_t src_pgtable; 2549 struct mmu_notifier_range range; 2550 int err = 0; 2551 2552 src_pmdval = *src_pmd; 2553 src_ptl = pmd_lockptr(mm, src_pmd); 2554 2555 lockdep_assert_held(src_ptl); 2556 vma_assert_locked(src_vma); 2557 vma_assert_locked(dst_vma); 2558 2559 /* Sanity checks before the operation */ 2560 if (WARN_ON_ONCE(!pmd_none(dst_pmdval)) || WARN_ON_ONCE(src_addr & ~HPAGE_PMD_MASK) || 2561 WARN_ON_ONCE(dst_addr & ~HPAGE_PMD_MASK)) { 2562 spin_unlock(src_ptl); 2563 return -EINVAL; 2564 } 2565 2566 if (!pmd_trans_huge(src_pmdval)) { 2567 spin_unlock(src_ptl); 2568 if (is_pmd_migration_entry(src_pmdval)) { 2569 pmd_migration_entry_wait(mm, &src_pmdval); 2570 return -EAGAIN; 2571 } 2572 return -ENOENT; 2573 } 2574 2575 src_page = pmd_page(src_pmdval); 2576 2577 if (!is_huge_zero_pmd(src_pmdval)) { 2578 if (unlikely(!PageAnonExclusive(src_page))) { 2579 spin_unlock(src_ptl); 2580 return -EBUSY; 2581 } 2582 2583 src_folio = page_folio(src_page); 2584 folio_get(src_folio); 2585 } else 2586 src_folio = NULL; 2587 2588 spin_unlock(src_ptl); 2589 2590 flush_cache_range(src_vma, src_addr, src_addr + HPAGE_PMD_SIZE); 2591 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, src_addr, 2592 src_addr + HPAGE_PMD_SIZE); 2593 mmu_notifier_invalidate_range_start(&range); 2594 2595 if (src_folio) { 2596 folio_lock(src_folio); 2597 2598 /* 2599 * split_huge_page walks the anon_vma chain without the page 2600 * lock. Serialize against it with the anon_vma lock, the page 2601 * lock is not enough. 2602 */ 2603 src_anon_vma = folio_get_anon_vma(src_folio); 2604 if (!src_anon_vma) { 2605 err = -EAGAIN; 2606 goto unlock_folio; 2607 } 2608 anon_vma_lock_write(src_anon_vma); 2609 } else 2610 src_anon_vma = NULL; 2611 2612 dst_ptl = pmd_lockptr(mm, dst_pmd); 2613 double_pt_lock(src_ptl, dst_ptl); 2614 if (unlikely(!pmd_same(*src_pmd, src_pmdval) || 2615 !pmd_same(*dst_pmd, dst_pmdval))) { 2616 err = -EAGAIN; 2617 goto unlock_ptls; 2618 } 2619 if (src_folio) { 2620 if (folio_maybe_dma_pinned(src_folio) || 2621 !PageAnonExclusive(&src_folio->page)) { 2622 err = -EBUSY; 2623 goto unlock_ptls; 2624 } 2625 2626 if (WARN_ON_ONCE(!folio_test_head(src_folio)) || 2627 WARN_ON_ONCE(!folio_test_anon(src_folio))) { 2628 err = -EBUSY; 2629 goto unlock_ptls; 2630 } 2631 2632 src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); 2633 /* Folio got pinned from under us. Put it back and fail the move. */ 2634 if (folio_maybe_dma_pinned(src_folio)) { 2635 set_pmd_at(mm, src_addr, src_pmd, src_pmdval); 2636 err = -EBUSY; 2637 goto unlock_ptls; 2638 } 2639 2640 folio_move_anon_rmap(src_folio, dst_vma); 2641 src_folio->index = linear_page_index(dst_vma, dst_addr); 2642 2643 _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); 2644 /* Follow mremap() behavior and treat the entry dirty after the move */ 2645 _dst_pmd = pmd_mkwrite(pmd_mkdirty(_dst_pmd), dst_vma); 2646 } else { 2647 src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); 2648 _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); 2649 } 2650 set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd); 2651 2652 src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd); 2653 pgtable_trans_huge_deposit(mm, dst_pmd, src_pgtable); 2654 unlock_ptls: 2655 double_pt_unlock(src_ptl, dst_ptl); 2656 if (src_anon_vma) { 2657 anon_vma_unlock_write(src_anon_vma); 2658 put_anon_vma(src_anon_vma); 2659 } 2660 unlock_folio: 2661 /* unblock rmap walks */ 2662 if (src_folio) 2663 folio_unlock(src_folio); 2664 mmu_notifier_invalidate_range_end(&range); 2665 if (src_folio) 2666 folio_put(src_folio); 2667 return err; 2668 } 2669 #endif /* CONFIG_USERFAULTFD */ 2670 2671 /* 2672 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise. 2673 * 2674 * Note that if it returns page table lock pointer, this routine returns without 2675 * unlocking page table lock. So callers must unlock it. 2676 */ 2677 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) 2678 { 2679 spinlock_t *ptl; 2680 ptl = pmd_lock(vma->vm_mm, pmd); 2681 if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd))) 2682 return ptl; 2683 spin_unlock(ptl); 2684 return NULL; 2685 } 2686 2687 /* 2688 * Returns page table lock pointer if a given pud maps a thp, NULL otherwise. 2689 * 2690 * Note that if it returns page table lock pointer, this routine returns without 2691 * unlocking page table lock. So callers must unlock it. 2692 */ 2693 spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma) 2694 { 2695 spinlock_t *ptl; 2696 2697 ptl = pud_lock(vma->vm_mm, pud); 2698 if (likely(pud_trans_huge(*pud))) 2699 return ptl; 2700 spin_unlock(ptl); 2701 return NULL; 2702 } 2703 2704 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2705 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 2706 pud_t *pud, unsigned long addr) 2707 { 2708 spinlock_t *ptl; 2709 pud_t orig_pud; 2710 2711 ptl = __pud_trans_huge_lock(pud, vma); 2712 if (!ptl) 2713 return 0; 2714 2715 orig_pud = pudp_huge_get_and_clear_full(vma, addr, pud, tlb->fullmm); 2716 arch_check_zapped_pud(vma, orig_pud); 2717 tlb_remove_pud_tlb_entry(tlb, pud, addr); 2718 if (!vma_is_dax(vma) && vma_is_special_huge(vma)) { 2719 spin_unlock(ptl); 2720 /* No zero page support yet */ 2721 } else { 2722 struct page *page = NULL; 2723 struct folio *folio; 2724 2725 /* No support for anonymous PUD pages or migration yet */ 2726 VM_WARN_ON_ONCE(vma_is_anonymous(vma) || 2727 !pud_present(orig_pud)); 2728 2729 page = pud_page(orig_pud); 2730 folio = page_folio(page); 2731 folio_remove_rmap_pud(folio, page, vma); 2732 add_mm_counter(tlb->mm, mm_counter_file(folio), -HPAGE_PUD_NR); 2733 2734 spin_unlock(ptl); 2735 tlb_remove_page_size(tlb, page, HPAGE_PUD_SIZE); 2736 } 2737 return 1; 2738 } 2739 2740 static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud, 2741 unsigned long haddr) 2742 { 2743 struct folio *folio; 2744 struct page *page; 2745 pud_t old_pud; 2746 2747 VM_BUG_ON(haddr & ~HPAGE_PUD_MASK); 2748 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2749 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma); 2750 VM_BUG_ON(!pud_trans_huge(*pud)); 2751 2752 count_vm_event(THP_SPLIT_PUD); 2753 2754 old_pud = pudp_huge_clear_flush(vma, haddr, pud); 2755 2756 if (!vma_is_dax(vma)) 2757 return; 2758 2759 page = pud_page(old_pud); 2760 folio = page_folio(page); 2761 2762 if (!folio_test_dirty(folio) && pud_dirty(old_pud)) 2763 folio_mark_dirty(folio); 2764 if (!folio_test_referenced(folio) && pud_young(old_pud)) 2765 folio_set_referenced(folio); 2766 folio_remove_rmap_pud(folio, page, vma); 2767 folio_put(folio); 2768 add_mm_counter(vma->vm_mm, mm_counter_file(folio), 2769 -HPAGE_PUD_NR); 2770 } 2771 2772 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 2773 unsigned long address) 2774 { 2775 spinlock_t *ptl; 2776 struct mmu_notifier_range range; 2777 2778 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 2779 address & HPAGE_PUD_MASK, 2780 (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE); 2781 mmu_notifier_invalidate_range_start(&range); 2782 ptl = pud_lock(vma->vm_mm, pud); 2783 if (unlikely(!pud_trans_huge(*pud))) 2784 goto out; 2785 __split_huge_pud_locked(vma, pud, range.start); 2786 2787 out: 2788 spin_unlock(ptl); 2789 mmu_notifier_invalidate_range_end(&range); 2790 } 2791 #else 2792 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 2793 unsigned long address) 2794 { 2795 } 2796 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 2797 2798 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 2799 unsigned long haddr, pmd_t *pmd) 2800 { 2801 struct mm_struct *mm = vma->vm_mm; 2802 pgtable_t pgtable; 2803 pmd_t _pmd, old_pmd; 2804 unsigned long addr; 2805 pte_t *pte; 2806 int i; 2807 2808 /* 2809 * Leave pmd empty until pte is filled note that it is fine to delay 2810 * notification until mmu_notifier_invalidate_range_end() as we are 2811 * replacing a zero pmd write protected page with a zero pte write 2812 * protected page. 2813 * 2814 * See Documentation/mm/mmu_notifier.rst 2815 */ 2816 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); 2817 2818 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2819 pmd_populate(mm, &_pmd, pgtable); 2820 2821 pte = pte_offset_map(&_pmd, haddr); 2822 VM_BUG_ON(!pte); 2823 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2824 pte_t entry; 2825 2826 entry = pfn_pte(my_zero_pfn(addr), vma->vm_page_prot); 2827 entry = pte_mkspecial(entry); 2828 if (pmd_uffd_wp(old_pmd)) 2829 entry = pte_mkuffd_wp(entry); 2830 VM_BUG_ON(!pte_none(ptep_get(pte))); 2831 set_pte_at(mm, addr, pte, entry); 2832 pte++; 2833 } 2834 pte_unmap(pte - 1); 2835 smp_wmb(); /* make pte visible before pmd */ 2836 pmd_populate(mm, pmd, pgtable); 2837 } 2838 2839 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 2840 unsigned long haddr, bool freeze) 2841 { 2842 struct mm_struct *mm = vma->vm_mm; 2843 struct folio *folio; 2844 struct page *page; 2845 pgtable_t pgtable; 2846 pmd_t old_pmd, _pmd; 2847 bool young, write, soft_dirty, pmd_migration = false, uffd_wp = false; 2848 bool anon_exclusive = false, dirty = false; 2849 unsigned long addr; 2850 pte_t *pte; 2851 int i; 2852 2853 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 2854 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2855 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 2856 VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd)); 2857 2858 count_vm_event(THP_SPLIT_PMD); 2859 2860 if (!vma_is_anonymous(vma)) { 2861 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); 2862 /* 2863 * We are going to unmap this huge page. So 2864 * just go ahead and zap it 2865 */ 2866 if (arch_needs_pgtable_deposit()) 2867 zap_deposited_table(mm, pmd); 2868 if (!vma_is_dax(vma) && vma_is_special_huge(vma)) 2869 return; 2870 if (unlikely(is_pmd_migration_entry(old_pmd))) { 2871 swp_entry_t entry; 2872 2873 entry = pmd_to_swp_entry(old_pmd); 2874 folio = pfn_swap_entry_folio(entry); 2875 } else if (is_huge_zero_pmd(old_pmd)) { 2876 return; 2877 } else { 2878 page = pmd_page(old_pmd); 2879 folio = page_folio(page); 2880 if (!folio_test_dirty(folio) && pmd_dirty(old_pmd)) 2881 folio_mark_dirty(folio); 2882 if (!folio_test_referenced(folio) && pmd_young(old_pmd)) 2883 folio_set_referenced(folio); 2884 folio_remove_rmap_pmd(folio, page, vma); 2885 folio_put(folio); 2886 } 2887 add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR); 2888 return; 2889 } 2890 2891 if (is_huge_zero_pmd(*pmd)) { 2892 /* 2893 * FIXME: Do we want to invalidate secondary mmu by calling 2894 * mmu_notifier_arch_invalidate_secondary_tlbs() see comments below 2895 * inside __split_huge_pmd() ? 2896 * 2897 * We are going from a zero huge page write protected to zero 2898 * small page also write protected so it does not seems useful 2899 * to invalidate secondary mmu at this time. 2900 */ 2901 return __split_huge_zero_page_pmd(vma, haddr, pmd); 2902 } 2903 2904 pmd_migration = is_pmd_migration_entry(*pmd); 2905 if (unlikely(pmd_migration)) { 2906 swp_entry_t entry; 2907 2908 old_pmd = *pmd; 2909 entry = pmd_to_swp_entry(old_pmd); 2910 page = pfn_swap_entry_to_page(entry); 2911 write = is_writable_migration_entry(entry); 2912 if (PageAnon(page)) 2913 anon_exclusive = is_readable_exclusive_migration_entry(entry); 2914 young = is_migration_entry_young(entry); 2915 dirty = is_migration_entry_dirty(entry); 2916 soft_dirty = pmd_swp_soft_dirty(old_pmd); 2917 uffd_wp = pmd_swp_uffd_wp(old_pmd); 2918 } else { 2919 /* 2920 * Up to this point the pmd is present and huge and userland has 2921 * the whole access to the hugepage during the split (which 2922 * happens in place). If we overwrite the pmd with the not-huge 2923 * version pointing to the pte here (which of course we could if 2924 * all CPUs were bug free), userland could trigger a small page 2925 * size TLB miss on the small sized TLB while the hugepage TLB 2926 * entry is still established in the huge TLB. Some CPU doesn't 2927 * like that. See 2928 * http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum 2929 * 383 on page 105. Intel should be safe but is also warns that 2930 * it's only safe if the permission and cache attributes of the 2931 * two entries loaded in the two TLB is identical (which should 2932 * be the case here). But it is generally safer to never allow 2933 * small and huge TLB entries for the same virtual address to be 2934 * loaded simultaneously. So instead of doing "pmd_populate(); 2935 * flush_pmd_tlb_range();" we first mark the current pmd 2936 * notpresent (atomically because here the pmd_trans_huge must 2937 * remain set at all times on the pmd until the split is 2938 * complete for this pmd), then we flush the SMP TLB and finally 2939 * we write the non-huge version of the pmd entry with 2940 * pmd_populate. 2941 */ 2942 old_pmd = pmdp_invalidate(vma, haddr, pmd); 2943 page = pmd_page(old_pmd); 2944 folio = page_folio(page); 2945 if (pmd_dirty(old_pmd)) { 2946 dirty = true; 2947 folio_set_dirty(folio); 2948 } 2949 write = pmd_write(old_pmd); 2950 young = pmd_young(old_pmd); 2951 soft_dirty = pmd_soft_dirty(old_pmd); 2952 uffd_wp = pmd_uffd_wp(old_pmd); 2953 2954 VM_WARN_ON_FOLIO(!folio_ref_count(folio), folio); 2955 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); 2956 2957 /* 2958 * Without "freeze", we'll simply split the PMD, propagating the 2959 * PageAnonExclusive() flag for each PTE by setting it for 2960 * each subpage -- no need to (temporarily) clear. 2961 * 2962 * With "freeze" we want to replace mapped pages by 2963 * migration entries right away. This is only possible if we 2964 * managed to clear PageAnonExclusive() -- see 2965 * set_pmd_migration_entry(). 2966 * 2967 * In case we cannot clear PageAnonExclusive(), split the PMD 2968 * only and let try_to_migrate_one() fail later. 2969 * 2970 * See folio_try_share_anon_rmap_pmd(): invalidate PMD first. 2971 */ 2972 anon_exclusive = PageAnonExclusive(page); 2973 if (freeze && anon_exclusive && 2974 folio_try_share_anon_rmap_pmd(folio, page)) 2975 freeze = false; 2976 if (!freeze) { 2977 rmap_t rmap_flags = RMAP_NONE; 2978 2979 folio_ref_add(folio, HPAGE_PMD_NR - 1); 2980 if (anon_exclusive) 2981 rmap_flags |= RMAP_EXCLUSIVE; 2982 folio_add_anon_rmap_ptes(folio, page, HPAGE_PMD_NR, 2983 vma, haddr, rmap_flags); 2984 } 2985 } 2986 2987 /* 2988 * Withdraw the table only after we mark the pmd entry invalid. 2989 * This's critical for some architectures (Power). 2990 */ 2991 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2992 pmd_populate(mm, &_pmd, pgtable); 2993 2994 pte = pte_offset_map(&_pmd, haddr); 2995 VM_BUG_ON(!pte); 2996 2997 /* 2998 * Note that NUMA hinting access restrictions are not transferred to 2999 * avoid any possibility of altering permissions across VMAs. 3000 */ 3001 if (freeze || pmd_migration) { 3002 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 3003 pte_t entry; 3004 swp_entry_t swp_entry; 3005 3006 if (write) 3007 swp_entry = make_writable_migration_entry( 3008 page_to_pfn(page + i)); 3009 else if (anon_exclusive) 3010 swp_entry = make_readable_exclusive_migration_entry( 3011 page_to_pfn(page + i)); 3012 else 3013 swp_entry = make_readable_migration_entry( 3014 page_to_pfn(page + i)); 3015 if (young) 3016 swp_entry = make_migration_entry_young(swp_entry); 3017 if (dirty) 3018 swp_entry = make_migration_entry_dirty(swp_entry); 3019 entry = swp_entry_to_pte(swp_entry); 3020 if (soft_dirty) 3021 entry = pte_swp_mksoft_dirty(entry); 3022 if (uffd_wp) 3023 entry = pte_swp_mkuffd_wp(entry); 3024 3025 VM_WARN_ON(!pte_none(ptep_get(pte + i))); 3026 set_pte_at(mm, addr, pte + i, entry); 3027 } 3028 } else { 3029 pte_t entry; 3030 3031 entry = mk_pte(page, READ_ONCE(vma->vm_page_prot)); 3032 if (write) 3033 entry = pte_mkwrite(entry, vma); 3034 if (!young) 3035 entry = pte_mkold(entry); 3036 /* NOTE: this may set soft-dirty too on some archs */ 3037 if (dirty) 3038 entry = pte_mkdirty(entry); 3039 if (soft_dirty) 3040 entry = pte_mksoft_dirty(entry); 3041 if (uffd_wp) 3042 entry = pte_mkuffd_wp(entry); 3043 3044 for (i = 0; i < HPAGE_PMD_NR; i++) 3045 VM_WARN_ON(!pte_none(ptep_get(pte + i))); 3046 3047 set_ptes(mm, haddr, pte, entry, HPAGE_PMD_NR); 3048 } 3049 pte_unmap(pte); 3050 3051 if (!pmd_migration) 3052 folio_remove_rmap_pmd(folio, page, vma); 3053 if (freeze) 3054 put_page(page); 3055 3056 smp_wmb(); /* make pte visible before pmd */ 3057 pmd_populate(mm, pmd, pgtable); 3058 } 3059 3060 void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, 3061 pmd_t *pmd, bool freeze) 3062 { 3063 VM_WARN_ON_ONCE(!IS_ALIGNED(address, HPAGE_PMD_SIZE)); 3064 if (pmd_trans_huge(*pmd) || is_pmd_migration_entry(*pmd)) 3065 __split_huge_pmd_locked(vma, pmd, address, freeze); 3066 } 3067 3068 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 3069 unsigned long address, bool freeze) 3070 { 3071 spinlock_t *ptl; 3072 struct mmu_notifier_range range; 3073 3074 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 3075 address & HPAGE_PMD_MASK, 3076 (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE); 3077 mmu_notifier_invalidate_range_start(&range); 3078 ptl = pmd_lock(vma->vm_mm, pmd); 3079 split_huge_pmd_locked(vma, range.start, pmd, freeze); 3080 spin_unlock(ptl); 3081 mmu_notifier_invalidate_range_end(&range); 3082 } 3083 3084 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, 3085 bool freeze) 3086 { 3087 pmd_t *pmd = mm_find_pmd(vma->vm_mm, address); 3088 3089 if (!pmd) 3090 return; 3091 3092 __split_huge_pmd(vma, pmd, address, freeze); 3093 } 3094 3095 static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address) 3096 { 3097 /* 3098 * If the new address isn't hpage aligned and it could previously 3099 * contain an hugepage: check if we need to split an huge pmd. 3100 */ 3101 if (!IS_ALIGNED(address, HPAGE_PMD_SIZE) && 3102 range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE), 3103 ALIGN(address, HPAGE_PMD_SIZE))) 3104 split_huge_pmd_address(vma, address, false); 3105 } 3106 3107 void vma_adjust_trans_huge(struct vm_area_struct *vma, 3108 unsigned long start, 3109 unsigned long end, 3110 struct vm_area_struct *next) 3111 { 3112 /* Check if we need to split start first. */ 3113 split_huge_pmd_if_needed(vma, start); 3114 3115 /* Check if we need to split end next. */ 3116 split_huge_pmd_if_needed(vma, end); 3117 3118 /* If we're incrementing next->vm_start, we might need to split it. */ 3119 if (next) 3120 split_huge_pmd_if_needed(next, end); 3121 } 3122 3123 static void unmap_folio(struct folio *folio) 3124 { 3125 enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SYNC | 3126 TTU_BATCH_FLUSH; 3127 3128 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 3129 3130 if (folio_test_pmd_mappable(folio)) 3131 ttu_flags |= TTU_SPLIT_HUGE_PMD; 3132 3133 /* 3134 * Anon pages need migration entries to preserve them, but file 3135 * pages can simply be left unmapped, then faulted back on demand. 3136 * If that is ever changed (perhaps for mlock), update remap_page(). 3137 */ 3138 if (folio_test_anon(folio)) 3139 try_to_migrate(folio, ttu_flags); 3140 else 3141 try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK); 3142 3143 try_to_unmap_flush(); 3144 } 3145 3146 static bool __discard_anon_folio_pmd_locked(struct vm_area_struct *vma, 3147 unsigned long addr, pmd_t *pmdp, 3148 struct folio *folio) 3149 { 3150 struct mm_struct *mm = vma->vm_mm; 3151 int ref_count, map_count; 3152 pmd_t orig_pmd = *pmdp; 3153 3154 if (pmd_dirty(orig_pmd)) 3155 folio_set_dirty(folio); 3156 if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { 3157 folio_set_swapbacked(folio); 3158 return false; 3159 } 3160 3161 orig_pmd = pmdp_huge_clear_flush(vma, addr, pmdp); 3162 3163 /* 3164 * Syncing against concurrent GUP-fast: 3165 * - clear PMD; barrier; read refcount 3166 * - inc refcount; barrier; read PMD 3167 */ 3168 smp_mb(); 3169 3170 ref_count = folio_ref_count(folio); 3171 map_count = folio_mapcount(folio); 3172 3173 /* 3174 * Order reads for folio refcount and dirty flag 3175 * (see comments in __remove_mapping()). 3176 */ 3177 smp_rmb(); 3178 3179 /* 3180 * If the folio or its PMD is redirtied at this point, or if there 3181 * are unexpected references, we will give up to discard this folio 3182 * and remap it. 3183 * 3184 * The only folio refs must be one from isolation plus the rmap(s). 3185 */ 3186 if (pmd_dirty(orig_pmd)) 3187 folio_set_dirty(folio); 3188 if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { 3189 folio_set_swapbacked(folio); 3190 set_pmd_at(mm, addr, pmdp, orig_pmd); 3191 return false; 3192 } 3193 3194 if (ref_count != map_count + 1) { 3195 set_pmd_at(mm, addr, pmdp, orig_pmd); 3196 return false; 3197 } 3198 3199 folio_remove_rmap_pmd(folio, pmd_page(orig_pmd), vma); 3200 zap_deposited_table(mm, pmdp); 3201 add_mm_counter(mm, MM_ANONPAGES, -HPAGE_PMD_NR); 3202 if (vma->vm_flags & VM_LOCKED) 3203 mlock_drain_local(); 3204 folio_put(folio); 3205 3206 return true; 3207 } 3208 3209 bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr, 3210 pmd_t *pmdp, struct folio *folio) 3211 { 3212 VM_WARN_ON_FOLIO(!folio_test_pmd_mappable(folio), folio); 3213 VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); 3214 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); 3215 VM_WARN_ON_FOLIO(folio_test_swapbacked(folio), folio); 3216 VM_WARN_ON_ONCE(!IS_ALIGNED(addr, HPAGE_PMD_SIZE)); 3217 3218 return __discard_anon_folio_pmd_locked(vma, addr, pmdp, folio); 3219 } 3220 3221 static void remap_page(struct folio *folio, unsigned long nr, int flags) 3222 { 3223 int i = 0; 3224 3225 /* If unmap_folio() uses try_to_migrate() on file, remove this check */ 3226 if (!folio_test_anon(folio)) 3227 return; 3228 for (;;) { 3229 remove_migration_ptes(folio, folio, RMP_LOCKED | flags); 3230 i += folio_nr_pages(folio); 3231 if (i >= nr) 3232 break; 3233 folio = folio_next(folio); 3234 } 3235 } 3236 3237 static void lru_add_split_folio(struct folio *folio, struct folio *new_folio, 3238 struct lruvec *lruvec, struct list_head *list) 3239 { 3240 VM_BUG_ON_FOLIO(folio_test_lru(new_folio), folio); 3241 lockdep_assert_held(&lruvec->lru_lock); 3242 3243 if (list) { 3244 /* page reclaim is reclaiming a huge page */ 3245 VM_WARN_ON(folio_test_lru(folio)); 3246 folio_get(new_folio); 3247 list_add_tail(&new_folio->lru, list); 3248 } else { 3249 /* head is still on lru (and we have it frozen) */ 3250 VM_WARN_ON(!folio_test_lru(folio)); 3251 if (folio_test_unevictable(folio)) 3252 new_folio->mlock_count = 0; 3253 else 3254 list_add_tail(&new_folio->lru, &folio->lru); 3255 folio_set_lru(new_folio); 3256 } 3257 } 3258 3259 /* Racy check whether the huge page can be split */ 3260 bool can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins) 3261 { 3262 int extra_pins; 3263 3264 /* Additional pins from page cache */ 3265 if (folio_test_anon(folio)) 3266 extra_pins = folio_test_swapcache(folio) ? 3267 folio_nr_pages(folio) : 0; 3268 else 3269 extra_pins = folio_nr_pages(folio); 3270 if (pextra_pins) 3271 *pextra_pins = extra_pins; 3272 return folio_mapcount(folio) == folio_ref_count(folio) - extra_pins - 3273 caller_pins; 3274 } 3275 3276 static bool page_range_has_hwpoisoned(struct page *page, long nr_pages) 3277 { 3278 for (; nr_pages; page++, nr_pages--) 3279 if (PageHWPoison(page)) 3280 return true; 3281 return false; 3282 } 3283 3284 /* 3285 * It splits @folio into @new_order folios and copies the @folio metadata to 3286 * all the resulting folios. 3287 */ 3288 static void __split_folio_to_order(struct folio *folio, int old_order, 3289 int new_order) 3290 { 3291 /* Scan poisoned pages when split a poisoned folio to large folios */ 3292 const bool handle_hwpoison = folio_test_has_hwpoisoned(folio) && new_order; 3293 long new_nr_pages = 1 << new_order; 3294 long nr_pages = 1 << old_order; 3295 long i; 3296 3297 folio_clear_has_hwpoisoned(folio); 3298 3299 /* Check first new_nr_pages since the loop below skips them */ 3300 if (handle_hwpoison && 3301 page_range_has_hwpoisoned(folio_page(folio, 0), new_nr_pages)) 3302 folio_set_has_hwpoisoned(folio); 3303 /* 3304 * Skip the first new_nr_pages, since the new folio from them have all 3305 * the flags from the original folio. 3306 */ 3307 for (i = new_nr_pages; i < nr_pages; i += new_nr_pages) { 3308 struct page *new_head = &folio->page + i; 3309 /* 3310 * Careful: new_folio is not a "real" folio before we cleared PageTail. 3311 * Don't pass it around before clear_compound_head(). 3312 */ 3313 struct folio *new_folio = (struct folio *)new_head; 3314 3315 VM_BUG_ON_PAGE(atomic_read(&new_folio->_mapcount) != -1, new_head); 3316 3317 /* 3318 * Clone page flags before unfreezing refcount. 3319 * 3320 * After successful get_page_unless_zero() might follow flags change, 3321 * for example lock_page() which set PG_waiters. 3322 * 3323 * Note that for mapped sub-pages of an anonymous THP, 3324 * PG_anon_exclusive has been cleared in unmap_folio() and is stored in 3325 * the migration entry instead from where remap_page() will restore it. 3326 * We can still have PG_anon_exclusive set on effectively unmapped and 3327 * unreferenced sub-pages of an anonymous THP: we can simply drop 3328 * PG_anon_exclusive (-> PG_mappedtodisk) for these here. 3329 */ 3330 new_folio->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP; 3331 new_folio->flags.f |= (folio->flags.f & 3332 ((1L << PG_referenced) | 3333 (1L << PG_swapbacked) | 3334 (1L << PG_swapcache) | 3335 (1L << PG_mlocked) | 3336 (1L << PG_uptodate) | 3337 (1L << PG_active) | 3338 (1L << PG_workingset) | 3339 (1L << PG_locked) | 3340 (1L << PG_unevictable) | 3341 #ifdef CONFIG_ARCH_USES_PG_ARCH_2 3342 (1L << PG_arch_2) | 3343 #endif 3344 #ifdef CONFIG_ARCH_USES_PG_ARCH_3 3345 (1L << PG_arch_3) | 3346 #endif 3347 (1L << PG_dirty) | 3348 LRU_GEN_MASK | LRU_REFS_MASK)); 3349 3350 if (handle_hwpoison && 3351 page_range_has_hwpoisoned(new_head, new_nr_pages)) 3352 folio_set_has_hwpoisoned(new_folio); 3353 3354 new_folio->mapping = folio->mapping; 3355 new_folio->index = folio->index + i; 3356 3357 /* 3358 * page->private should not be set in tail pages. Fix up and warn once 3359 * if private is unexpectedly set. 3360 */ 3361 if (unlikely(new_folio->private)) { 3362 VM_WARN_ON_ONCE_PAGE(true, new_head); 3363 new_folio->private = NULL; 3364 } 3365 3366 if (folio_test_swapcache(folio)) 3367 new_folio->swap.val = folio->swap.val + i; 3368 3369 /* Page flags must be visible before we make the page non-compound. */ 3370 smp_wmb(); 3371 3372 /* 3373 * Clear PageTail before unfreezing page refcount. 3374 * 3375 * After successful get_page_unless_zero() might follow put_page() 3376 * which needs correct compound_head(). 3377 */ 3378 clear_compound_head(new_head); 3379 if (new_order) { 3380 prep_compound_page(new_head, new_order); 3381 folio_set_large_rmappable(new_folio); 3382 } 3383 3384 if (folio_test_young(folio)) 3385 folio_set_young(new_folio); 3386 if (folio_test_idle(folio)) 3387 folio_set_idle(new_folio); 3388 #ifdef CONFIG_MEMCG 3389 new_folio->memcg_data = folio->memcg_data; 3390 #endif 3391 3392 folio_xchg_last_cpupid(new_folio, folio_last_cpupid(folio)); 3393 } 3394 3395 if (new_order) 3396 folio_set_order(folio, new_order); 3397 else 3398 ClearPageCompound(&folio->page); 3399 } 3400 3401 /* 3402 * It splits an unmapped @folio to lower order smaller folios in two ways. 3403 * @folio: the to-be-split folio 3404 * @new_order: the smallest order of the after split folios (since buddy 3405 * allocator like split generates folios with orders from @folio's 3406 * order - 1 to new_order). 3407 * @split_at: in buddy allocator like split, the folio containing @split_at 3408 * will be split until its order becomes @new_order. 3409 * @xas: xa_state pointing to folio->mapping->i_pages and locked by caller 3410 * @mapping: @folio->mapping 3411 * @uniform_split: if the split is uniform or not (buddy allocator like split) 3412 * 3413 * 3414 * 1. uniform split: the given @folio into multiple @new_order small folios, 3415 * where all small folios have the same order. This is done when 3416 * uniform_split is true. 3417 * 2. buddy allocator like (non-uniform) split: the given @folio is split into 3418 * half and one of the half (containing the given page) is split into half 3419 * until the given @page's order becomes @new_order. This is done when 3420 * uniform_split is false. 3421 * 3422 * The high level flow for these two methods are: 3423 * 1. uniform split: a single __split_folio_to_order() is called to split the 3424 * @folio into @new_order, then we traverse all the resulting folios one by 3425 * one in PFN ascending order and perform stats, unfreeze, adding to list, 3426 * and file mapping index operations. 3427 * 2. non-uniform split: in general, folio_order - @new_order calls to 3428 * __split_folio_to_order() are made in a for loop to split the @folio 3429 * to one lower order at a time. The resulting small folios are processed 3430 * like what is done during the traversal in 1, except the one containing 3431 * @page, which is split in next for loop. 3432 * 3433 * After splitting, the caller's folio reference will be transferred to the 3434 * folio containing @page. The caller needs to unlock and/or free after-split 3435 * folios if necessary. 3436 * 3437 * For !uniform_split, when -ENOMEM is returned, the original folio might be 3438 * split. The caller needs to check the input folio. 3439 */ 3440 static int __split_unmapped_folio(struct folio *folio, int new_order, 3441 struct page *split_at, struct xa_state *xas, 3442 struct address_space *mapping, bool uniform_split) 3443 { 3444 int order = folio_order(folio); 3445 int start_order = uniform_split ? new_order : order - 1; 3446 bool stop_split = false; 3447 struct folio *next; 3448 int split_order; 3449 int ret = 0; 3450 3451 if (folio_test_anon(folio)) 3452 mod_mthp_stat(order, MTHP_STAT_NR_ANON, -1); 3453 3454 /* 3455 * split to new_order one order at a time. For uniform split, 3456 * folio is split to new_order directly. 3457 */ 3458 for (split_order = start_order; 3459 split_order >= new_order && !stop_split; 3460 split_order--) { 3461 struct folio *end_folio = folio_next(folio); 3462 int old_order = folio_order(folio); 3463 struct folio *new_folio; 3464 3465 /* order-1 anonymous folio is not supported */ 3466 if (folio_test_anon(folio) && split_order == 1) 3467 continue; 3468 if (uniform_split && split_order != new_order) 3469 continue; 3470 3471 if (mapping) { 3472 /* 3473 * uniform split has xas_split_alloc() called before 3474 * irq is disabled to allocate enough memory, whereas 3475 * non-uniform split can handle ENOMEM. 3476 */ 3477 if (uniform_split) 3478 xas_split(xas, folio, old_order); 3479 else { 3480 xas_set_order(xas, folio->index, split_order); 3481 xas_try_split(xas, folio, old_order); 3482 if (xas_error(xas)) { 3483 ret = xas_error(xas); 3484 stop_split = true; 3485 } 3486 } 3487 } 3488 3489 if (!stop_split) { 3490 folio_split_memcg_refs(folio, old_order, split_order); 3491 split_page_owner(&folio->page, old_order, split_order); 3492 pgalloc_tag_split(folio, old_order, split_order); 3493 3494 __split_folio_to_order(folio, old_order, split_order); 3495 } 3496 3497 /* 3498 * Iterate through after-split folios and update folio stats. 3499 * But in buddy allocator like split, the folio 3500 * containing the specified page is skipped until its order 3501 * is new_order, since the folio will be worked on in next 3502 * iteration. 3503 */ 3504 for (new_folio = folio; new_folio != end_folio; new_folio = next) { 3505 next = folio_next(new_folio); 3506 /* 3507 * for buddy allocator like split, new_folio containing 3508 * @split_at page could be split again, thus do not 3509 * change stats yet. Wait until new_folio's order is 3510 * @new_order or stop_split is set to true by the above 3511 * xas_split() failure. 3512 */ 3513 if (new_folio == page_folio(split_at)) { 3514 folio = new_folio; 3515 if (split_order != new_order && !stop_split) 3516 continue; 3517 } 3518 if (folio_test_anon(new_folio)) 3519 mod_mthp_stat(folio_order(new_folio), 3520 MTHP_STAT_NR_ANON, 1); 3521 } 3522 } 3523 3524 return ret; 3525 } 3526 3527 bool non_uniform_split_supported(struct folio *folio, unsigned int new_order, 3528 bool warns) 3529 { 3530 if (folio_test_anon(folio)) { 3531 /* order-1 is not supported for anonymous THP. */ 3532 VM_WARN_ONCE(warns && new_order == 1, 3533 "Cannot split to order-1 folio"); 3534 if (new_order == 1) 3535 return false; 3536 } else if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && 3537 !mapping_large_folio_support(folio->mapping)) { 3538 /* 3539 * No split if the file system does not support large folio. 3540 * Note that we might still have THPs in such mappings due to 3541 * CONFIG_READ_ONLY_THP_FOR_FS. But in that case, the mapping 3542 * does not actually support large folios properly. 3543 */ 3544 VM_WARN_ONCE(warns, 3545 "Cannot split file folio to non-0 order"); 3546 return false; 3547 } 3548 3549 /* Only swapping a whole PMD-mapped folio is supported */ 3550 if (folio_test_swapcache(folio)) { 3551 VM_WARN_ONCE(warns, 3552 "Cannot split swapcache folio to non-0 order"); 3553 return false; 3554 } 3555 3556 return true; 3557 } 3558 3559 /* See comments in non_uniform_split_supported() */ 3560 bool uniform_split_supported(struct folio *folio, unsigned int new_order, 3561 bool warns) 3562 { 3563 if (folio_test_anon(folio)) { 3564 VM_WARN_ONCE(warns && new_order == 1, 3565 "Cannot split to order-1 folio"); 3566 if (new_order == 1) 3567 return false; 3568 } else if (new_order) { 3569 if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && 3570 !mapping_large_folio_support(folio->mapping)) { 3571 VM_WARN_ONCE(warns, 3572 "Cannot split file folio to non-0 order"); 3573 return false; 3574 } 3575 } 3576 3577 if (new_order && folio_test_swapcache(folio)) { 3578 VM_WARN_ONCE(warns, 3579 "Cannot split swapcache folio to non-0 order"); 3580 return false; 3581 } 3582 3583 return true; 3584 } 3585 3586 /* 3587 * __folio_split: split a folio at @split_at to a @new_order folio 3588 * @folio: folio to split 3589 * @new_order: the order of the new folio 3590 * @split_at: a page within the new folio 3591 * @lock_at: a page within @folio to be left locked to caller 3592 * @list: after-split folios will be put on it if non NULL 3593 * @uniform_split: perform uniform split or not (non-uniform split) 3594 * 3595 * It calls __split_unmapped_folio() to perform uniform and non-uniform split. 3596 * It is in charge of checking whether the split is supported or not and 3597 * preparing @folio for __split_unmapped_folio(). 3598 * 3599 * After splitting, the after-split folio containing @lock_at remains locked 3600 * and others are unlocked: 3601 * 1. for uniform split, @lock_at points to one of @folio's subpages; 3602 * 2. for buddy allocator like (non-uniform) split, @lock_at points to @folio. 3603 * 3604 * return: 0: successful, <0 failed (if -ENOMEM is returned, @folio might be 3605 * split but not to @new_order, the caller needs to check) 3606 */ 3607 static int __folio_split(struct folio *folio, unsigned int new_order, 3608 struct page *split_at, struct page *lock_at, 3609 struct list_head *list, bool uniform_split) 3610 { 3611 struct deferred_split *ds_queue = get_deferred_split_queue(folio); 3612 XA_STATE(xas, &folio->mapping->i_pages, folio->index); 3613 struct folio *end_folio = folio_next(folio); 3614 bool is_anon = folio_test_anon(folio); 3615 struct address_space *mapping = NULL; 3616 struct anon_vma *anon_vma = NULL; 3617 int order = folio_order(folio); 3618 struct folio *new_folio, *next; 3619 int nr_shmem_dropped = 0; 3620 int remap_flags = 0; 3621 int extra_pins, ret; 3622 pgoff_t end; 3623 bool is_hzp; 3624 3625 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); 3626 VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio); 3627 3628 if (folio != page_folio(split_at) || folio != page_folio(lock_at)) 3629 return -EINVAL; 3630 3631 /* 3632 * Folios that just got truncated cannot get split. Signal to the 3633 * caller that there was a race. 3634 * 3635 * TODO: this will also currently refuse shmem folios that are in the 3636 * swapcache. 3637 */ 3638 if (!is_anon && !folio->mapping) 3639 return -EBUSY; 3640 3641 if (new_order >= folio_order(folio)) 3642 return -EINVAL; 3643 3644 if (uniform_split && !uniform_split_supported(folio, new_order, true)) 3645 return -EINVAL; 3646 3647 if (!uniform_split && 3648 !non_uniform_split_supported(folio, new_order, true)) 3649 return -EINVAL; 3650 3651 is_hzp = is_huge_zero_folio(folio); 3652 if (is_hzp) { 3653 pr_warn_ratelimited("Called split_huge_page for huge zero page\n"); 3654 return -EBUSY; 3655 } 3656 3657 if (folio_test_writeback(folio)) 3658 return -EBUSY; 3659 3660 if (is_anon) { 3661 /* 3662 * The caller does not necessarily hold an mmap_lock that would 3663 * prevent the anon_vma disappearing so we first we take a 3664 * reference to it and then lock the anon_vma for write. This 3665 * is similar to folio_lock_anon_vma_read except the write lock 3666 * is taken to serialise against parallel split or collapse 3667 * operations. 3668 */ 3669 anon_vma = folio_get_anon_vma(folio); 3670 if (!anon_vma) { 3671 ret = -EBUSY; 3672 goto out; 3673 } 3674 mapping = NULL; 3675 anon_vma_lock_write(anon_vma); 3676 } else { 3677 unsigned int min_order; 3678 gfp_t gfp; 3679 3680 mapping = folio->mapping; 3681 min_order = mapping_min_folio_order(folio->mapping); 3682 if (new_order < min_order) { 3683 ret = -EINVAL; 3684 goto out; 3685 } 3686 3687 gfp = current_gfp_context(mapping_gfp_mask(mapping) & 3688 GFP_RECLAIM_MASK); 3689 3690 if (!filemap_release_folio(folio, gfp)) { 3691 ret = -EBUSY; 3692 goto out; 3693 } 3694 3695 if (uniform_split) { 3696 xas_set_order(&xas, folio->index, new_order); 3697 xas_split_alloc(&xas, folio, folio_order(folio), gfp); 3698 if (xas_error(&xas)) { 3699 ret = xas_error(&xas); 3700 goto out; 3701 } 3702 } 3703 3704 anon_vma = NULL; 3705 i_mmap_lock_read(mapping); 3706 3707 /* 3708 *__split_unmapped_folio() may need to trim off pages beyond 3709 * EOF: but on 32-bit, i_size_read() takes an irq-unsafe 3710 * seqlock, which cannot be nested inside the page tree lock. 3711 * So note end now: i_size itself may be changed at any moment, 3712 * but folio lock is good enough to serialize the trimming. 3713 */ 3714 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); 3715 if (shmem_mapping(mapping)) 3716 end = shmem_fallocend(mapping->host, end); 3717 } 3718 3719 /* 3720 * Racy check if we can split the page, before unmap_folio() will 3721 * split PMDs 3722 */ 3723 if (!can_split_folio(folio, 1, &extra_pins)) { 3724 ret = -EAGAIN; 3725 goto out_unlock; 3726 } 3727 3728 unmap_folio(folio); 3729 3730 /* block interrupt reentry in xa_lock and spinlock */ 3731 local_irq_disable(); 3732 if (mapping) { 3733 /* 3734 * Check if the folio is present in page cache. 3735 * We assume all tail are present too, if folio is there. 3736 */ 3737 xas_lock(&xas); 3738 xas_reset(&xas); 3739 if (xas_load(&xas) != folio) { 3740 ret = -EAGAIN; 3741 goto fail; 3742 } 3743 } 3744 3745 /* Prevent deferred_split_scan() touching ->_refcount */ 3746 spin_lock(&ds_queue->split_queue_lock); 3747 if (folio_ref_freeze(folio, 1 + extra_pins)) { 3748 struct swap_cluster_info *ci = NULL; 3749 struct lruvec *lruvec; 3750 int expected_refs; 3751 3752 if (folio_order(folio) > 1 && 3753 !list_empty(&folio->_deferred_list)) { 3754 ds_queue->split_queue_len--; 3755 if (folio_test_partially_mapped(folio)) { 3756 folio_clear_partially_mapped(folio); 3757 mod_mthp_stat(folio_order(folio), 3758 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 3759 } 3760 /* 3761 * Reinitialize page_deferred_list after removing the 3762 * page from the split_queue, otherwise a subsequent 3763 * split will see list corruption when checking the 3764 * page_deferred_list. 3765 */ 3766 list_del_init(&folio->_deferred_list); 3767 } 3768 spin_unlock(&ds_queue->split_queue_lock); 3769 if (mapping) { 3770 int nr = folio_nr_pages(folio); 3771 3772 if (folio_test_pmd_mappable(folio) && 3773 new_order < HPAGE_PMD_ORDER) { 3774 if (folio_test_swapbacked(folio)) { 3775 __lruvec_stat_mod_folio(folio, 3776 NR_SHMEM_THPS, -nr); 3777 } else { 3778 __lruvec_stat_mod_folio(folio, 3779 NR_FILE_THPS, -nr); 3780 filemap_nr_thps_dec(mapping); 3781 } 3782 } 3783 } 3784 3785 if (folio_test_swapcache(folio)) { 3786 if (mapping) { 3787 VM_WARN_ON_ONCE_FOLIO(mapping, folio); 3788 ret = -EINVAL; 3789 goto fail; 3790 } 3791 3792 ci = swap_cluster_get_and_lock(folio); 3793 } 3794 3795 /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ 3796 lruvec = folio_lruvec_lock(folio); 3797 3798 ret = __split_unmapped_folio(folio, new_order, split_at, &xas, 3799 mapping, uniform_split); 3800 3801 /* 3802 * Unfreeze after-split folios and put them back to the right 3803 * list. @folio should be kept frozon until page cache 3804 * entries are updated with all the other after-split folios 3805 * to prevent others seeing stale page cache entries. 3806 * As a result, new_folio starts from the next folio of 3807 * @folio. 3808 */ 3809 for (new_folio = folio_next(folio); new_folio != end_folio; 3810 new_folio = next) { 3811 unsigned long nr_pages = folio_nr_pages(new_folio); 3812 3813 next = folio_next(new_folio); 3814 3815 expected_refs = folio_expected_ref_count(new_folio) + 1; 3816 folio_ref_unfreeze(new_folio, expected_refs); 3817 3818 lru_add_split_folio(folio, new_folio, lruvec, list); 3819 3820 /* 3821 * Anonymous folio with swap cache. 3822 * NOTE: shmem in swap cache is not supported yet. 3823 */ 3824 if (ci) { 3825 __swap_cache_replace_folio(ci, folio, new_folio); 3826 continue; 3827 } 3828 3829 /* Anonymous folio without swap cache */ 3830 if (!mapping) 3831 continue; 3832 3833 /* Add the new folio to the page cache. */ 3834 if (new_folio->index < end) { 3835 __xa_store(&mapping->i_pages, new_folio->index, 3836 new_folio, 0); 3837 continue; 3838 } 3839 3840 /* Drop folio beyond EOF: ->index >= end */ 3841 if (shmem_mapping(mapping)) 3842 nr_shmem_dropped += nr_pages; 3843 else if (folio_test_clear_dirty(new_folio)) 3844 folio_account_cleaned( 3845 new_folio, inode_to_wb(mapping->host)); 3846 __filemap_remove_folio(new_folio, NULL); 3847 folio_put_refs(new_folio, nr_pages); 3848 } 3849 /* 3850 * Unfreeze @folio only after all page cache entries, which 3851 * used to point to it, have been updated with new folios. 3852 * Otherwise, a parallel folio_try_get() can grab @folio 3853 * and its caller can see stale page cache entries. 3854 */ 3855 expected_refs = folio_expected_ref_count(folio) + 1; 3856 folio_ref_unfreeze(folio, expected_refs); 3857 3858 unlock_page_lruvec(lruvec); 3859 3860 if (ci) 3861 swap_cluster_unlock(ci); 3862 } else { 3863 spin_unlock(&ds_queue->split_queue_lock); 3864 ret = -EAGAIN; 3865 } 3866 fail: 3867 if (mapping) 3868 xas_unlock(&xas); 3869 3870 local_irq_enable(); 3871 3872 if (nr_shmem_dropped) 3873 shmem_uncharge(mapping->host, nr_shmem_dropped); 3874 3875 if (!ret && is_anon) 3876 remap_flags = RMP_USE_SHARED_ZEROPAGE; 3877 remap_page(folio, 1 << order, remap_flags); 3878 3879 /* 3880 * Unlock all after-split folios except the one containing 3881 * @lock_at page. If @folio is not split, it will be kept locked. 3882 */ 3883 for (new_folio = folio; new_folio != end_folio; new_folio = next) { 3884 next = folio_next(new_folio); 3885 if (new_folio == page_folio(lock_at)) 3886 continue; 3887 3888 folio_unlock(new_folio); 3889 /* 3890 * Subpages may be freed if there wasn't any mapping 3891 * like if add_to_swap() is running on a lru page that 3892 * had its mapping zapped. And freeing these pages 3893 * requires taking the lru_lock so we do the put_page 3894 * of the tail pages after the split is complete. 3895 */ 3896 free_folio_and_swap_cache(new_folio); 3897 } 3898 3899 out_unlock: 3900 if (anon_vma) { 3901 anon_vma_unlock_write(anon_vma); 3902 put_anon_vma(anon_vma); 3903 } 3904 if (mapping) 3905 i_mmap_unlock_read(mapping); 3906 out: 3907 xas_destroy(&xas); 3908 if (order == HPAGE_PMD_ORDER) 3909 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); 3910 count_mthp_stat(order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED); 3911 return ret; 3912 } 3913 3914 /* 3915 * This function splits a large folio into smaller folios of order @new_order. 3916 * @page can point to any page of the large folio to split. The split operation 3917 * does not change the position of @page. 3918 * 3919 * Prerequisites: 3920 * 3921 * 1) The caller must hold a reference on the @page's owning folio, also known 3922 * as the large folio. 3923 * 3924 * 2) The large folio must be locked. 3925 * 3926 * 3) The folio must not be pinned. Any unexpected folio references, including 3927 * GUP pins, will result in the folio not getting split; instead, the caller 3928 * will receive an -EAGAIN. 3929 * 3930 * 4) @new_order > 1, usually. Splitting to order-1 anonymous folios is not 3931 * supported for non-file-backed folios, because folio->_deferred_list, which 3932 * is used by partially mapped folios, is stored in subpage 2, but an order-1 3933 * folio only has subpages 0 and 1. File-backed order-1 folios are supported, 3934 * since they do not use _deferred_list. 3935 * 3936 * After splitting, the caller's folio reference will be transferred to @page, 3937 * resulting in a raised refcount of @page after this call. The other pages may 3938 * be freed if they are not mapped. 3939 * 3940 * If @list is null, tail pages will be added to LRU list, otherwise, to @list. 3941 * 3942 * Pages in @new_order will inherit the mapping, flags, and so on from the 3943 * huge page. 3944 * 3945 * Returns 0 if the huge page was split successfully. 3946 * 3947 * Returns -EAGAIN if the folio has unexpected reference (e.g., GUP) or if 3948 * the folio was concurrently removed from the page cache. 3949 * 3950 * Returns -EBUSY when trying to split the huge zeropage, if the folio is 3951 * under writeback, if fs-specific folio metadata cannot currently be 3952 * released, or if some unexpected race happened (e.g., anon VMA disappeared, 3953 * truncation). 3954 * 3955 * Callers should ensure that the order respects the address space mapping 3956 * min-order if one is set for non-anonymous folios. 3957 * 3958 * Returns -EINVAL when trying to split to an order that is incompatible 3959 * with the folio. Splitting to order 0 is compatible with all folios. 3960 */ 3961 int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, 3962 unsigned int new_order) 3963 { 3964 struct folio *folio = page_folio(page); 3965 3966 return __folio_split(folio, new_order, &folio->page, page, list, true); 3967 } 3968 3969 /* 3970 * folio_split: split a folio at @split_at to a @new_order folio 3971 * @folio: folio to split 3972 * @new_order: the order of the new folio 3973 * @split_at: a page within the new folio 3974 * 3975 * return: 0: successful, <0 failed (if -ENOMEM is returned, @folio might be 3976 * split but not to @new_order, the caller needs to check) 3977 * 3978 * It has the same prerequisites and returns as 3979 * split_huge_page_to_list_to_order(). 3980 * 3981 * Split a folio at @split_at to a new_order folio, leave the 3982 * remaining subpages of the original folio as large as possible. For example, 3983 * in the case of splitting an order-9 folio at its third order-3 subpages to 3984 * an order-3 folio, there are 2^(9-3)=64 order-3 subpages in the order-9 folio. 3985 * After the split, there will be a group of folios with different orders and 3986 * the new folio containing @split_at is marked in bracket: 3987 * [order-4, {order-3}, order-3, order-5, order-6, order-7, order-8]. 3988 * 3989 * After split, folio is left locked for caller. 3990 */ 3991 int folio_split(struct folio *folio, unsigned int new_order, 3992 struct page *split_at, struct list_head *list) 3993 { 3994 return __folio_split(folio, new_order, split_at, &folio->page, list, 3995 false); 3996 } 3997 3998 int min_order_for_split(struct folio *folio) 3999 { 4000 if (folio_test_anon(folio)) 4001 return 0; 4002 4003 if (!folio->mapping) { 4004 if (folio_test_pmd_mappable(folio)) 4005 count_vm_event(THP_SPLIT_PAGE_FAILED); 4006 return -EBUSY; 4007 } 4008 4009 return mapping_min_folio_order(folio->mapping); 4010 } 4011 4012 int split_folio_to_list(struct folio *folio, struct list_head *list) 4013 { 4014 return split_huge_page_to_list_to_order(&folio->page, list, 0); 4015 } 4016 4017 /* 4018 * __folio_unqueue_deferred_split() is not to be called directly: 4019 * the folio_unqueue_deferred_split() inline wrapper in mm/internal.h 4020 * limits its calls to those folios which may have a _deferred_list for 4021 * queueing THP splits, and that list is (racily observed to be) non-empty. 4022 * 4023 * It is unsafe to call folio_unqueue_deferred_split() until folio refcount is 4024 * zero: because even when split_queue_lock is held, a non-empty _deferred_list 4025 * might be in use on deferred_split_scan()'s unlocked on-stack list. 4026 * 4027 * If memory cgroups are enabled, split_queue_lock is in the mem_cgroup: it is 4028 * therefore important to unqueue deferred split before changing folio memcg. 4029 */ 4030 bool __folio_unqueue_deferred_split(struct folio *folio) 4031 { 4032 struct deferred_split *ds_queue; 4033 unsigned long flags; 4034 bool unqueued = false; 4035 4036 WARN_ON_ONCE(folio_ref_count(folio)); 4037 WARN_ON_ONCE(!mem_cgroup_disabled() && !folio_memcg(folio)); 4038 4039 ds_queue = get_deferred_split_queue(folio); 4040 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 4041 if (!list_empty(&folio->_deferred_list)) { 4042 ds_queue->split_queue_len--; 4043 if (folio_test_partially_mapped(folio)) { 4044 folio_clear_partially_mapped(folio); 4045 mod_mthp_stat(folio_order(folio), 4046 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 4047 } 4048 list_del_init(&folio->_deferred_list); 4049 unqueued = true; 4050 } 4051 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 4052 4053 return unqueued; /* useful for debug warnings */ 4054 } 4055 4056 /* partially_mapped=false won't clear PG_partially_mapped folio flag */ 4057 void deferred_split_folio(struct folio *folio, bool partially_mapped) 4058 { 4059 struct deferred_split *ds_queue = get_deferred_split_queue(folio); 4060 #ifdef CONFIG_MEMCG 4061 struct mem_cgroup *memcg = folio_memcg(folio); 4062 #endif 4063 unsigned long flags; 4064 4065 /* 4066 * Order 1 folios have no space for a deferred list, but we also 4067 * won't waste much memory by not adding them to the deferred list. 4068 */ 4069 if (folio_order(folio) <= 1) 4070 return; 4071 4072 if (!partially_mapped && !split_underused_thp) 4073 return; 4074 4075 /* 4076 * Exclude swapcache: originally to avoid a corrupt deferred split 4077 * queue. Nowadays that is fully prevented by memcg1_swapout(); 4078 * but if page reclaim is already handling the same folio, it is 4079 * unnecessary to handle it again in the shrinker, so excluding 4080 * swapcache here may still be a useful optimization. 4081 */ 4082 if (folio_test_swapcache(folio)) 4083 return; 4084 4085 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 4086 if (partially_mapped) { 4087 if (!folio_test_partially_mapped(folio)) { 4088 folio_set_partially_mapped(folio); 4089 if (folio_test_pmd_mappable(folio)) 4090 count_vm_event(THP_DEFERRED_SPLIT_PAGE); 4091 count_mthp_stat(folio_order(folio), MTHP_STAT_SPLIT_DEFERRED); 4092 mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, 1); 4093 4094 } 4095 } else { 4096 /* partially mapped folios cannot become non-partially mapped */ 4097 VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio); 4098 } 4099 if (list_empty(&folio->_deferred_list)) { 4100 list_add_tail(&folio->_deferred_list, &ds_queue->split_queue); 4101 ds_queue->split_queue_len++; 4102 #ifdef CONFIG_MEMCG 4103 if (memcg) 4104 set_shrinker_bit(memcg, folio_nid(folio), 4105 deferred_split_shrinker->id); 4106 #endif 4107 } 4108 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 4109 } 4110 4111 static unsigned long deferred_split_count(struct shrinker *shrink, 4112 struct shrink_control *sc) 4113 { 4114 struct pglist_data *pgdata = NODE_DATA(sc->nid); 4115 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 4116 4117 #ifdef CONFIG_MEMCG 4118 if (sc->memcg) 4119 ds_queue = &sc->memcg->deferred_split_queue; 4120 #endif 4121 return READ_ONCE(ds_queue->split_queue_len); 4122 } 4123 4124 static bool thp_underused(struct folio *folio) 4125 { 4126 int num_zero_pages = 0, num_filled_pages = 0; 4127 int i; 4128 4129 if (khugepaged_max_ptes_none == HPAGE_PMD_NR - 1) 4130 return false; 4131 4132 if (folio_contain_hwpoisoned_page(folio)) 4133 return false; 4134 4135 for (i = 0; i < folio_nr_pages(folio); i++) { 4136 if (pages_identical(folio_page(folio, i), ZERO_PAGE(0))) { 4137 if (++num_zero_pages > khugepaged_max_ptes_none) 4138 return true; 4139 } else { 4140 /* 4141 * Another path for early exit once the number 4142 * of non-zero filled pages exceeds threshold. 4143 */ 4144 if (++num_filled_pages >= HPAGE_PMD_NR - khugepaged_max_ptes_none) 4145 return false; 4146 } 4147 } 4148 return false; 4149 } 4150 4151 static unsigned long deferred_split_scan(struct shrinker *shrink, 4152 struct shrink_control *sc) 4153 { 4154 struct pglist_data *pgdata = NODE_DATA(sc->nid); 4155 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 4156 unsigned long flags; 4157 LIST_HEAD(list); 4158 struct folio *folio, *next, *prev = NULL; 4159 int split = 0, removed = 0; 4160 4161 #ifdef CONFIG_MEMCG 4162 if (sc->memcg) 4163 ds_queue = &sc->memcg->deferred_split_queue; 4164 #endif 4165 4166 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 4167 /* Take pin on all head pages to avoid freeing them under us */ 4168 list_for_each_entry_safe(folio, next, &ds_queue->split_queue, 4169 _deferred_list) { 4170 if (folio_try_get(folio)) { 4171 list_move(&folio->_deferred_list, &list); 4172 } else { 4173 /* We lost race with folio_put() */ 4174 if (folio_test_partially_mapped(folio)) { 4175 folio_clear_partially_mapped(folio); 4176 mod_mthp_stat(folio_order(folio), 4177 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 4178 } 4179 list_del_init(&folio->_deferred_list); 4180 ds_queue->split_queue_len--; 4181 } 4182 if (!--sc->nr_to_scan) 4183 break; 4184 } 4185 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 4186 4187 list_for_each_entry_safe(folio, next, &list, _deferred_list) { 4188 bool did_split = false; 4189 bool underused = false; 4190 4191 if (!folio_test_partially_mapped(folio)) { 4192 /* 4193 * See try_to_map_unused_to_zeropage(): we cannot 4194 * optimize zero-filled pages after splitting an 4195 * mlocked folio. 4196 */ 4197 if (folio_test_mlocked(folio)) 4198 goto next; 4199 underused = thp_underused(folio); 4200 if (!underused) 4201 goto next; 4202 } 4203 if (!folio_trylock(folio)) 4204 goto next; 4205 if (!split_folio(folio)) { 4206 did_split = true; 4207 if (underused) 4208 count_vm_event(THP_UNDERUSED_SPLIT_PAGE); 4209 split++; 4210 } 4211 folio_unlock(folio); 4212 next: 4213 /* 4214 * split_folio() removes folio from list on success. 4215 * Only add back to the queue if folio is partially mapped. 4216 * If thp_underused returns false, or if split_folio fails 4217 * in the case it was underused, then consider it used and 4218 * don't add it back to split_queue. 4219 */ 4220 if (did_split) { 4221 ; /* folio already removed from list */ 4222 } else if (!folio_test_partially_mapped(folio)) { 4223 list_del_init(&folio->_deferred_list); 4224 removed++; 4225 } else { 4226 /* 4227 * That unlocked list_del_init() above would be unsafe, 4228 * unless its folio is separated from any earlier folios 4229 * left on the list (which may be concurrently unqueued) 4230 * by one safe folio with refcount still raised. 4231 */ 4232 swap(folio, prev); 4233 } 4234 if (folio) 4235 folio_put(folio); 4236 } 4237 4238 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 4239 list_splice_tail(&list, &ds_queue->split_queue); 4240 ds_queue->split_queue_len -= removed; 4241 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 4242 4243 if (prev) 4244 folio_put(prev); 4245 4246 /* 4247 * Stop shrinker if we didn't split any page, but the queue is empty. 4248 * This can happen if pages were freed under us. 4249 */ 4250 if (!split && list_empty(&ds_queue->split_queue)) 4251 return SHRINK_STOP; 4252 return split; 4253 } 4254 4255 #ifdef CONFIG_DEBUG_FS 4256 static void split_huge_pages_all(void) 4257 { 4258 struct zone *zone; 4259 struct page *page; 4260 struct folio *folio; 4261 unsigned long pfn, max_zone_pfn; 4262 unsigned long total = 0, split = 0; 4263 4264 pr_debug("Split all THPs\n"); 4265 for_each_zone(zone) { 4266 if (!managed_zone(zone)) 4267 continue; 4268 max_zone_pfn = zone_end_pfn(zone); 4269 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 4270 int nr_pages; 4271 4272 page = pfn_to_online_page(pfn); 4273 if (!page || PageTail(page)) 4274 continue; 4275 folio = page_folio(page); 4276 if (!folio_try_get(folio)) 4277 continue; 4278 4279 if (unlikely(page_folio(page) != folio)) 4280 goto next; 4281 4282 if (zone != folio_zone(folio)) 4283 goto next; 4284 4285 if (!folio_test_large(folio) 4286 || folio_test_hugetlb(folio) 4287 || !folio_test_lru(folio)) 4288 goto next; 4289 4290 total++; 4291 folio_lock(folio); 4292 nr_pages = folio_nr_pages(folio); 4293 if (!split_folio(folio)) 4294 split++; 4295 pfn += nr_pages - 1; 4296 folio_unlock(folio); 4297 next: 4298 folio_put(folio); 4299 cond_resched(); 4300 } 4301 } 4302 4303 pr_debug("%lu of %lu THP split\n", split, total); 4304 } 4305 4306 static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma) 4307 { 4308 return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) || 4309 is_vm_hugetlb_page(vma); 4310 } 4311 4312 static int split_huge_pages_pid(int pid, unsigned long vaddr_start, 4313 unsigned long vaddr_end, unsigned int new_order, 4314 long in_folio_offset) 4315 { 4316 int ret = 0; 4317 struct task_struct *task; 4318 struct mm_struct *mm; 4319 unsigned long total = 0, split = 0; 4320 unsigned long addr; 4321 4322 vaddr_start &= PAGE_MASK; 4323 vaddr_end &= PAGE_MASK; 4324 4325 task = find_get_task_by_vpid(pid); 4326 if (!task) { 4327 ret = -ESRCH; 4328 goto out; 4329 } 4330 4331 /* Find the mm_struct */ 4332 mm = get_task_mm(task); 4333 put_task_struct(task); 4334 4335 if (!mm) { 4336 ret = -EINVAL; 4337 goto out; 4338 } 4339 4340 pr_debug("Split huge pages in pid: %d, vaddr: [0x%lx - 0x%lx], new_order: %u, in_folio_offset: %ld\n", 4341 pid, vaddr_start, vaddr_end, new_order, in_folio_offset); 4342 4343 mmap_read_lock(mm); 4344 /* 4345 * always increase addr by PAGE_SIZE, since we could have a PTE page 4346 * table filled with PTE-mapped THPs, each of which is distinct. 4347 */ 4348 for (addr = vaddr_start; addr < vaddr_end; addr += PAGE_SIZE) { 4349 struct vm_area_struct *vma = vma_lookup(mm, addr); 4350 struct folio_walk fw; 4351 struct folio *folio; 4352 struct address_space *mapping; 4353 unsigned int target_order = new_order; 4354 4355 if (!vma) 4356 break; 4357 4358 /* skip special VMA and hugetlb VMA */ 4359 if (vma_not_suitable_for_thp_split(vma)) { 4360 addr = vma->vm_end; 4361 continue; 4362 } 4363 4364 folio = folio_walk_start(&fw, vma, addr, 0); 4365 if (!folio) 4366 continue; 4367 4368 if (!is_transparent_hugepage(folio)) 4369 goto next; 4370 4371 if (!folio_test_anon(folio)) { 4372 mapping = folio->mapping; 4373 target_order = max(new_order, 4374 mapping_min_folio_order(mapping)); 4375 } 4376 4377 if (target_order >= folio_order(folio)) 4378 goto next; 4379 4380 total++; 4381 /* 4382 * For folios with private, split_huge_page_to_list_to_order() 4383 * will try to drop it before split and then check if the folio 4384 * can be split or not. So skip the check here. 4385 */ 4386 if (!folio_test_private(folio) && 4387 !can_split_folio(folio, 0, NULL)) 4388 goto next; 4389 4390 if (!folio_trylock(folio)) 4391 goto next; 4392 folio_get(folio); 4393 folio_walk_end(&fw, vma); 4394 4395 if (!folio_test_anon(folio) && folio->mapping != mapping) 4396 goto unlock; 4397 4398 if (in_folio_offset < 0 || 4399 in_folio_offset >= folio_nr_pages(folio)) { 4400 if (!split_folio_to_order(folio, target_order)) 4401 split++; 4402 } else { 4403 struct page *split_at = folio_page(folio, 4404 in_folio_offset); 4405 if (!folio_split(folio, target_order, split_at, NULL)) 4406 split++; 4407 } 4408 4409 unlock: 4410 4411 folio_unlock(folio); 4412 folio_put(folio); 4413 4414 cond_resched(); 4415 continue; 4416 next: 4417 folio_walk_end(&fw, vma); 4418 cond_resched(); 4419 } 4420 mmap_read_unlock(mm); 4421 mmput(mm); 4422 4423 pr_debug("%lu of %lu THP split\n", split, total); 4424 4425 out: 4426 return ret; 4427 } 4428 4429 static int split_huge_pages_in_file(const char *file_path, pgoff_t off_start, 4430 pgoff_t off_end, unsigned int new_order, 4431 long in_folio_offset) 4432 { 4433 struct filename *file; 4434 struct file *candidate; 4435 struct address_space *mapping; 4436 int ret = -EINVAL; 4437 pgoff_t index; 4438 int nr_pages = 1; 4439 unsigned long total = 0, split = 0; 4440 unsigned int min_order; 4441 unsigned int target_order; 4442 4443 file = getname_kernel(file_path); 4444 if (IS_ERR(file)) 4445 return ret; 4446 4447 candidate = file_open_name(file, O_RDONLY, 0); 4448 if (IS_ERR(candidate)) 4449 goto out; 4450 4451 pr_debug("split file-backed THPs in file: %s, page offset: [0x%lx - 0x%lx], new_order: %u, in_folio_offset: %ld\n", 4452 file_path, off_start, off_end, new_order, in_folio_offset); 4453 4454 mapping = candidate->f_mapping; 4455 min_order = mapping_min_folio_order(mapping); 4456 target_order = max(new_order, min_order); 4457 4458 for (index = off_start; index < off_end; index += nr_pages) { 4459 struct folio *folio = filemap_get_folio(mapping, index); 4460 4461 nr_pages = 1; 4462 if (IS_ERR(folio)) 4463 continue; 4464 4465 if (!folio_test_large(folio)) 4466 goto next; 4467 4468 total++; 4469 nr_pages = folio_nr_pages(folio); 4470 4471 if (target_order >= folio_order(folio)) 4472 goto next; 4473 4474 if (!folio_trylock(folio)) 4475 goto next; 4476 4477 if (folio->mapping != mapping) 4478 goto unlock; 4479 4480 if (in_folio_offset < 0 || in_folio_offset >= nr_pages) { 4481 if (!split_folio_to_order(folio, target_order)) 4482 split++; 4483 } else { 4484 struct page *split_at = folio_page(folio, 4485 in_folio_offset); 4486 if (!folio_split(folio, target_order, split_at, NULL)) 4487 split++; 4488 } 4489 4490 unlock: 4491 folio_unlock(folio); 4492 next: 4493 folio_put(folio); 4494 cond_resched(); 4495 } 4496 4497 filp_close(candidate, NULL); 4498 ret = 0; 4499 4500 pr_debug("%lu of %lu file-backed THP split\n", split, total); 4501 out: 4502 putname(file); 4503 return ret; 4504 } 4505 4506 #define MAX_INPUT_BUF_SZ 255 4507 4508 static ssize_t split_huge_pages_write(struct file *file, const char __user *buf, 4509 size_t count, loff_t *ppops) 4510 { 4511 static DEFINE_MUTEX(split_debug_mutex); 4512 ssize_t ret; 4513 /* 4514 * hold pid, start_vaddr, end_vaddr, new_order or 4515 * file_path, off_start, off_end, new_order 4516 */ 4517 char input_buf[MAX_INPUT_BUF_SZ]; 4518 int pid; 4519 unsigned long vaddr_start, vaddr_end; 4520 unsigned int new_order = 0; 4521 long in_folio_offset = -1; 4522 4523 ret = mutex_lock_interruptible(&split_debug_mutex); 4524 if (ret) 4525 return ret; 4526 4527 ret = -EFAULT; 4528 4529 memset(input_buf, 0, MAX_INPUT_BUF_SZ); 4530 if (copy_from_user(input_buf, buf, min_t(size_t, count, MAX_INPUT_BUF_SZ))) 4531 goto out; 4532 4533 input_buf[MAX_INPUT_BUF_SZ - 1] = '\0'; 4534 4535 if (input_buf[0] == '/') { 4536 char *tok; 4537 char *tok_buf = input_buf; 4538 char file_path[MAX_INPUT_BUF_SZ]; 4539 pgoff_t off_start = 0, off_end = 0; 4540 size_t input_len = strlen(input_buf); 4541 4542 tok = strsep(&tok_buf, ","); 4543 if (tok && tok_buf) { 4544 strscpy(file_path, tok); 4545 } else { 4546 ret = -EINVAL; 4547 goto out; 4548 } 4549 4550 ret = sscanf(tok_buf, "0x%lx,0x%lx,%d,%ld", &off_start, &off_end, 4551 &new_order, &in_folio_offset); 4552 if (ret != 2 && ret != 3 && ret != 4) { 4553 ret = -EINVAL; 4554 goto out; 4555 } 4556 ret = split_huge_pages_in_file(file_path, off_start, off_end, 4557 new_order, in_folio_offset); 4558 if (!ret) 4559 ret = input_len; 4560 4561 goto out; 4562 } 4563 4564 ret = sscanf(input_buf, "%d,0x%lx,0x%lx,%d,%ld", &pid, &vaddr_start, 4565 &vaddr_end, &new_order, &in_folio_offset); 4566 if (ret == 1 && pid == 1) { 4567 split_huge_pages_all(); 4568 ret = strlen(input_buf); 4569 goto out; 4570 } else if (ret != 3 && ret != 4 && ret != 5) { 4571 ret = -EINVAL; 4572 goto out; 4573 } 4574 4575 ret = split_huge_pages_pid(pid, vaddr_start, vaddr_end, new_order, 4576 in_folio_offset); 4577 if (!ret) 4578 ret = strlen(input_buf); 4579 out: 4580 mutex_unlock(&split_debug_mutex); 4581 return ret; 4582 4583 } 4584 4585 static const struct file_operations split_huge_pages_fops = { 4586 .owner = THIS_MODULE, 4587 .write = split_huge_pages_write, 4588 }; 4589 4590 static int __init split_huge_pages_debugfs(void) 4591 { 4592 debugfs_create_file("split_huge_pages", 0200, NULL, NULL, 4593 &split_huge_pages_fops); 4594 return 0; 4595 } 4596 late_initcall(split_huge_pages_debugfs); 4597 #endif 4598 4599 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 4600 int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 4601 struct page *page) 4602 { 4603 struct folio *folio = page_folio(page); 4604 struct vm_area_struct *vma = pvmw->vma; 4605 struct mm_struct *mm = vma->vm_mm; 4606 unsigned long address = pvmw->address; 4607 bool anon_exclusive; 4608 pmd_t pmdval; 4609 swp_entry_t entry; 4610 pmd_t pmdswp; 4611 4612 if (!(pvmw->pmd && !pvmw->pte)) 4613 return 0; 4614 4615 flush_cache_range(vma, address, address + HPAGE_PMD_SIZE); 4616 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); 4617 4618 /* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. */ 4619 anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(page); 4620 if (anon_exclusive && folio_try_share_anon_rmap_pmd(folio, page)) { 4621 set_pmd_at(mm, address, pvmw->pmd, pmdval); 4622 return -EBUSY; 4623 } 4624 4625 if (pmd_dirty(pmdval)) 4626 folio_mark_dirty(folio); 4627 if (pmd_write(pmdval)) 4628 entry = make_writable_migration_entry(page_to_pfn(page)); 4629 else if (anon_exclusive) 4630 entry = make_readable_exclusive_migration_entry(page_to_pfn(page)); 4631 else 4632 entry = make_readable_migration_entry(page_to_pfn(page)); 4633 if (pmd_young(pmdval)) 4634 entry = make_migration_entry_young(entry); 4635 if (pmd_dirty(pmdval)) 4636 entry = make_migration_entry_dirty(entry); 4637 pmdswp = swp_entry_to_pmd(entry); 4638 if (pmd_soft_dirty(pmdval)) 4639 pmdswp = pmd_swp_mksoft_dirty(pmdswp); 4640 if (pmd_uffd_wp(pmdval)) 4641 pmdswp = pmd_swp_mkuffd_wp(pmdswp); 4642 set_pmd_at(mm, address, pvmw->pmd, pmdswp); 4643 folio_remove_rmap_pmd(folio, page, vma); 4644 folio_put(folio); 4645 trace_set_migration_pmd(address, pmd_val(pmdswp)); 4646 4647 return 0; 4648 } 4649 4650 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) 4651 { 4652 struct folio *folio = page_folio(new); 4653 struct vm_area_struct *vma = pvmw->vma; 4654 struct mm_struct *mm = vma->vm_mm; 4655 unsigned long address = pvmw->address; 4656 unsigned long haddr = address & HPAGE_PMD_MASK; 4657 pmd_t pmde; 4658 swp_entry_t entry; 4659 4660 if (!(pvmw->pmd && !pvmw->pte)) 4661 return; 4662 4663 entry = pmd_to_swp_entry(*pvmw->pmd); 4664 folio_get(folio); 4665 pmde = folio_mk_pmd(folio, READ_ONCE(vma->vm_page_prot)); 4666 if (pmd_swp_soft_dirty(*pvmw->pmd)) 4667 pmde = pmd_mksoft_dirty(pmde); 4668 if (is_writable_migration_entry(entry)) 4669 pmde = pmd_mkwrite(pmde, vma); 4670 if (pmd_swp_uffd_wp(*pvmw->pmd)) 4671 pmde = pmd_mkuffd_wp(pmde); 4672 if (!is_migration_entry_young(entry)) 4673 pmde = pmd_mkold(pmde); 4674 /* NOTE: this may contain setting soft-dirty on some archs */ 4675 if (folio_test_dirty(folio) && is_migration_entry_dirty(entry)) 4676 pmde = pmd_mkdirty(pmde); 4677 4678 if (folio_test_anon(folio)) { 4679 rmap_t rmap_flags = RMAP_NONE; 4680 4681 if (!is_readable_migration_entry(entry)) 4682 rmap_flags |= RMAP_EXCLUSIVE; 4683 4684 folio_add_anon_rmap_pmd(folio, new, vma, haddr, rmap_flags); 4685 } else { 4686 folio_add_file_rmap_pmd(folio, new, vma); 4687 } 4688 VM_BUG_ON(pmd_write(pmde) && folio_test_anon(folio) && !PageAnonExclusive(new)); 4689 set_pmd_at(mm, haddr, pvmw->pmd, pmde); 4690 4691 /* No need to invalidate - it was non-present before */ 4692 update_mmu_cache_pmd(vma, address, pvmw->pmd); 4693 trace_remove_migration_pmd(address, pmd_val(pmde)); 4694 } 4695 #endif 4696