1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2009 Red Hat, Inc. 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/mm.h> 9 #include <linux/sched.h> 10 #include <linux/sched/mm.h> 11 #include <linux/sched/numa_balancing.h> 12 #include <linux/highmem.h> 13 #include <linux/hugetlb.h> 14 #include <linux/mmu_notifier.h> 15 #include <linux/rmap.h> 16 #include <linux/swap.h> 17 #include <linux/shrinker.h> 18 #include <linux/mm_inline.h> 19 #include <linux/swapops.h> 20 #include <linux/backing-dev.h> 21 #include <linux/dax.h> 22 #include <linux/mm_types.h> 23 #include <linux/khugepaged.h> 24 #include <linux/freezer.h> 25 #include <linux/mman.h> 26 #include <linux/memremap.h> 27 #include <linux/pagemap.h> 28 #include <linux/debugfs.h> 29 #include <linux/migrate.h> 30 #include <linux/hashtable.h> 31 #include <linux/userfaultfd_k.h> 32 #include <linux/page_idle.h> 33 #include <linux/shmem_fs.h> 34 #include <linux/oom.h> 35 #include <linux/numa.h> 36 #include <linux/page_owner.h> 37 #include <linux/sched/sysctl.h> 38 #include <linux/memory-tiers.h> 39 #include <linux/compat.h> 40 #include <linux/pgalloc_tag.h> 41 #include <linux/pagewalk.h> 42 43 #include <asm/tlb.h> 44 #include <asm/pgalloc.h> 45 #include "internal.h" 46 #include "swap.h" 47 48 #define CREATE_TRACE_POINTS 49 #include <trace/events/thp.h> 50 51 /* 52 * By default, transparent hugepage support is disabled in order to avoid 53 * risking an increased memory footprint for applications that are not 54 * guaranteed to benefit from it. When transparent hugepage support is 55 * enabled, it is for all mappings, and khugepaged scans all mappings. 56 * Defrag is invoked by khugepaged hugepage allocations and by page faults 57 * for all hugepage allocations. 58 */ 59 unsigned long transparent_hugepage_flags __read_mostly = 60 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS 61 (1<<TRANSPARENT_HUGEPAGE_FLAG)| 62 #endif 63 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE 64 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| 65 #endif 66 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)| 67 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| 68 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 69 70 static struct shrinker *deferred_split_shrinker; 71 static unsigned long deferred_split_count(struct shrinker *shrink, 72 struct shrink_control *sc); 73 static unsigned long deferred_split_scan(struct shrinker *shrink, 74 struct shrink_control *sc); 75 static bool split_underused_thp = true; 76 77 static atomic_t huge_zero_refcount; 78 struct folio *huge_zero_folio __read_mostly; 79 unsigned long huge_zero_pfn __read_mostly = ~0UL; 80 unsigned long huge_anon_orders_always __read_mostly; 81 unsigned long huge_anon_orders_madvise __read_mostly; 82 unsigned long huge_anon_orders_inherit __read_mostly; 83 static bool anon_orders_configured __initdata; 84 85 static inline bool file_thp_enabled(struct vm_area_struct *vma) 86 { 87 struct inode *inode; 88 89 if (!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS)) 90 return false; 91 92 if (!vma->vm_file) 93 return false; 94 95 inode = file_inode(vma->vm_file); 96 97 return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); 98 } 99 100 unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, 101 vm_flags_t vm_flags, 102 enum tva_type type, 103 unsigned long orders) 104 { 105 const bool smaps = type == TVA_SMAPS; 106 const bool in_pf = type == TVA_PAGEFAULT; 107 const bool forced_collapse = type == TVA_FORCED_COLLAPSE; 108 unsigned long supported_orders; 109 110 /* Check the intersection of requested and supported orders. */ 111 if (vma_is_anonymous(vma)) 112 supported_orders = THP_ORDERS_ALL_ANON; 113 else if (vma_is_special_huge(vma)) 114 supported_orders = THP_ORDERS_ALL_SPECIAL; 115 else 116 supported_orders = THP_ORDERS_ALL_FILE_DEFAULT; 117 118 orders &= supported_orders; 119 if (!orders) 120 return 0; 121 122 if (!vma->vm_mm) /* vdso */ 123 return 0; 124 125 if (thp_disabled_by_hw() || vma_thp_disabled(vma, vm_flags, forced_collapse)) 126 return 0; 127 128 /* khugepaged doesn't collapse DAX vma, but page fault is fine. */ 129 if (vma_is_dax(vma)) 130 return in_pf ? orders : 0; 131 132 /* 133 * khugepaged special VMA and hugetlb VMA. 134 * Must be checked after dax since some dax mappings may have 135 * VM_MIXEDMAP set. 136 */ 137 if (!in_pf && !smaps && (vm_flags & VM_NO_KHUGEPAGED)) 138 return 0; 139 140 /* 141 * Check alignment for file vma and size for both file and anon vma by 142 * filtering out the unsuitable orders. 143 * 144 * Skip the check for page fault. Huge fault does the check in fault 145 * handlers. 146 */ 147 if (!in_pf) { 148 int order = highest_order(orders); 149 unsigned long addr; 150 151 while (orders) { 152 addr = vma->vm_end - (PAGE_SIZE << order); 153 if (thp_vma_suitable_order(vma, addr, order)) 154 break; 155 order = next_order(&orders, order); 156 } 157 158 if (!orders) 159 return 0; 160 } 161 162 /* 163 * Enabled via shmem mount options or sysfs settings. 164 * Must be done before hugepage flags check since shmem has its 165 * own flags. 166 */ 167 if (!in_pf && shmem_file(vma->vm_file)) 168 return orders & shmem_allowable_huge_orders(file_inode(vma->vm_file), 169 vma, vma->vm_pgoff, 0, 170 forced_collapse); 171 172 if (!vma_is_anonymous(vma)) { 173 /* 174 * Enforce THP collapse requirements as necessary. Anonymous vmas 175 * were already handled in thp_vma_allowable_orders(). 176 */ 177 if (!forced_collapse && 178 (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) && 179 !hugepage_global_always()))) 180 return 0; 181 182 /* 183 * Trust that ->huge_fault() handlers know what they are doing 184 * in fault path. 185 */ 186 if (((in_pf || smaps)) && vma->vm_ops->huge_fault) 187 return orders; 188 /* Only regular file is valid in collapse path */ 189 if (((!in_pf || smaps)) && file_thp_enabled(vma)) 190 return orders; 191 return 0; 192 } 193 194 if (vma_is_temporary_stack(vma)) 195 return 0; 196 197 /* 198 * THPeligible bit of smaps should show 1 for proper VMAs even 199 * though anon_vma is not initialized yet. 200 * 201 * Allow page fault since anon_vma may be not initialized until 202 * the first page fault. 203 */ 204 if (!vma->anon_vma) 205 return (smaps || in_pf) ? orders : 0; 206 207 return orders; 208 } 209 210 static bool get_huge_zero_folio(void) 211 { 212 struct folio *zero_folio; 213 retry: 214 if (likely(atomic_inc_not_zero(&huge_zero_refcount))) 215 return true; 216 217 zero_folio = folio_alloc((GFP_TRANSHUGE | __GFP_ZERO | __GFP_ZEROTAGS) & 218 ~__GFP_MOVABLE, 219 HPAGE_PMD_ORDER); 220 if (!zero_folio) { 221 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); 222 return false; 223 } 224 /* Ensure zero folio won't have large_rmappable flag set. */ 225 folio_clear_large_rmappable(zero_folio); 226 preempt_disable(); 227 if (cmpxchg(&huge_zero_folio, NULL, zero_folio)) { 228 preempt_enable(); 229 folio_put(zero_folio); 230 goto retry; 231 } 232 WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio)); 233 234 /* We take additional reference here. It will be put back by shrinker */ 235 atomic_set(&huge_zero_refcount, 2); 236 preempt_enable(); 237 count_vm_event(THP_ZERO_PAGE_ALLOC); 238 return true; 239 } 240 241 static void put_huge_zero_folio(void) 242 { 243 /* 244 * Counter should never go to zero here. Only shrinker can put 245 * last reference. 246 */ 247 BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); 248 } 249 250 struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) 251 { 252 if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) 253 return huge_zero_folio; 254 255 if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm)) 256 return READ_ONCE(huge_zero_folio); 257 258 if (!get_huge_zero_folio()) 259 return NULL; 260 261 if (mm_flags_test_and_set(MMF_HUGE_ZERO_FOLIO, mm)) 262 put_huge_zero_folio(); 263 264 return READ_ONCE(huge_zero_folio); 265 } 266 267 void mm_put_huge_zero_folio(struct mm_struct *mm) 268 { 269 if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) 270 return; 271 272 if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm)) 273 put_huge_zero_folio(); 274 } 275 276 static unsigned long shrink_huge_zero_folio_count(struct shrinker *shrink, 277 struct shrink_control *sc) 278 { 279 /* we can free zero page only if last reference remains */ 280 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; 281 } 282 283 static unsigned long shrink_huge_zero_folio_scan(struct shrinker *shrink, 284 struct shrink_control *sc) 285 { 286 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { 287 struct folio *zero_folio = xchg(&huge_zero_folio, NULL); 288 BUG_ON(zero_folio == NULL); 289 WRITE_ONCE(huge_zero_pfn, ~0UL); 290 folio_put(zero_folio); 291 return HPAGE_PMD_NR; 292 } 293 294 return 0; 295 } 296 297 static struct shrinker *huge_zero_folio_shrinker; 298 299 #ifdef CONFIG_SYSFS 300 static ssize_t enabled_show(struct kobject *kobj, 301 struct kobj_attribute *attr, char *buf) 302 { 303 const char *output; 304 305 if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags)) 306 output = "[always] madvise never"; 307 else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 308 &transparent_hugepage_flags)) 309 output = "always [madvise] never"; 310 else 311 output = "always madvise [never]"; 312 313 return sysfs_emit(buf, "%s\n", output); 314 } 315 316 static ssize_t enabled_store(struct kobject *kobj, 317 struct kobj_attribute *attr, 318 const char *buf, size_t count) 319 { 320 ssize_t ret = count; 321 322 if (sysfs_streq(buf, "always")) { 323 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 324 set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 325 } else if (sysfs_streq(buf, "madvise")) { 326 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 327 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 328 } else if (sysfs_streq(buf, "never")) { 329 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 330 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 331 } else 332 ret = -EINVAL; 333 334 if (ret > 0) { 335 int err = start_stop_khugepaged(); 336 if (err) 337 ret = err; 338 } 339 return ret; 340 } 341 342 static struct kobj_attribute enabled_attr = __ATTR_RW(enabled); 343 344 ssize_t single_hugepage_flag_show(struct kobject *kobj, 345 struct kobj_attribute *attr, char *buf, 346 enum transparent_hugepage_flag flag) 347 { 348 return sysfs_emit(buf, "%d\n", 349 !!test_bit(flag, &transparent_hugepage_flags)); 350 } 351 352 ssize_t single_hugepage_flag_store(struct kobject *kobj, 353 struct kobj_attribute *attr, 354 const char *buf, size_t count, 355 enum transparent_hugepage_flag flag) 356 { 357 unsigned long value; 358 int ret; 359 360 ret = kstrtoul(buf, 10, &value); 361 if (ret < 0) 362 return ret; 363 if (value > 1) 364 return -EINVAL; 365 366 if (value) 367 set_bit(flag, &transparent_hugepage_flags); 368 else 369 clear_bit(flag, &transparent_hugepage_flags); 370 371 return count; 372 } 373 374 static ssize_t defrag_show(struct kobject *kobj, 375 struct kobj_attribute *attr, char *buf) 376 { 377 const char *output; 378 379 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, 380 &transparent_hugepage_flags)) 381 output = "[always] defer defer+madvise madvise never"; 382 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, 383 &transparent_hugepage_flags)) 384 output = "always [defer] defer+madvise madvise never"; 385 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, 386 &transparent_hugepage_flags)) 387 output = "always defer [defer+madvise] madvise never"; 388 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, 389 &transparent_hugepage_flags)) 390 output = "always defer defer+madvise [madvise] never"; 391 else 392 output = "always defer defer+madvise madvise [never]"; 393 394 return sysfs_emit(buf, "%s\n", output); 395 } 396 397 static ssize_t defrag_store(struct kobject *kobj, 398 struct kobj_attribute *attr, 399 const char *buf, size_t count) 400 { 401 if (sysfs_streq(buf, "always")) { 402 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 403 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 404 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 405 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 406 } else if (sysfs_streq(buf, "defer+madvise")) { 407 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 408 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 409 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 410 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 411 } else if (sysfs_streq(buf, "defer")) { 412 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 413 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 414 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 415 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 416 } else if (sysfs_streq(buf, "madvise")) { 417 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 418 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 419 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 420 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 421 } else if (sysfs_streq(buf, "never")) { 422 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 423 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 424 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 425 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 426 } else 427 return -EINVAL; 428 429 return count; 430 } 431 static struct kobj_attribute defrag_attr = __ATTR_RW(defrag); 432 433 static ssize_t use_zero_page_show(struct kobject *kobj, 434 struct kobj_attribute *attr, char *buf) 435 { 436 return single_hugepage_flag_show(kobj, attr, buf, 437 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 438 } 439 static ssize_t use_zero_page_store(struct kobject *kobj, 440 struct kobj_attribute *attr, const char *buf, size_t count) 441 { 442 return single_hugepage_flag_store(kobj, attr, buf, count, 443 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 444 } 445 static struct kobj_attribute use_zero_page_attr = __ATTR_RW(use_zero_page); 446 447 static ssize_t hpage_pmd_size_show(struct kobject *kobj, 448 struct kobj_attribute *attr, char *buf) 449 { 450 return sysfs_emit(buf, "%lu\n", HPAGE_PMD_SIZE); 451 } 452 static struct kobj_attribute hpage_pmd_size_attr = 453 __ATTR_RO(hpage_pmd_size); 454 455 static ssize_t split_underused_thp_show(struct kobject *kobj, 456 struct kobj_attribute *attr, char *buf) 457 { 458 return sysfs_emit(buf, "%d\n", split_underused_thp); 459 } 460 461 static ssize_t split_underused_thp_store(struct kobject *kobj, 462 struct kobj_attribute *attr, 463 const char *buf, size_t count) 464 { 465 int err = kstrtobool(buf, &split_underused_thp); 466 467 if (err < 0) 468 return err; 469 470 return count; 471 } 472 473 static struct kobj_attribute split_underused_thp_attr = __ATTR( 474 shrink_underused, 0644, split_underused_thp_show, split_underused_thp_store); 475 476 static struct attribute *hugepage_attr[] = { 477 &enabled_attr.attr, 478 &defrag_attr.attr, 479 &use_zero_page_attr.attr, 480 &hpage_pmd_size_attr.attr, 481 #ifdef CONFIG_SHMEM 482 &shmem_enabled_attr.attr, 483 #endif 484 &split_underused_thp_attr.attr, 485 NULL, 486 }; 487 488 static const struct attribute_group hugepage_attr_group = { 489 .attrs = hugepage_attr, 490 }; 491 492 static void hugepage_exit_sysfs(struct kobject *hugepage_kobj); 493 static void thpsize_release(struct kobject *kobj); 494 static DEFINE_SPINLOCK(huge_anon_orders_lock); 495 static LIST_HEAD(thpsize_list); 496 497 static ssize_t anon_enabled_show(struct kobject *kobj, 498 struct kobj_attribute *attr, char *buf) 499 { 500 int order = to_thpsize(kobj)->order; 501 const char *output; 502 503 if (test_bit(order, &huge_anon_orders_always)) 504 output = "[always] inherit madvise never"; 505 else if (test_bit(order, &huge_anon_orders_inherit)) 506 output = "always [inherit] madvise never"; 507 else if (test_bit(order, &huge_anon_orders_madvise)) 508 output = "always inherit [madvise] never"; 509 else 510 output = "always inherit madvise [never]"; 511 512 return sysfs_emit(buf, "%s\n", output); 513 } 514 515 static ssize_t anon_enabled_store(struct kobject *kobj, 516 struct kobj_attribute *attr, 517 const char *buf, size_t count) 518 { 519 int order = to_thpsize(kobj)->order; 520 ssize_t ret = count; 521 522 if (sysfs_streq(buf, "always")) { 523 spin_lock(&huge_anon_orders_lock); 524 clear_bit(order, &huge_anon_orders_inherit); 525 clear_bit(order, &huge_anon_orders_madvise); 526 set_bit(order, &huge_anon_orders_always); 527 spin_unlock(&huge_anon_orders_lock); 528 } else if (sysfs_streq(buf, "inherit")) { 529 spin_lock(&huge_anon_orders_lock); 530 clear_bit(order, &huge_anon_orders_always); 531 clear_bit(order, &huge_anon_orders_madvise); 532 set_bit(order, &huge_anon_orders_inherit); 533 spin_unlock(&huge_anon_orders_lock); 534 } else if (sysfs_streq(buf, "madvise")) { 535 spin_lock(&huge_anon_orders_lock); 536 clear_bit(order, &huge_anon_orders_always); 537 clear_bit(order, &huge_anon_orders_inherit); 538 set_bit(order, &huge_anon_orders_madvise); 539 spin_unlock(&huge_anon_orders_lock); 540 } else if (sysfs_streq(buf, "never")) { 541 spin_lock(&huge_anon_orders_lock); 542 clear_bit(order, &huge_anon_orders_always); 543 clear_bit(order, &huge_anon_orders_inherit); 544 clear_bit(order, &huge_anon_orders_madvise); 545 spin_unlock(&huge_anon_orders_lock); 546 } else 547 ret = -EINVAL; 548 549 if (ret > 0) { 550 int err; 551 552 err = start_stop_khugepaged(); 553 if (err) 554 ret = err; 555 } 556 return ret; 557 } 558 559 static struct kobj_attribute anon_enabled_attr = 560 __ATTR(enabled, 0644, anon_enabled_show, anon_enabled_store); 561 562 static struct attribute *anon_ctrl_attrs[] = { 563 &anon_enabled_attr.attr, 564 NULL, 565 }; 566 567 static const struct attribute_group anon_ctrl_attr_grp = { 568 .attrs = anon_ctrl_attrs, 569 }; 570 571 static struct attribute *file_ctrl_attrs[] = { 572 #ifdef CONFIG_SHMEM 573 &thpsize_shmem_enabled_attr.attr, 574 #endif 575 NULL, 576 }; 577 578 static const struct attribute_group file_ctrl_attr_grp = { 579 .attrs = file_ctrl_attrs, 580 }; 581 582 static struct attribute *any_ctrl_attrs[] = { 583 NULL, 584 }; 585 586 static const struct attribute_group any_ctrl_attr_grp = { 587 .attrs = any_ctrl_attrs, 588 }; 589 590 static const struct kobj_type thpsize_ktype = { 591 .release = &thpsize_release, 592 .sysfs_ops = &kobj_sysfs_ops, 593 }; 594 595 DEFINE_PER_CPU(struct mthp_stat, mthp_stats) = {{{0}}}; 596 597 static unsigned long sum_mthp_stat(int order, enum mthp_stat_item item) 598 { 599 unsigned long sum = 0; 600 int cpu; 601 602 for_each_possible_cpu(cpu) { 603 struct mthp_stat *this = &per_cpu(mthp_stats, cpu); 604 605 sum += this->stats[order][item]; 606 } 607 608 return sum; 609 } 610 611 #define DEFINE_MTHP_STAT_ATTR(_name, _index) \ 612 static ssize_t _name##_show(struct kobject *kobj, \ 613 struct kobj_attribute *attr, char *buf) \ 614 { \ 615 int order = to_thpsize(kobj)->order; \ 616 \ 617 return sysfs_emit(buf, "%lu\n", sum_mthp_stat(order, _index)); \ 618 } \ 619 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 620 621 DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC); 622 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK); 623 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); 624 DEFINE_MTHP_STAT_ATTR(zswpout, MTHP_STAT_ZSWPOUT); 625 DEFINE_MTHP_STAT_ATTR(swpin, MTHP_STAT_SWPIN); 626 DEFINE_MTHP_STAT_ATTR(swpin_fallback, MTHP_STAT_SWPIN_FALLBACK); 627 DEFINE_MTHP_STAT_ATTR(swpin_fallback_charge, MTHP_STAT_SWPIN_FALLBACK_CHARGE); 628 DEFINE_MTHP_STAT_ATTR(swpout, MTHP_STAT_SWPOUT); 629 DEFINE_MTHP_STAT_ATTR(swpout_fallback, MTHP_STAT_SWPOUT_FALLBACK); 630 #ifdef CONFIG_SHMEM 631 DEFINE_MTHP_STAT_ATTR(shmem_alloc, MTHP_STAT_SHMEM_ALLOC); 632 DEFINE_MTHP_STAT_ATTR(shmem_fallback, MTHP_STAT_SHMEM_FALLBACK); 633 DEFINE_MTHP_STAT_ATTR(shmem_fallback_charge, MTHP_STAT_SHMEM_FALLBACK_CHARGE); 634 #endif 635 DEFINE_MTHP_STAT_ATTR(split, MTHP_STAT_SPLIT); 636 DEFINE_MTHP_STAT_ATTR(split_failed, MTHP_STAT_SPLIT_FAILED); 637 DEFINE_MTHP_STAT_ATTR(split_deferred, MTHP_STAT_SPLIT_DEFERRED); 638 DEFINE_MTHP_STAT_ATTR(nr_anon, MTHP_STAT_NR_ANON); 639 DEFINE_MTHP_STAT_ATTR(nr_anon_partially_mapped, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED); 640 641 static struct attribute *anon_stats_attrs[] = { 642 &anon_fault_alloc_attr.attr, 643 &anon_fault_fallback_attr.attr, 644 &anon_fault_fallback_charge_attr.attr, 645 #ifndef CONFIG_SHMEM 646 &zswpout_attr.attr, 647 &swpin_attr.attr, 648 &swpin_fallback_attr.attr, 649 &swpin_fallback_charge_attr.attr, 650 &swpout_attr.attr, 651 &swpout_fallback_attr.attr, 652 #endif 653 &split_deferred_attr.attr, 654 &nr_anon_attr.attr, 655 &nr_anon_partially_mapped_attr.attr, 656 NULL, 657 }; 658 659 static struct attribute_group anon_stats_attr_grp = { 660 .name = "stats", 661 .attrs = anon_stats_attrs, 662 }; 663 664 static struct attribute *file_stats_attrs[] = { 665 #ifdef CONFIG_SHMEM 666 &shmem_alloc_attr.attr, 667 &shmem_fallback_attr.attr, 668 &shmem_fallback_charge_attr.attr, 669 #endif 670 NULL, 671 }; 672 673 static struct attribute_group file_stats_attr_grp = { 674 .name = "stats", 675 .attrs = file_stats_attrs, 676 }; 677 678 static struct attribute *any_stats_attrs[] = { 679 #ifdef CONFIG_SHMEM 680 &zswpout_attr.attr, 681 &swpin_attr.attr, 682 &swpin_fallback_attr.attr, 683 &swpin_fallback_charge_attr.attr, 684 &swpout_attr.attr, 685 &swpout_fallback_attr.attr, 686 #endif 687 &split_attr.attr, 688 &split_failed_attr.attr, 689 NULL, 690 }; 691 692 static struct attribute_group any_stats_attr_grp = { 693 .name = "stats", 694 .attrs = any_stats_attrs, 695 }; 696 697 static int sysfs_add_group(struct kobject *kobj, 698 const struct attribute_group *grp) 699 { 700 int ret = -ENOENT; 701 702 /* 703 * If the group is named, try to merge first, assuming the subdirectory 704 * was already created. This avoids the warning emitted by 705 * sysfs_create_group() if the directory already exists. 706 */ 707 if (grp->name) 708 ret = sysfs_merge_group(kobj, grp); 709 if (ret) 710 ret = sysfs_create_group(kobj, grp); 711 712 return ret; 713 } 714 715 static struct thpsize *thpsize_create(int order, struct kobject *parent) 716 { 717 unsigned long size = (PAGE_SIZE << order) / SZ_1K; 718 struct thpsize *thpsize; 719 int ret = -ENOMEM; 720 721 thpsize = kzalloc(sizeof(*thpsize), GFP_KERNEL); 722 if (!thpsize) 723 goto err; 724 725 thpsize->order = order; 726 727 ret = kobject_init_and_add(&thpsize->kobj, &thpsize_ktype, parent, 728 "hugepages-%lukB", size); 729 if (ret) { 730 kfree(thpsize); 731 goto err; 732 } 733 734 735 ret = sysfs_add_group(&thpsize->kobj, &any_ctrl_attr_grp); 736 if (ret) 737 goto err_put; 738 739 ret = sysfs_add_group(&thpsize->kobj, &any_stats_attr_grp); 740 if (ret) 741 goto err_put; 742 743 if (BIT(order) & THP_ORDERS_ALL_ANON) { 744 ret = sysfs_add_group(&thpsize->kobj, &anon_ctrl_attr_grp); 745 if (ret) 746 goto err_put; 747 748 ret = sysfs_add_group(&thpsize->kobj, &anon_stats_attr_grp); 749 if (ret) 750 goto err_put; 751 } 752 753 if (BIT(order) & THP_ORDERS_ALL_FILE_DEFAULT) { 754 ret = sysfs_add_group(&thpsize->kobj, &file_ctrl_attr_grp); 755 if (ret) 756 goto err_put; 757 758 ret = sysfs_add_group(&thpsize->kobj, &file_stats_attr_grp); 759 if (ret) 760 goto err_put; 761 } 762 763 return thpsize; 764 err_put: 765 kobject_put(&thpsize->kobj); 766 err: 767 return ERR_PTR(ret); 768 } 769 770 static void thpsize_release(struct kobject *kobj) 771 { 772 kfree(to_thpsize(kobj)); 773 } 774 775 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj) 776 { 777 int err; 778 struct thpsize *thpsize; 779 unsigned long orders; 780 int order; 781 782 /* 783 * Default to setting PMD-sized THP to inherit the global setting and 784 * disable all other sizes. powerpc's PMD_ORDER isn't a compile-time 785 * constant so we have to do this here. 786 */ 787 if (!anon_orders_configured) 788 huge_anon_orders_inherit = BIT(PMD_ORDER); 789 790 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj); 791 if (unlikely(!*hugepage_kobj)) { 792 pr_err("failed to create transparent hugepage kobject\n"); 793 return -ENOMEM; 794 } 795 796 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group); 797 if (err) { 798 pr_err("failed to register transparent hugepage group\n"); 799 goto delete_obj; 800 } 801 802 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group); 803 if (err) { 804 pr_err("failed to register transparent hugepage group\n"); 805 goto remove_hp_group; 806 } 807 808 orders = THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_FILE_DEFAULT; 809 order = highest_order(orders); 810 while (orders) { 811 thpsize = thpsize_create(order, *hugepage_kobj); 812 if (IS_ERR(thpsize)) { 813 pr_err("failed to create thpsize for order %d\n", order); 814 err = PTR_ERR(thpsize); 815 goto remove_all; 816 } 817 list_add(&thpsize->node, &thpsize_list); 818 order = next_order(&orders, order); 819 } 820 821 return 0; 822 823 remove_all: 824 hugepage_exit_sysfs(*hugepage_kobj); 825 return err; 826 remove_hp_group: 827 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group); 828 delete_obj: 829 kobject_put(*hugepage_kobj); 830 return err; 831 } 832 833 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj) 834 { 835 struct thpsize *thpsize, *tmp; 836 837 list_for_each_entry_safe(thpsize, tmp, &thpsize_list, node) { 838 list_del(&thpsize->node); 839 kobject_put(&thpsize->kobj); 840 } 841 842 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group); 843 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group); 844 kobject_put(hugepage_kobj); 845 } 846 #else 847 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj) 848 { 849 return 0; 850 } 851 852 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) 853 { 854 } 855 #endif /* CONFIG_SYSFS */ 856 857 static int __init thp_shrinker_init(void) 858 { 859 deferred_split_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE | 860 SHRINKER_MEMCG_AWARE | 861 SHRINKER_NONSLAB, 862 "thp-deferred_split"); 863 if (!deferred_split_shrinker) 864 return -ENOMEM; 865 866 deferred_split_shrinker->count_objects = deferred_split_count; 867 deferred_split_shrinker->scan_objects = deferred_split_scan; 868 shrinker_register(deferred_split_shrinker); 869 870 if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) { 871 /* 872 * Bump the reference of the huge_zero_folio and do not 873 * initialize the shrinker. 874 * 875 * huge_zero_folio will always be NULL on failure. We assume 876 * that get_huge_zero_folio() will most likely not fail as 877 * thp_shrinker_init() is invoked early on during boot. 878 */ 879 if (!get_huge_zero_folio()) 880 pr_warn("Allocating persistent huge zero folio failed\n"); 881 return 0; 882 } 883 884 huge_zero_folio_shrinker = shrinker_alloc(0, "thp-zero"); 885 if (!huge_zero_folio_shrinker) { 886 shrinker_free(deferred_split_shrinker); 887 return -ENOMEM; 888 } 889 890 huge_zero_folio_shrinker->count_objects = shrink_huge_zero_folio_count; 891 huge_zero_folio_shrinker->scan_objects = shrink_huge_zero_folio_scan; 892 shrinker_register(huge_zero_folio_shrinker); 893 894 return 0; 895 } 896 897 static void __init thp_shrinker_exit(void) 898 { 899 shrinker_free(huge_zero_folio_shrinker); 900 shrinker_free(deferred_split_shrinker); 901 } 902 903 static int __init hugepage_init(void) 904 { 905 int err; 906 struct kobject *hugepage_kobj; 907 908 if (!has_transparent_hugepage()) { 909 transparent_hugepage_flags = 1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED; 910 return -EINVAL; 911 } 912 913 /* 914 * hugepages can't be allocated by the buddy allocator 915 */ 916 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER > MAX_PAGE_ORDER); 917 918 err = hugepage_init_sysfs(&hugepage_kobj); 919 if (err) 920 goto err_sysfs; 921 922 err = khugepaged_init(); 923 if (err) 924 goto err_slab; 925 926 err = thp_shrinker_init(); 927 if (err) 928 goto err_shrinker; 929 930 /* 931 * By default disable transparent hugepages on smaller systems, 932 * where the extra memory used could hurt more than TLB overhead 933 * is likely to save. The admin can still enable it through /sys. 934 */ 935 if (totalram_pages() < MB_TO_PAGES(512)) { 936 transparent_hugepage_flags = 0; 937 return 0; 938 } 939 940 err = start_stop_khugepaged(); 941 if (err) 942 goto err_khugepaged; 943 944 return 0; 945 err_khugepaged: 946 thp_shrinker_exit(); 947 err_shrinker: 948 khugepaged_destroy(); 949 err_slab: 950 hugepage_exit_sysfs(hugepage_kobj); 951 err_sysfs: 952 return err; 953 } 954 subsys_initcall(hugepage_init); 955 956 static int __init setup_transparent_hugepage(char *str) 957 { 958 int ret = 0; 959 if (!str) 960 goto out; 961 if (!strcmp(str, "always")) { 962 set_bit(TRANSPARENT_HUGEPAGE_FLAG, 963 &transparent_hugepage_flags); 964 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 965 &transparent_hugepage_flags); 966 ret = 1; 967 } else if (!strcmp(str, "madvise")) { 968 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 969 &transparent_hugepage_flags); 970 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 971 &transparent_hugepage_flags); 972 ret = 1; 973 } else if (!strcmp(str, "never")) { 974 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 975 &transparent_hugepage_flags); 976 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 977 &transparent_hugepage_flags); 978 ret = 1; 979 } 980 out: 981 if (!ret) 982 pr_warn("transparent_hugepage= cannot parse, ignored\n"); 983 return ret; 984 } 985 __setup("transparent_hugepage=", setup_transparent_hugepage); 986 987 static char str_dup[PAGE_SIZE] __initdata; 988 static int __init setup_thp_anon(char *str) 989 { 990 char *token, *range, *policy, *subtoken; 991 unsigned long always, inherit, madvise; 992 char *start_size, *end_size; 993 int start, end, nr; 994 char *p; 995 996 if (!str || strlen(str) + 1 > PAGE_SIZE) 997 goto err; 998 strscpy(str_dup, str); 999 1000 always = huge_anon_orders_always; 1001 madvise = huge_anon_orders_madvise; 1002 inherit = huge_anon_orders_inherit; 1003 p = str_dup; 1004 while ((token = strsep(&p, ";")) != NULL) { 1005 range = strsep(&token, ":"); 1006 policy = token; 1007 1008 if (!policy) 1009 goto err; 1010 1011 while ((subtoken = strsep(&range, ",")) != NULL) { 1012 if (strchr(subtoken, '-')) { 1013 start_size = strsep(&subtoken, "-"); 1014 end_size = subtoken; 1015 1016 start = get_order_from_str(start_size, THP_ORDERS_ALL_ANON); 1017 end = get_order_from_str(end_size, THP_ORDERS_ALL_ANON); 1018 } else { 1019 start_size = end_size = subtoken; 1020 start = end = get_order_from_str(subtoken, 1021 THP_ORDERS_ALL_ANON); 1022 } 1023 1024 if (start == -EINVAL) { 1025 pr_err("invalid size %s in thp_anon boot parameter\n", start_size); 1026 goto err; 1027 } 1028 1029 if (end == -EINVAL) { 1030 pr_err("invalid size %s in thp_anon boot parameter\n", end_size); 1031 goto err; 1032 } 1033 1034 if (start < 0 || end < 0 || start > end) 1035 goto err; 1036 1037 nr = end - start + 1; 1038 if (!strcmp(policy, "always")) { 1039 bitmap_set(&always, start, nr); 1040 bitmap_clear(&inherit, start, nr); 1041 bitmap_clear(&madvise, start, nr); 1042 } else if (!strcmp(policy, "madvise")) { 1043 bitmap_set(&madvise, start, nr); 1044 bitmap_clear(&inherit, start, nr); 1045 bitmap_clear(&always, start, nr); 1046 } else if (!strcmp(policy, "inherit")) { 1047 bitmap_set(&inherit, start, nr); 1048 bitmap_clear(&madvise, start, nr); 1049 bitmap_clear(&always, start, nr); 1050 } else if (!strcmp(policy, "never")) { 1051 bitmap_clear(&inherit, start, nr); 1052 bitmap_clear(&madvise, start, nr); 1053 bitmap_clear(&always, start, nr); 1054 } else { 1055 pr_err("invalid policy %s in thp_anon boot parameter\n", policy); 1056 goto err; 1057 } 1058 } 1059 } 1060 1061 huge_anon_orders_always = always; 1062 huge_anon_orders_madvise = madvise; 1063 huge_anon_orders_inherit = inherit; 1064 anon_orders_configured = true; 1065 return 1; 1066 1067 err: 1068 pr_warn("thp_anon=%s: error parsing string, ignoring setting\n", str); 1069 return 0; 1070 } 1071 __setup("thp_anon=", setup_thp_anon); 1072 1073 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 1074 { 1075 if (likely(vma->vm_flags & VM_WRITE)) 1076 pmd = pmd_mkwrite(pmd, vma); 1077 return pmd; 1078 } 1079 1080 #ifdef CONFIG_MEMCG 1081 static inline 1082 struct deferred_split *get_deferred_split_queue(struct folio *folio) 1083 { 1084 struct mem_cgroup *memcg = folio_memcg(folio); 1085 struct pglist_data *pgdat = NODE_DATA(folio_nid(folio)); 1086 1087 if (memcg) 1088 return &memcg->deferred_split_queue; 1089 else 1090 return &pgdat->deferred_split_queue; 1091 } 1092 #else 1093 static inline 1094 struct deferred_split *get_deferred_split_queue(struct folio *folio) 1095 { 1096 struct pglist_data *pgdat = NODE_DATA(folio_nid(folio)); 1097 1098 return &pgdat->deferred_split_queue; 1099 } 1100 #endif 1101 1102 static inline bool is_transparent_hugepage(const struct folio *folio) 1103 { 1104 if (!folio_test_large(folio)) 1105 return false; 1106 1107 return is_huge_zero_folio(folio) || 1108 folio_test_large_rmappable(folio); 1109 } 1110 1111 static unsigned long __thp_get_unmapped_area(struct file *filp, 1112 unsigned long addr, unsigned long len, 1113 loff_t off, unsigned long flags, unsigned long size, 1114 vm_flags_t vm_flags) 1115 { 1116 loff_t off_end = off + len; 1117 loff_t off_align = round_up(off, size); 1118 unsigned long len_pad, ret, off_sub; 1119 1120 if (!IS_ENABLED(CONFIG_64BIT) || in_compat_syscall()) 1121 return 0; 1122 1123 if (off_end <= off_align || (off_end - off_align) < size) 1124 return 0; 1125 1126 len_pad = len + size; 1127 if (len_pad < len || (off + len_pad) < off) 1128 return 0; 1129 1130 ret = mm_get_unmapped_area_vmflags(current->mm, filp, addr, len_pad, 1131 off >> PAGE_SHIFT, flags, vm_flags); 1132 1133 /* 1134 * The failure might be due to length padding. The caller will retry 1135 * without the padding. 1136 */ 1137 if (IS_ERR_VALUE(ret)) 1138 return 0; 1139 1140 /* 1141 * Do not try to align to THP boundary if allocation at the address 1142 * hint succeeds. 1143 */ 1144 if (ret == addr) 1145 return addr; 1146 1147 off_sub = (off - ret) & (size - 1); 1148 1149 if (mm_flags_test(MMF_TOPDOWN, current->mm) && !off_sub) 1150 return ret + size; 1151 1152 ret += off_sub; 1153 return ret; 1154 } 1155 1156 unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, 1157 unsigned long len, unsigned long pgoff, unsigned long flags, 1158 vm_flags_t vm_flags) 1159 { 1160 unsigned long ret; 1161 loff_t off = (loff_t)pgoff << PAGE_SHIFT; 1162 1163 ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, vm_flags); 1164 if (ret) 1165 return ret; 1166 1167 return mm_get_unmapped_area_vmflags(current->mm, filp, addr, len, pgoff, flags, 1168 vm_flags); 1169 } 1170 1171 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, 1172 unsigned long len, unsigned long pgoff, unsigned long flags) 1173 { 1174 return thp_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0); 1175 } 1176 EXPORT_SYMBOL_GPL(thp_get_unmapped_area); 1177 1178 static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma, 1179 unsigned long addr) 1180 { 1181 gfp_t gfp = vma_thp_gfp_mask(vma); 1182 const int order = HPAGE_PMD_ORDER; 1183 struct folio *folio; 1184 1185 folio = vma_alloc_folio(gfp, order, vma, addr & HPAGE_PMD_MASK); 1186 1187 if (unlikely(!folio)) { 1188 count_vm_event(THP_FAULT_FALLBACK); 1189 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK); 1190 return NULL; 1191 } 1192 1193 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 1194 if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) { 1195 folio_put(folio); 1196 count_vm_event(THP_FAULT_FALLBACK); 1197 count_vm_event(THP_FAULT_FALLBACK_CHARGE); 1198 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK); 1199 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); 1200 return NULL; 1201 } 1202 folio_throttle_swaprate(folio, gfp); 1203 1204 /* 1205 * When a folio is not zeroed during allocation (__GFP_ZERO not used) 1206 * or user folios require special handling, folio_zero_user() is used to 1207 * make sure that the page corresponding to the faulting address will be 1208 * hot in the cache after zeroing. 1209 */ 1210 if (user_alloc_needs_zeroing()) 1211 folio_zero_user(folio, addr); 1212 /* 1213 * The memory barrier inside __folio_mark_uptodate makes sure that 1214 * folio_zero_user writes become visible before the set_pmd_at() 1215 * write. 1216 */ 1217 __folio_mark_uptodate(folio); 1218 return folio; 1219 } 1220 1221 static void map_anon_folio_pmd(struct folio *folio, pmd_t *pmd, 1222 struct vm_area_struct *vma, unsigned long haddr) 1223 { 1224 pmd_t entry; 1225 1226 entry = folio_mk_pmd(folio, vma->vm_page_prot); 1227 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1228 folio_add_new_anon_rmap(folio, vma, haddr, RMAP_EXCLUSIVE); 1229 folio_add_lru_vma(folio, vma); 1230 set_pmd_at(vma->vm_mm, haddr, pmd, entry); 1231 update_mmu_cache_pmd(vma, haddr, pmd); 1232 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1233 count_vm_event(THP_FAULT_ALLOC); 1234 count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_ALLOC); 1235 count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); 1236 } 1237 1238 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf) 1239 { 1240 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1241 struct vm_area_struct *vma = vmf->vma; 1242 struct folio *folio; 1243 pgtable_t pgtable; 1244 vm_fault_t ret = 0; 1245 1246 folio = vma_alloc_anon_folio_pmd(vma, vmf->address); 1247 if (unlikely(!folio)) 1248 return VM_FAULT_FALLBACK; 1249 1250 pgtable = pte_alloc_one(vma->vm_mm); 1251 if (unlikely(!pgtable)) { 1252 ret = VM_FAULT_OOM; 1253 goto release; 1254 } 1255 1256 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1257 if (unlikely(!pmd_none(*vmf->pmd))) { 1258 goto unlock_release; 1259 } else { 1260 ret = check_stable_address_space(vma->vm_mm); 1261 if (ret) 1262 goto unlock_release; 1263 1264 /* Deliver the page fault to userland */ 1265 if (userfaultfd_missing(vma)) { 1266 spin_unlock(vmf->ptl); 1267 folio_put(folio); 1268 pte_free(vma->vm_mm, pgtable); 1269 ret = handle_userfault(vmf, VM_UFFD_MISSING); 1270 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 1271 return ret; 1272 } 1273 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); 1274 map_anon_folio_pmd(folio, vmf->pmd, vma, haddr); 1275 mm_inc_nr_ptes(vma->vm_mm); 1276 deferred_split_folio(folio, false); 1277 spin_unlock(vmf->ptl); 1278 } 1279 1280 return 0; 1281 unlock_release: 1282 spin_unlock(vmf->ptl); 1283 release: 1284 if (pgtable) 1285 pte_free(vma->vm_mm, pgtable); 1286 folio_put(folio); 1287 return ret; 1288 1289 } 1290 1291 /* 1292 * always: directly stall for all thp allocations 1293 * defer: wake kswapd and fail if not immediately available 1294 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise 1295 * fail if not immediately available 1296 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately 1297 * available 1298 * never: never stall for any thp allocation 1299 */ 1300 gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma) 1301 { 1302 const bool vma_madvised = vma && (vma->vm_flags & VM_HUGEPAGE); 1303 1304 /* Always do synchronous compaction */ 1305 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) 1306 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY); 1307 1308 /* Kick kcompactd and fail quickly */ 1309 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) 1310 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM; 1311 1312 /* Synchronous compaction if madvised, otherwise kick kcompactd */ 1313 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags)) 1314 return GFP_TRANSHUGE_LIGHT | 1315 (vma_madvised ? __GFP_DIRECT_RECLAIM : 1316 __GFP_KSWAPD_RECLAIM); 1317 1318 /* Only do synchronous compaction if madvised */ 1319 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags)) 1320 return GFP_TRANSHUGE_LIGHT | 1321 (vma_madvised ? __GFP_DIRECT_RECLAIM : 0); 1322 1323 return GFP_TRANSHUGE_LIGHT; 1324 } 1325 1326 /* Caller must hold page table lock. */ 1327 static void set_huge_zero_folio(pgtable_t pgtable, struct mm_struct *mm, 1328 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, 1329 struct folio *zero_folio) 1330 { 1331 pmd_t entry; 1332 entry = folio_mk_pmd(zero_folio, vma->vm_page_prot); 1333 entry = pmd_mkspecial(entry); 1334 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1335 set_pmd_at(mm, haddr, pmd, entry); 1336 mm_inc_nr_ptes(mm); 1337 } 1338 1339 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) 1340 { 1341 struct vm_area_struct *vma = vmf->vma; 1342 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1343 vm_fault_t ret; 1344 1345 if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER)) 1346 return VM_FAULT_FALLBACK; 1347 ret = vmf_anon_prepare(vmf); 1348 if (ret) 1349 return ret; 1350 khugepaged_enter_vma(vma, vma->vm_flags); 1351 1352 if (!(vmf->flags & FAULT_FLAG_WRITE) && 1353 !mm_forbids_zeropage(vma->vm_mm) && 1354 transparent_hugepage_use_zero_page()) { 1355 pgtable_t pgtable; 1356 struct folio *zero_folio; 1357 vm_fault_t ret; 1358 1359 pgtable = pte_alloc_one(vma->vm_mm); 1360 if (unlikely(!pgtable)) 1361 return VM_FAULT_OOM; 1362 zero_folio = mm_get_huge_zero_folio(vma->vm_mm); 1363 if (unlikely(!zero_folio)) { 1364 pte_free(vma->vm_mm, pgtable); 1365 count_vm_event(THP_FAULT_FALLBACK); 1366 return VM_FAULT_FALLBACK; 1367 } 1368 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1369 ret = 0; 1370 if (pmd_none(*vmf->pmd)) { 1371 ret = check_stable_address_space(vma->vm_mm); 1372 if (ret) { 1373 spin_unlock(vmf->ptl); 1374 pte_free(vma->vm_mm, pgtable); 1375 } else if (userfaultfd_missing(vma)) { 1376 spin_unlock(vmf->ptl); 1377 pte_free(vma->vm_mm, pgtable); 1378 ret = handle_userfault(vmf, VM_UFFD_MISSING); 1379 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 1380 } else { 1381 set_huge_zero_folio(pgtable, vma->vm_mm, vma, 1382 haddr, vmf->pmd, zero_folio); 1383 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1384 spin_unlock(vmf->ptl); 1385 } 1386 } else { 1387 spin_unlock(vmf->ptl); 1388 pte_free(vma->vm_mm, pgtable); 1389 } 1390 return ret; 1391 } 1392 1393 return __do_huge_pmd_anonymous_page(vmf); 1394 } 1395 1396 struct folio_or_pfn { 1397 union { 1398 struct folio *folio; 1399 unsigned long pfn; 1400 }; 1401 bool is_folio; 1402 }; 1403 1404 static vm_fault_t insert_pmd(struct vm_area_struct *vma, unsigned long addr, 1405 pmd_t *pmd, struct folio_or_pfn fop, pgprot_t prot, 1406 bool write) 1407 { 1408 struct mm_struct *mm = vma->vm_mm; 1409 pgtable_t pgtable = NULL; 1410 spinlock_t *ptl; 1411 pmd_t entry; 1412 1413 if (addr < vma->vm_start || addr >= vma->vm_end) 1414 return VM_FAULT_SIGBUS; 1415 1416 if (arch_needs_pgtable_deposit()) { 1417 pgtable = pte_alloc_one(vma->vm_mm); 1418 if (!pgtable) 1419 return VM_FAULT_OOM; 1420 } 1421 1422 ptl = pmd_lock(mm, pmd); 1423 if (!pmd_none(*pmd)) { 1424 const unsigned long pfn = fop.is_folio ? folio_pfn(fop.folio) : 1425 fop.pfn; 1426 1427 if (write) { 1428 if (pmd_pfn(*pmd) != pfn) { 1429 WARN_ON_ONCE(!is_huge_zero_pmd(*pmd)); 1430 goto out_unlock; 1431 } 1432 entry = pmd_mkyoung(*pmd); 1433 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1434 if (pmdp_set_access_flags(vma, addr, pmd, entry, 1)) 1435 update_mmu_cache_pmd(vma, addr, pmd); 1436 } 1437 goto out_unlock; 1438 } 1439 1440 if (fop.is_folio) { 1441 entry = folio_mk_pmd(fop.folio, vma->vm_page_prot); 1442 1443 if (is_huge_zero_folio(fop.folio)) { 1444 entry = pmd_mkspecial(entry); 1445 } else { 1446 folio_get(fop.folio); 1447 folio_add_file_rmap_pmd(fop.folio, &fop.folio->page, vma); 1448 add_mm_counter(mm, mm_counter_file(fop.folio), HPAGE_PMD_NR); 1449 } 1450 } else { 1451 entry = pmd_mkhuge(pfn_pmd(fop.pfn, prot)); 1452 entry = pmd_mkspecial(entry); 1453 } 1454 if (write) { 1455 entry = pmd_mkyoung(pmd_mkdirty(entry)); 1456 entry = maybe_pmd_mkwrite(entry, vma); 1457 } 1458 1459 if (pgtable) { 1460 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1461 mm_inc_nr_ptes(mm); 1462 pgtable = NULL; 1463 } 1464 1465 set_pmd_at(mm, addr, pmd, entry); 1466 update_mmu_cache_pmd(vma, addr, pmd); 1467 1468 out_unlock: 1469 spin_unlock(ptl); 1470 if (pgtable) 1471 pte_free(mm, pgtable); 1472 return VM_FAULT_NOPAGE; 1473 } 1474 1475 /** 1476 * vmf_insert_pfn_pmd - insert a pmd size pfn 1477 * @vmf: Structure describing the fault 1478 * @pfn: pfn to insert 1479 * @write: whether it's a write fault 1480 * 1481 * Insert a pmd size pfn. See vmf_insert_pfn() for additional info. 1482 * 1483 * Return: vm_fault_t value. 1484 */ 1485 vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, unsigned long pfn, 1486 bool write) 1487 { 1488 unsigned long addr = vmf->address & PMD_MASK; 1489 struct vm_area_struct *vma = vmf->vma; 1490 pgprot_t pgprot = vma->vm_page_prot; 1491 struct folio_or_pfn fop = { 1492 .pfn = pfn, 1493 }; 1494 1495 /* 1496 * If we had pmd_special, we could avoid all these restrictions, 1497 * but we need to be consistent with PTEs and architectures that 1498 * can't support a 'special' bit. 1499 */ 1500 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 1501 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1502 (VM_PFNMAP|VM_MIXEDMAP)); 1503 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1504 1505 pfnmap_setup_cachemode_pfn(pfn, &pgprot); 1506 1507 return insert_pmd(vma, addr, vmf->pmd, fop, pgprot, write); 1508 } 1509 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd); 1510 1511 vm_fault_t vmf_insert_folio_pmd(struct vm_fault *vmf, struct folio *folio, 1512 bool write) 1513 { 1514 struct vm_area_struct *vma = vmf->vma; 1515 unsigned long addr = vmf->address & PMD_MASK; 1516 struct folio_or_pfn fop = { 1517 .folio = folio, 1518 .is_folio = true, 1519 }; 1520 1521 if (WARN_ON_ONCE(folio_order(folio) != PMD_ORDER)) 1522 return VM_FAULT_SIGBUS; 1523 1524 return insert_pmd(vma, addr, vmf->pmd, fop, vma->vm_page_prot, write); 1525 } 1526 EXPORT_SYMBOL_GPL(vmf_insert_folio_pmd); 1527 1528 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1529 static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma) 1530 { 1531 if (likely(vma->vm_flags & VM_WRITE)) 1532 pud = pud_mkwrite(pud); 1533 return pud; 1534 } 1535 1536 static vm_fault_t insert_pud(struct vm_area_struct *vma, unsigned long addr, 1537 pud_t *pud, struct folio_or_pfn fop, pgprot_t prot, bool write) 1538 { 1539 struct mm_struct *mm = vma->vm_mm; 1540 spinlock_t *ptl; 1541 pud_t entry; 1542 1543 if (addr < vma->vm_start || addr >= vma->vm_end) 1544 return VM_FAULT_SIGBUS; 1545 1546 ptl = pud_lock(mm, pud); 1547 if (!pud_none(*pud)) { 1548 const unsigned long pfn = fop.is_folio ? folio_pfn(fop.folio) : 1549 fop.pfn; 1550 1551 if (write) { 1552 if (WARN_ON_ONCE(pud_pfn(*pud) != pfn)) 1553 goto out_unlock; 1554 entry = pud_mkyoung(*pud); 1555 entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma); 1556 if (pudp_set_access_flags(vma, addr, pud, entry, 1)) 1557 update_mmu_cache_pud(vma, addr, pud); 1558 } 1559 goto out_unlock; 1560 } 1561 1562 if (fop.is_folio) { 1563 entry = folio_mk_pud(fop.folio, vma->vm_page_prot); 1564 1565 folio_get(fop.folio); 1566 folio_add_file_rmap_pud(fop.folio, &fop.folio->page, vma); 1567 add_mm_counter(mm, mm_counter_file(fop.folio), HPAGE_PUD_NR); 1568 } else { 1569 entry = pud_mkhuge(pfn_pud(fop.pfn, prot)); 1570 entry = pud_mkspecial(entry); 1571 } 1572 if (write) { 1573 entry = pud_mkyoung(pud_mkdirty(entry)); 1574 entry = maybe_pud_mkwrite(entry, vma); 1575 } 1576 set_pud_at(mm, addr, pud, entry); 1577 update_mmu_cache_pud(vma, addr, pud); 1578 out_unlock: 1579 spin_unlock(ptl); 1580 return VM_FAULT_NOPAGE; 1581 } 1582 1583 /** 1584 * vmf_insert_pfn_pud - insert a pud size pfn 1585 * @vmf: Structure describing the fault 1586 * @pfn: pfn to insert 1587 * @write: whether it's a write fault 1588 * 1589 * Insert a pud size pfn. See vmf_insert_pfn() for additional info. 1590 * 1591 * Return: vm_fault_t value. 1592 */ 1593 vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, unsigned long pfn, 1594 bool write) 1595 { 1596 unsigned long addr = vmf->address & PUD_MASK; 1597 struct vm_area_struct *vma = vmf->vma; 1598 pgprot_t pgprot = vma->vm_page_prot; 1599 struct folio_or_pfn fop = { 1600 .pfn = pfn, 1601 }; 1602 1603 /* 1604 * If we had pud_special, we could avoid all these restrictions, 1605 * but we need to be consistent with PTEs and architectures that 1606 * can't support a 'special' bit. 1607 */ 1608 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 1609 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1610 (VM_PFNMAP|VM_MIXEDMAP)); 1611 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1612 1613 pfnmap_setup_cachemode_pfn(pfn, &pgprot); 1614 1615 return insert_pud(vma, addr, vmf->pud, fop, pgprot, write); 1616 } 1617 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud); 1618 1619 /** 1620 * vmf_insert_folio_pud - insert a pud size folio mapped by a pud entry 1621 * @vmf: Structure describing the fault 1622 * @folio: folio to insert 1623 * @write: whether it's a write fault 1624 * 1625 * Return: vm_fault_t value. 1626 */ 1627 vm_fault_t vmf_insert_folio_pud(struct vm_fault *vmf, struct folio *folio, 1628 bool write) 1629 { 1630 struct vm_area_struct *vma = vmf->vma; 1631 unsigned long addr = vmf->address & PUD_MASK; 1632 struct folio_or_pfn fop = { 1633 .folio = folio, 1634 .is_folio = true, 1635 }; 1636 1637 if (WARN_ON_ONCE(folio_order(folio) != PUD_ORDER)) 1638 return VM_FAULT_SIGBUS; 1639 1640 return insert_pud(vma, addr, vmf->pud, fop, vma->vm_page_prot, write); 1641 } 1642 EXPORT_SYMBOL_GPL(vmf_insert_folio_pud); 1643 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1644 1645 void touch_pmd(struct vm_area_struct *vma, unsigned long addr, 1646 pmd_t *pmd, bool write) 1647 { 1648 pmd_t _pmd; 1649 1650 _pmd = pmd_mkyoung(*pmd); 1651 if (write) 1652 _pmd = pmd_mkdirty(_pmd); 1653 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, 1654 pmd, _pmd, write)) 1655 update_mmu_cache_pmd(vma, addr, pmd); 1656 } 1657 1658 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1659 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 1660 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) 1661 { 1662 spinlock_t *dst_ptl, *src_ptl; 1663 struct page *src_page; 1664 struct folio *src_folio; 1665 pmd_t pmd; 1666 pgtable_t pgtable = NULL; 1667 int ret = -ENOMEM; 1668 1669 pmd = pmdp_get_lockless(src_pmd); 1670 if (unlikely(pmd_present(pmd) && pmd_special(pmd) && 1671 !is_huge_zero_pmd(pmd))) { 1672 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1673 src_ptl = pmd_lockptr(src_mm, src_pmd); 1674 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1675 /* 1676 * No need to recheck the pmd, it can't change with write 1677 * mmap lock held here. 1678 * 1679 * Meanwhile, making sure it's not a CoW VMA with writable 1680 * mapping, otherwise it means either the anon page wrongly 1681 * applied special bit, or we made the PRIVATE mapping be 1682 * able to wrongly write to the backend MMIO. 1683 */ 1684 VM_WARN_ON_ONCE(is_cow_mapping(src_vma->vm_flags) && pmd_write(pmd)); 1685 goto set_pmd; 1686 } 1687 1688 /* Skip if can be re-fill on fault */ 1689 if (!vma_is_anonymous(dst_vma)) 1690 return 0; 1691 1692 pgtable = pte_alloc_one(dst_mm); 1693 if (unlikely(!pgtable)) 1694 goto out; 1695 1696 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1697 src_ptl = pmd_lockptr(src_mm, src_pmd); 1698 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1699 1700 ret = -EAGAIN; 1701 pmd = *src_pmd; 1702 1703 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 1704 if (unlikely(is_swap_pmd(pmd))) { 1705 swp_entry_t entry = pmd_to_swp_entry(pmd); 1706 1707 VM_BUG_ON(!is_pmd_migration_entry(pmd)); 1708 if (!is_readable_migration_entry(entry)) { 1709 entry = make_readable_migration_entry( 1710 swp_offset(entry)); 1711 pmd = swp_entry_to_pmd(entry); 1712 if (pmd_swp_soft_dirty(*src_pmd)) 1713 pmd = pmd_swp_mksoft_dirty(pmd); 1714 if (pmd_swp_uffd_wp(*src_pmd)) 1715 pmd = pmd_swp_mkuffd_wp(pmd); 1716 set_pmd_at(src_mm, addr, src_pmd, pmd); 1717 } 1718 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1719 mm_inc_nr_ptes(dst_mm); 1720 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1721 if (!userfaultfd_wp(dst_vma)) 1722 pmd = pmd_swp_clear_uffd_wp(pmd); 1723 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1724 ret = 0; 1725 goto out_unlock; 1726 } 1727 #endif 1728 1729 if (unlikely(!pmd_trans_huge(pmd))) { 1730 pte_free(dst_mm, pgtable); 1731 goto out_unlock; 1732 } 1733 /* 1734 * When page table lock is held, the huge zero pmd should not be 1735 * under splitting since we don't split the page itself, only pmd to 1736 * a page table. 1737 */ 1738 if (is_huge_zero_pmd(pmd)) { 1739 /* 1740 * mm_get_huge_zero_folio() will never allocate a new 1741 * folio here, since we already have a zero page to 1742 * copy. It just takes a reference. 1743 */ 1744 mm_get_huge_zero_folio(dst_mm); 1745 goto out_zero_page; 1746 } 1747 1748 src_page = pmd_page(pmd); 1749 VM_BUG_ON_PAGE(!PageHead(src_page), src_page); 1750 src_folio = page_folio(src_page); 1751 1752 folio_get(src_folio); 1753 if (unlikely(folio_try_dup_anon_rmap_pmd(src_folio, src_page, dst_vma, src_vma))) { 1754 /* Page maybe pinned: split and retry the fault on PTEs. */ 1755 folio_put(src_folio); 1756 pte_free(dst_mm, pgtable); 1757 spin_unlock(src_ptl); 1758 spin_unlock(dst_ptl); 1759 __split_huge_pmd(src_vma, src_pmd, addr, false); 1760 return -EAGAIN; 1761 } 1762 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1763 out_zero_page: 1764 mm_inc_nr_ptes(dst_mm); 1765 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1766 pmdp_set_wrprotect(src_mm, addr, src_pmd); 1767 if (!userfaultfd_wp(dst_vma)) 1768 pmd = pmd_clear_uffd_wp(pmd); 1769 pmd = pmd_wrprotect(pmd); 1770 set_pmd: 1771 pmd = pmd_mkold(pmd); 1772 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1773 1774 ret = 0; 1775 out_unlock: 1776 spin_unlock(src_ptl); 1777 spin_unlock(dst_ptl); 1778 out: 1779 return ret; 1780 } 1781 1782 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1783 void touch_pud(struct vm_area_struct *vma, unsigned long addr, 1784 pud_t *pud, bool write) 1785 { 1786 pud_t _pud; 1787 1788 _pud = pud_mkyoung(*pud); 1789 if (write) 1790 _pud = pud_mkdirty(_pud); 1791 if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK, 1792 pud, _pud, write)) 1793 update_mmu_cache_pud(vma, addr, pud); 1794 } 1795 1796 int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1797 pud_t *dst_pud, pud_t *src_pud, unsigned long addr, 1798 struct vm_area_struct *vma) 1799 { 1800 spinlock_t *dst_ptl, *src_ptl; 1801 pud_t pud; 1802 int ret; 1803 1804 dst_ptl = pud_lock(dst_mm, dst_pud); 1805 src_ptl = pud_lockptr(src_mm, src_pud); 1806 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1807 1808 ret = -EAGAIN; 1809 pud = *src_pud; 1810 if (unlikely(!pud_trans_huge(pud))) 1811 goto out_unlock; 1812 1813 /* 1814 * TODO: once we support anonymous pages, use 1815 * folio_try_dup_anon_rmap_*() and split if duplicating fails. 1816 */ 1817 if (is_cow_mapping(vma->vm_flags) && pud_write(pud)) { 1818 pudp_set_wrprotect(src_mm, addr, src_pud); 1819 pud = pud_wrprotect(pud); 1820 } 1821 pud = pud_mkold(pud); 1822 set_pud_at(dst_mm, addr, dst_pud, pud); 1823 1824 ret = 0; 1825 out_unlock: 1826 spin_unlock(src_ptl); 1827 spin_unlock(dst_ptl); 1828 return ret; 1829 } 1830 1831 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud) 1832 { 1833 bool write = vmf->flags & FAULT_FLAG_WRITE; 1834 1835 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); 1836 if (unlikely(!pud_same(*vmf->pud, orig_pud))) 1837 goto unlock; 1838 1839 touch_pud(vmf->vma, vmf->address, vmf->pud, write); 1840 unlock: 1841 spin_unlock(vmf->ptl); 1842 } 1843 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1844 1845 void huge_pmd_set_accessed(struct vm_fault *vmf) 1846 { 1847 bool write = vmf->flags & FAULT_FLAG_WRITE; 1848 1849 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); 1850 if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd))) 1851 goto unlock; 1852 1853 touch_pmd(vmf->vma, vmf->address, vmf->pmd, write); 1854 1855 unlock: 1856 spin_unlock(vmf->ptl); 1857 } 1858 1859 static vm_fault_t do_huge_zero_wp_pmd(struct vm_fault *vmf) 1860 { 1861 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1862 struct vm_area_struct *vma = vmf->vma; 1863 struct mmu_notifier_range range; 1864 struct folio *folio; 1865 vm_fault_t ret = 0; 1866 1867 folio = vma_alloc_anon_folio_pmd(vma, vmf->address); 1868 if (unlikely(!folio)) 1869 return VM_FAULT_FALLBACK; 1870 1871 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, haddr, 1872 haddr + HPAGE_PMD_SIZE); 1873 mmu_notifier_invalidate_range_start(&range); 1874 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1875 if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) 1876 goto release; 1877 ret = check_stable_address_space(vma->vm_mm); 1878 if (ret) 1879 goto release; 1880 (void)pmdp_huge_clear_flush(vma, haddr, vmf->pmd); 1881 map_anon_folio_pmd(folio, vmf->pmd, vma, haddr); 1882 goto unlock; 1883 release: 1884 folio_put(folio); 1885 unlock: 1886 spin_unlock(vmf->ptl); 1887 mmu_notifier_invalidate_range_end(&range); 1888 return ret; 1889 } 1890 1891 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) 1892 { 1893 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; 1894 struct vm_area_struct *vma = vmf->vma; 1895 struct folio *folio; 1896 struct page *page; 1897 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1898 pmd_t orig_pmd = vmf->orig_pmd; 1899 1900 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); 1901 VM_BUG_ON_VMA(!vma->anon_vma, vma); 1902 1903 if (is_huge_zero_pmd(orig_pmd)) { 1904 vm_fault_t ret = do_huge_zero_wp_pmd(vmf); 1905 1906 if (!(ret & VM_FAULT_FALLBACK)) 1907 return ret; 1908 1909 /* Fallback to splitting PMD if THP cannot be allocated */ 1910 goto fallback; 1911 } 1912 1913 spin_lock(vmf->ptl); 1914 1915 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1916 spin_unlock(vmf->ptl); 1917 return 0; 1918 } 1919 1920 page = pmd_page(orig_pmd); 1921 folio = page_folio(page); 1922 VM_BUG_ON_PAGE(!PageHead(page), page); 1923 1924 /* Early check when only holding the PT lock. */ 1925 if (PageAnonExclusive(page)) 1926 goto reuse; 1927 1928 if (!folio_trylock(folio)) { 1929 folio_get(folio); 1930 spin_unlock(vmf->ptl); 1931 folio_lock(folio); 1932 spin_lock(vmf->ptl); 1933 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1934 spin_unlock(vmf->ptl); 1935 folio_unlock(folio); 1936 folio_put(folio); 1937 return 0; 1938 } 1939 folio_put(folio); 1940 } 1941 1942 /* Recheck after temporarily dropping the PT lock. */ 1943 if (PageAnonExclusive(page)) { 1944 folio_unlock(folio); 1945 goto reuse; 1946 } 1947 1948 /* 1949 * See do_wp_page(): we can only reuse the folio exclusively if 1950 * there are no additional references. Note that we always drain 1951 * the LRU cache immediately after adding a THP. 1952 */ 1953 if (folio_ref_count(folio) > 1954 1 + folio_test_swapcache(folio) * folio_nr_pages(folio)) 1955 goto unlock_fallback; 1956 if (folio_test_swapcache(folio)) 1957 folio_free_swap(folio); 1958 if (folio_ref_count(folio) == 1) { 1959 pmd_t entry; 1960 1961 folio_move_anon_rmap(folio, vma); 1962 SetPageAnonExclusive(page); 1963 folio_unlock(folio); 1964 reuse: 1965 if (unlikely(unshare)) { 1966 spin_unlock(vmf->ptl); 1967 return 0; 1968 } 1969 entry = pmd_mkyoung(orig_pmd); 1970 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1971 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) 1972 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1973 spin_unlock(vmf->ptl); 1974 return 0; 1975 } 1976 1977 unlock_fallback: 1978 folio_unlock(folio); 1979 spin_unlock(vmf->ptl); 1980 fallback: 1981 __split_huge_pmd(vma, vmf->pmd, vmf->address, false); 1982 return VM_FAULT_FALLBACK; 1983 } 1984 1985 static inline bool can_change_pmd_writable(struct vm_area_struct *vma, 1986 unsigned long addr, pmd_t pmd) 1987 { 1988 struct page *page; 1989 1990 if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) 1991 return false; 1992 1993 /* Don't touch entries that are not even readable (NUMA hinting). */ 1994 if (pmd_protnone(pmd)) 1995 return false; 1996 1997 /* Do we need write faults for softdirty tracking? */ 1998 if (pmd_needs_soft_dirty_wp(vma, pmd)) 1999 return false; 2000 2001 /* Do we need write faults for uffd-wp tracking? */ 2002 if (userfaultfd_huge_pmd_wp(vma, pmd)) 2003 return false; 2004 2005 if (!(vma->vm_flags & VM_SHARED)) { 2006 /* See can_change_pte_writable(). */ 2007 page = vm_normal_page_pmd(vma, addr, pmd); 2008 return page && PageAnon(page) && PageAnonExclusive(page); 2009 } 2010 2011 /* See can_change_pte_writable(). */ 2012 return pmd_dirty(pmd); 2013 } 2014 2015 /* NUMA hinting page fault entry point for trans huge pmds */ 2016 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) 2017 { 2018 struct vm_area_struct *vma = vmf->vma; 2019 struct folio *folio; 2020 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 2021 int nid = NUMA_NO_NODE; 2022 int target_nid, last_cpupid; 2023 pmd_t pmd, old_pmd; 2024 bool writable = false; 2025 int flags = 0; 2026 2027 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 2028 old_pmd = pmdp_get(vmf->pmd); 2029 2030 if (unlikely(!pmd_same(old_pmd, vmf->orig_pmd))) { 2031 spin_unlock(vmf->ptl); 2032 return 0; 2033 } 2034 2035 pmd = pmd_modify(old_pmd, vma->vm_page_prot); 2036 2037 /* 2038 * Detect now whether the PMD could be writable; this information 2039 * is only valid while holding the PT lock. 2040 */ 2041 writable = pmd_write(pmd); 2042 if (!writable && vma_wants_manual_pte_write_upgrade(vma) && 2043 can_change_pmd_writable(vma, vmf->address, pmd)) 2044 writable = true; 2045 2046 folio = vm_normal_folio_pmd(vma, haddr, pmd); 2047 if (!folio) 2048 goto out_map; 2049 2050 nid = folio_nid(folio); 2051 2052 target_nid = numa_migrate_check(folio, vmf, haddr, &flags, writable, 2053 &last_cpupid); 2054 if (target_nid == NUMA_NO_NODE) 2055 goto out_map; 2056 if (migrate_misplaced_folio_prepare(folio, vma, target_nid)) { 2057 flags |= TNF_MIGRATE_FAIL; 2058 goto out_map; 2059 } 2060 /* The folio is isolated and isolation code holds a folio reference. */ 2061 spin_unlock(vmf->ptl); 2062 writable = false; 2063 2064 if (!migrate_misplaced_folio(folio, target_nid)) { 2065 flags |= TNF_MIGRATED; 2066 nid = target_nid; 2067 task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags); 2068 return 0; 2069 } 2070 2071 flags |= TNF_MIGRATE_FAIL; 2072 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 2073 if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) { 2074 spin_unlock(vmf->ptl); 2075 return 0; 2076 } 2077 out_map: 2078 /* Restore the PMD */ 2079 pmd = pmd_modify(pmdp_get(vmf->pmd), vma->vm_page_prot); 2080 pmd = pmd_mkyoung(pmd); 2081 if (writable) 2082 pmd = pmd_mkwrite(pmd, vma); 2083 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); 2084 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 2085 spin_unlock(vmf->ptl); 2086 2087 if (nid != NUMA_NO_NODE) 2088 task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags); 2089 return 0; 2090 } 2091 2092 /* 2093 * Return true if we do MADV_FREE successfully on entire pmd page. 2094 * Otherwise, return false. 2095 */ 2096 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2097 pmd_t *pmd, unsigned long addr, unsigned long next) 2098 { 2099 spinlock_t *ptl; 2100 pmd_t orig_pmd; 2101 struct folio *folio; 2102 struct mm_struct *mm = tlb->mm; 2103 bool ret = false; 2104 2105 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2106 2107 ptl = pmd_trans_huge_lock(pmd, vma); 2108 if (!ptl) 2109 goto out_unlocked; 2110 2111 orig_pmd = *pmd; 2112 if (is_huge_zero_pmd(orig_pmd)) 2113 goto out; 2114 2115 if (unlikely(!pmd_present(orig_pmd))) { 2116 VM_BUG_ON(thp_migration_supported() && 2117 !is_pmd_migration_entry(orig_pmd)); 2118 goto out; 2119 } 2120 2121 folio = pmd_folio(orig_pmd); 2122 /* 2123 * If other processes are mapping this folio, we couldn't discard 2124 * the folio unless they all do MADV_FREE so let's skip the folio. 2125 */ 2126 if (folio_maybe_mapped_shared(folio)) 2127 goto out; 2128 2129 if (!folio_trylock(folio)) 2130 goto out; 2131 2132 /* 2133 * If user want to discard part-pages of THP, split it so MADV_FREE 2134 * will deactivate only them. 2135 */ 2136 if (next - addr != HPAGE_PMD_SIZE) { 2137 folio_get(folio); 2138 spin_unlock(ptl); 2139 split_folio(folio); 2140 folio_unlock(folio); 2141 folio_put(folio); 2142 goto out_unlocked; 2143 } 2144 2145 if (folio_test_dirty(folio)) 2146 folio_clear_dirty(folio); 2147 folio_unlock(folio); 2148 2149 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) { 2150 pmdp_invalidate(vma, addr, pmd); 2151 orig_pmd = pmd_mkold(orig_pmd); 2152 orig_pmd = pmd_mkclean(orig_pmd); 2153 2154 set_pmd_at(mm, addr, pmd, orig_pmd); 2155 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 2156 } 2157 2158 folio_mark_lazyfree(folio); 2159 ret = true; 2160 out: 2161 spin_unlock(ptl); 2162 out_unlocked: 2163 return ret; 2164 } 2165 2166 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd) 2167 { 2168 pgtable_t pgtable; 2169 2170 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2171 pte_free(mm, pgtable); 2172 mm_dec_nr_ptes(mm); 2173 } 2174 2175 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2176 pmd_t *pmd, unsigned long addr) 2177 { 2178 pmd_t orig_pmd; 2179 spinlock_t *ptl; 2180 2181 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2182 2183 ptl = __pmd_trans_huge_lock(pmd, vma); 2184 if (!ptl) 2185 return 0; 2186 /* 2187 * For architectures like ppc64 we look at deposited pgtable 2188 * when calling pmdp_huge_get_and_clear. So do the 2189 * pgtable_trans_huge_withdraw after finishing pmdp related 2190 * operations. 2191 */ 2192 orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd, 2193 tlb->fullmm); 2194 arch_check_zapped_pmd(vma, orig_pmd); 2195 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 2196 if (!vma_is_dax(vma) && vma_is_special_huge(vma)) { 2197 if (arch_needs_pgtable_deposit()) 2198 zap_deposited_table(tlb->mm, pmd); 2199 spin_unlock(ptl); 2200 } else if (is_huge_zero_pmd(orig_pmd)) { 2201 if (!vma_is_dax(vma) || arch_needs_pgtable_deposit()) 2202 zap_deposited_table(tlb->mm, pmd); 2203 spin_unlock(ptl); 2204 } else { 2205 struct folio *folio = NULL; 2206 int flush_needed = 1; 2207 2208 if (pmd_present(orig_pmd)) { 2209 struct page *page = pmd_page(orig_pmd); 2210 2211 folio = page_folio(page); 2212 folio_remove_rmap_pmd(folio, page, vma); 2213 WARN_ON_ONCE(folio_mapcount(folio) < 0); 2214 VM_BUG_ON_PAGE(!PageHead(page), page); 2215 } else if (thp_migration_supported()) { 2216 swp_entry_t entry; 2217 2218 VM_BUG_ON(!is_pmd_migration_entry(orig_pmd)); 2219 entry = pmd_to_swp_entry(orig_pmd); 2220 folio = pfn_swap_entry_folio(entry); 2221 flush_needed = 0; 2222 } else 2223 WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!"); 2224 2225 if (folio_test_anon(folio)) { 2226 zap_deposited_table(tlb->mm, pmd); 2227 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); 2228 } else { 2229 if (arch_needs_pgtable_deposit()) 2230 zap_deposited_table(tlb->mm, pmd); 2231 add_mm_counter(tlb->mm, mm_counter_file(folio), 2232 -HPAGE_PMD_NR); 2233 2234 /* 2235 * Use flush_needed to indicate whether the PMD entry 2236 * is present, instead of checking pmd_present() again. 2237 */ 2238 if (flush_needed && pmd_young(orig_pmd) && 2239 likely(vma_has_recency(vma))) 2240 folio_mark_accessed(folio); 2241 } 2242 2243 spin_unlock(ptl); 2244 if (flush_needed) 2245 tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE); 2246 } 2247 return 1; 2248 } 2249 2250 #ifndef pmd_move_must_withdraw 2251 static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl, 2252 spinlock_t *old_pmd_ptl, 2253 struct vm_area_struct *vma) 2254 { 2255 /* 2256 * With split pmd lock we also need to move preallocated 2257 * PTE page table if new_pmd is on different PMD page table. 2258 * 2259 * We also don't deposit and withdraw tables for file pages. 2260 */ 2261 return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma); 2262 } 2263 #endif 2264 2265 static pmd_t move_soft_dirty_pmd(pmd_t pmd) 2266 { 2267 #ifdef CONFIG_MEM_SOFT_DIRTY 2268 if (unlikely(is_pmd_migration_entry(pmd))) 2269 pmd = pmd_swp_mksoft_dirty(pmd); 2270 else if (pmd_present(pmd)) 2271 pmd = pmd_mksoft_dirty(pmd); 2272 #endif 2273 return pmd; 2274 } 2275 2276 static pmd_t clear_uffd_wp_pmd(pmd_t pmd) 2277 { 2278 if (pmd_present(pmd)) 2279 pmd = pmd_clear_uffd_wp(pmd); 2280 else if (is_swap_pmd(pmd)) 2281 pmd = pmd_swp_clear_uffd_wp(pmd); 2282 2283 return pmd; 2284 } 2285 2286 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, 2287 unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd) 2288 { 2289 spinlock_t *old_ptl, *new_ptl; 2290 pmd_t pmd; 2291 struct mm_struct *mm = vma->vm_mm; 2292 bool force_flush = false; 2293 2294 /* 2295 * The destination pmd shouldn't be established, free_pgtables() 2296 * should have released it; but move_page_tables() might have already 2297 * inserted a page table, if racing against shmem/file collapse. 2298 */ 2299 if (!pmd_none(*new_pmd)) { 2300 VM_BUG_ON(pmd_trans_huge(*new_pmd)); 2301 return false; 2302 } 2303 2304 /* 2305 * We don't have to worry about the ordering of src and dst 2306 * ptlocks because exclusive mmap_lock prevents deadlock. 2307 */ 2308 old_ptl = __pmd_trans_huge_lock(old_pmd, vma); 2309 if (old_ptl) { 2310 new_ptl = pmd_lockptr(mm, new_pmd); 2311 if (new_ptl != old_ptl) 2312 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 2313 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); 2314 if (pmd_present(pmd)) 2315 force_flush = true; 2316 VM_BUG_ON(!pmd_none(*new_pmd)); 2317 2318 if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) { 2319 pgtable_t pgtable; 2320 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); 2321 pgtable_trans_huge_deposit(mm, new_pmd, pgtable); 2322 } 2323 pmd = move_soft_dirty_pmd(pmd); 2324 if (vma_has_uffd_without_event_remap(vma)) 2325 pmd = clear_uffd_wp_pmd(pmd); 2326 set_pmd_at(mm, new_addr, new_pmd, pmd); 2327 if (force_flush) 2328 flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE); 2329 if (new_ptl != old_ptl) 2330 spin_unlock(new_ptl); 2331 spin_unlock(old_ptl); 2332 return true; 2333 } 2334 return false; 2335 } 2336 2337 /* 2338 * Returns 2339 * - 0 if PMD could not be locked 2340 * - 1 if PMD was locked but protections unchanged and TLB flush unnecessary 2341 * or if prot_numa but THP migration is not supported 2342 * - HPAGE_PMD_NR if protections changed and TLB flush necessary 2343 */ 2344 int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2345 pmd_t *pmd, unsigned long addr, pgprot_t newprot, 2346 unsigned long cp_flags) 2347 { 2348 struct mm_struct *mm = vma->vm_mm; 2349 spinlock_t *ptl; 2350 pmd_t oldpmd, entry; 2351 bool prot_numa = cp_flags & MM_CP_PROT_NUMA; 2352 bool uffd_wp = cp_flags & MM_CP_UFFD_WP; 2353 bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; 2354 int ret = 1; 2355 2356 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2357 2358 if (prot_numa && !thp_migration_supported()) 2359 return 1; 2360 2361 ptl = __pmd_trans_huge_lock(pmd, vma); 2362 if (!ptl) 2363 return 0; 2364 2365 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 2366 if (is_swap_pmd(*pmd)) { 2367 swp_entry_t entry = pmd_to_swp_entry(*pmd); 2368 struct folio *folio = pfn_swap_entry_folio(entry); 2369 pmd_t newpmd; 2370 2371 VM_BUG_ON(!is_pmd_migration_entry(*pmd)); 2372 if (is_writable_migration_entry(entry)) { 2373 /* 2374 * A protection check is difficult so 2375 * just be safe and disable write 2376 */ 2377 if (folio_test_anon(folio)) 2378 entry = make_readable_exclusive_migration_entry(swp_offset(entry)); 2379 else 2380 entry = make_readable_migration_entry(swp_offset(entry)); 2381 newpmd = swp_entry_to_pmd(entry); 2382 if (pmd_swp_soft_dirty(*pmd)) 2383 newpmd = pmd_swp_mksoft_dirty(newpmd); 2384 } else { 2385 newpmd = *pmd; 2386 } 2387 2388 if (uffd_wp) 2389 newpmd = pmd_swp_mkuffd_wp(newpmd); 2390 else if (uffd_wp_resolve) 2391 newpmd = pmd_swp_clear_uffd_wp(newpmd); 2392 if (!pmd_same(*pmd, newpmd)) 2393 set_pmd_at(mm, addr, pmd, newpmd); 2394 goto unlock; 2395 } 2396 #endif 2397 2398 if (prot_numa) { 2399 struct folio *folio; 2400 bool toptier; 2401 /* 2402 * Avoid trapping faults against the zero page. The read-only 2403 * data is likely to be read-cached on the local CPU and 2404 * local/remote hits to the zero page are not interesting. 2405 */ 2406 if (is_huge_zero_pmd(*pmd)) 2407 goto unlock; 2408 2409 if (pmd_protnone(*pmd)) 2410 goto unlock; 2411 2412 folio = pmd_folio(*pmd); 2413 toptier = node_is_toptier(folio_nid(folio)); 2414 /* 2415 * Skip scanning top tier node if normal numa 2416 * balancing is disabled 2417 */ 2418 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) && 2419 toptier) 2420 goto unlock; 2421 2422 if (folio_use_access_time(folio)) 2423 folio_xchg_access_time(folio, 2424 jiffies_to_msecs(jiffies)); 2425 } 2426 /* 2427 * In case prot_numa, we are under mmap_read_lock(mm). It's critical 2428 * to not clear pmd intermittently to avoid race with MADV_DONTNEED 2429 * which is also under mmap_read_lock(mm): 2430 * 2431 * CPU0: CPU1: 2432 * change_huge_pmd(prot_numa=1) 2433 * pmdp_huge_get_and_clear_notify() 2434 * madvise_dontneed() 2435 * zap_pmd_range() 2436 * pmd_trans_huge(*pmd) == 0 (without ptl) 2437 * // skip the pmd 2438 * set_pmd_at(); 2439 * // pmd is re-established 2440 * 2441 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it 2442 * which may break userspace. 2443 * 2444 * pmdp_invalidate_ad() is required to make sure we don't miss 2445 * dirty/young flags set by hardware. 2446 */ 2447 oldpmd = pmdp_invalidate_ad(vma, addr, pmd); 2448 2449 entry = pmd_modify(oldpmd, newprot); 2450 if (uffd_wp) 2451 entry = pmd_mkuffd_wp(entry); 2452 else if (uffd_wp_resolve) 2453 /* 2454 * Leave the write bit to be handled by PF interrupt 2455 * handler, then things like COW could be properly 2456 * handled. 2457 */ 2458 entry = pmd_clear_uffd_wp(entry); 2459 2460 /* See change_pte_range(). */ 2461 if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) && 2462 can_change_pmd_writable(vma, addr, entry)) 2463 entry = pmd_mkwrite(entry, vma); 2464 2465 ret = HPAGE_PMD_NR; 2466 set_pmd_at(mm, addr, pmd, entry); 2467 2468 if (huge_pmd_needs_flush(oldpmd, entry)) 2469 tlb_flush_pmd_range(tlb, addr, HPAGE_PMD_SIZE); 2470 unlock: 2471 spin_unlock(ptl); 2472 return ret; 2473 } 2474 2475 /* 2476 * Returns: 2477 * 2478 * - 0: if pud leaf changed from under us 2479 * - 1: if pud can be skipped 2480 * - HPAGE_PUD_NR: if pud was successfully processed 2481 */ 2482 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2483 int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 2484 pud_t *pudp, unsigned long addr, pgprot_t newprot, 2485 unsigned long cp_flags) 2486 { 2487 struct mm_struct *mm = vma->vm_mm; 2488 pud_t oldpud, entry; 2489 spinlock_t *ptl; 2490 2491 tlb_change_page_size(tlb, HPAGE_PUD_SIZE); 2492 2493 /* NUMA balancing doesn't apply to dax */ 2494 if (cp_flags & MM_CP_PROT_NUMA) 2495 return 1; 2496 2497 /* 2498 * Huge entries on userfault-wp only works with anonymous, while we 2499 * don't have anonymous PUDs yet. 2500 */ 2501 if (WARN_ON_ONCE(cp_flags & MM_CP_UFFD_WP_ALL)) 2502 return 1; 2503 2504 ptl = __pud_trans_huge_lock(pudp, vma); 2505 if (!ptl) 2506 return 0; 2507 2508 /* 2509 * Can't clear PUD or it can race with concurrent zapping. See 2510 * change_huge_pmd(). 2511 */ 2512 oldpud = pudp_invalidate(vma, addr, pudp); 2513 entry = pud_modify(oldpud, newprot); 2514 set_pud_at(mm, addr, pudp, entry); 2515 tlb_flush_pud_range(tlb, addr, HPAGE_PUD_SIZE); 2516 2517 spin_unlock(ptl); 2518 return HPAGE_PUD_NR; 2519 } 2520 #endif 2521 2522 #ifdef CONFIG_USERFAULTFD 2523 /* 2524 * The PT lock for src_pmd and dst_vma/src_vma (for reading) are locked by 2525 * the caller, but it must return after releasing the page_table_lock. 2526 * Just move the page from src_pmd to dst_pmd if possible. 2527 * Return zero if succeeded in moving the page, -EAGAIN if it needs to be 2528 * repeated by the caller, or other errors in case of failure. 2529 */ 2530 int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval, 2531 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, 2532 unsigned long dst_addr, unsigned long src_addr) 2533 { 2534 pmd_t _dst_pmd, src_pmdval; 2535 struct page *src_page; 2536 struct folio *src_folio; 2537 struct anon_vma *src_anon_vma; 2538 spinlock_t *src_ptl, *dst_ptl; 2539 pgtable_t src_pgtable; 2540 struct mmu_notifier_range range; 2541 int err = 0; 2542 2543 src_pmdval = *src_pmd; 2544 src_ptl = pmd_lockptr(mm, src_pmd); 2545 2546 lockdep_assert_held(src_ptl); 2547 vma_assert_locked(src_vma); 2548 vma_assert_locked(dst_vma); 2549 2550 /* Sanity checks before the operation */ 2551 if (WARN_ON_ONCE(!pmd_none(dst_pmdval)) || WARN_ON_ONCE(src_addr & ~HPAGE_PMD_MASK) || 2552 WARN_ON_ONCE(dst_addr & ~HPAGE_PMD_MASK)) { 2553 spin_unlock(src_ptl); 2554 return -EINVAL; 2555 } 2556 2557 if (!pmd_trans_huge(src_pmdval)) { 2558 spin_unlock(src_ptl); 2559 if (is_pmd_migration_entry(src_pmdval)) { 2560 pmd_migration_entry_wait(mm, &src_pmdval); 2561 return -EAGAIN; 2562 } 2563 return -ENOENT; 2564 } 2565 2566 src_page = pmd_page(src_pmdval); 2567 2568 if (!is_huge_zero_pmd(src_pmdval)) { 2569 if (unlikely(!PageAnonExclusive(src_page))) { 2570 spin_unlock(src_ptl); 2571 return -EBUSY; 2572 } 2573 2574 src_folio = page_folio(src_page); 2575 folio_get(src_folio); 2576 } else 2577 src_folio = NULL; 2578 2579 spin_unlock(src_ptl); 2580 2581 flush_cache_range(src_vma, src_addr, src_addr + HPAGE_PMD_SIZE); 2582 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, src_addr, 2583 src_addr + HPAGE_PMD_SIZE); 2584 mmu_notifier_invalidate_range_start(&range); 2585 2586 if (src_folio) { 2587 folio_lock(src_folio); 2588 2589 /* 2590 * split_huge_page walks the anon_vma chain without the page 2591 * lock. Serialize against it with the anon_vma lock, the page 2592 * lock is not enough. 2593 */ 2594 src_anon_vma = folio_get_anon_vma(src_folio); 2595 if (!src_anon_vma) { 2596 err = -EAGAIN; 2597 goto unlock_folio; 2598 } 2599 anon_vma_lock_write(src_anon_vma); 2600 } else 2601 src_anon_vma = NULL; 2602 2603 dst_ptl = pmd_lockptr(mm, dst_pmd); 2604 double_pt_lock(src_ptl, dst_ptl); 2605 if (unlikely(!pmd_same(*src_pmd, src_pmdval) || 2606 !pmd_same(*dst_pmd, dst_pmdval))) { 2607 err = -EAGAIN; 2608 goto unlock_ptls; 2609 } 2610 if (src_folio) { 2611 if (folio_maybe_dma_pinned(src_folio) || 2612 !PageAnonExclusive(&src_folio->page)) { 2613 err = -EBUSY; 2614 goto unlock_ptls; 2615 } 2616 2617 if (WARN_ON_ONCE(!folio_test_head(src_folio)) || 2618 WARN_ON_ONCE(!folio_test_anon(src_folio))) { 2619 err = -EBUSY; 2620 goto unlock_ptls; 2621 } 2622 2623 src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); 2624 /* Folio got pinned from under us. Put it back and fail the move. */ 2625 if (folio_maybe_dma_pinned(src_folio)) { 2626 set_pmd_at(mm, src_addr, src_pmd, src_pmdval); 2627 err = -EBUSY; 2628 goto unlock_ptls; 2629 } 2630 2631 folio_move_anon_rmap(src_folio, dst_vma); 2632 src_folio->index = linear_page_index(dst_vma, dst_addr); 2633 2634 _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); 2635 /* Follow mremap() behavior and treat the entry dirty after the move */ 2636 _dst_pmd = pmd_mkwrite(pmd_mkdirty(_dst_pmd), dst_vma); 2637 } else { 2638 src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); 2639 _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); 2640 } 2641 set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd); 2642 2643 src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd); 2644 pgtable_trans_huge_deposit(mm, dst_pmd, src_pgtable); 2645 unlock_ptls: 2646 double_pt_unlock(src_ptl, dst_ptl); 2647 if (src_anon_vma) { 2648 anon_vma_unlock_write(src_anon_vma); 2649 put_anon_vma(src_anon_vma); 2650 } 2651 unlock_folio: 2652 /* unblock rmap walks */ 2653 if (src_folio) 2654 folio_unlock(src_folio); 2655 mmu_notifier_invalidate_range_end(&range); 2656 if (src_folio) 2657 folio_put(src_folio); 2658 return err; 2659 } 2660 #endif /* CONFIG_USERFAULTFD */ 2661 2662 /* 2663 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise. 2664 * 2665 * Note that if it returns page table lock pointer, this routine returns without 2666 * unlocking page table lock. So callers must unlock it. 2667 */ 2668 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) 2669 { 2670 spinlock_t *ptl; 2671 ptl = pmd_lock(vma->vm_mm, pmd); 2672 if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd))) 2673 return ptl; 2674 spin_unlock(ptl); 2675 return NULL; 2676 } 2677 2678 /* 2679 * Returns page table lock pointer if a given pud maps a thp, NULL otherwise. 2680 * 2681 * Note that if it returns page table lock pointer, this routine returns without 2682 * unlocking page table lock. So callers must unlock it. 2683 */ 2684 spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma) 2685 { 2686 spinlock_t *ptl; 2687 2688 ptl = pud_lock(vma->vm_mm, pud); 2689 if (likely(pud_trans_huge(*pud))) 2690 return ptl; 2691 spin_unlock(ptl); 2692 return NULL; 2693 } 2694 2695 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2696 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 2697 pud_t *pud, unsigned long addr) 2698 { 2699 spinlock_t *ptl; 2700 pud_t orig_pud; 2701 2702 ptl = __pud_trans_huge_lock(pud, vma); 2703 if (!ptl) 2704 return 0; 2705 2706 orig_pud = pudp_huge_get_and_clear_full(vma, addr, pud, tlb->fullmm); 2707 arch_check_zapped_pud(vma, orig_pud); 2708 tlb_remove_pud_tlb_entry(tlb, pud, addr); 2709 if (!vma_is_dax(vma) && vma_is_special_huge(vma)) { 2710 spin_unlock(ptl); 2711 /* No zero page support yet */ 2712 } else { 2713 struct page *page = NULL; 2714 struct folio *folio; 2715 2716 /* No support for anonymous PUD pages or migration yet */ 2717 VM_WARN_ON_ONCE(vma_is_anonymous(vma) || 2718 !pud_present(orig_pud)); 2719 2720 page = pud_page(orig_pud); 2721 folio = page_folio(page); 2722 folio_remove_rmap_pud(folio, page, vma); 2723 add_mm_counter(tlb->mm, mm_counter_file(folio), -HPAGE_PUD_NR); 2724 2725 spin_unlock(ptl); 2726 tlb_remove_page_size(tlb, page, HPAGE_PUD_SIZE); 2727 } 2728 return 1; 2729 } 2730 2731 static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud, 2732 unsigned long haddr) 2733 { 2734 struct folio *folio; 2735 struct page *page; 2736 pud_t old_pud; 2737 2738 VM_BUG_ON(haddr & ~HPAGE_PUD_MASK); 2739 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2740 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma); 2741 VM_BUG_ON(!pud_trans_huge(*pud)); 2742 2743 count_vm_event(THP_SPLIT_PUD); 2744 2745 old_pud = pudp_huge_clear_flush(vma, haddr, pud); 2746 2747 if (!vma_is_dax(vma)) 2748 return; 2749 2750 page = pud_page(old_pud); 2751 folio = page_folio(page); 2752 2753 if (!folio_test_dirty(folio) && pud_dirty(old_pud)) 2754 folio_mark_dirty(folio); 2755 if (!folio_test_referenced(folio) && pud_young(old_pud)) 2756 folio_set_referenced(folio); 2757 folio_remove_rmap_pud(folio, page, vma); 2758 folio_put(folio); 2759 add_mm_counter(vma->vm_mm, mm_counter_file(folio), 2760 -HPAGE_PUD_NR); 2761 } 2762 2763 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 2764 unsigned long address) 2765 { 2766 spinlock_t *ptl; 2767 struct mmu_notifier_range range; 2768 2769 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 2770 address & HPAGE_PUD_MASK, 2771 (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE); 2772 mmu_notifier_invalidate_range_start(&range); 2773 ptl = pud_lock(vma->vm_mm, pud); 2774 if (unlikely(!pud_trans_huge(*pud))) 2775 goto out; 2776 __split_huge_pud_locked(vma, pud, range.start); 2777 2778 out: 2779 spin_unlock(ptl); 2780 mmu_notifier_invalidate_range_end(&range); 2781 } 2782 #else 2783 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 2784 unsigned long address) 2785 { 2786 } 2787 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 2788 2789 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 2790 unsigned long haddr, pmd_t *pmd) 2791 { 2792 struct mm_struct *mm = vma->vm_mm; 2793 pgtable_t pgtable; 2794 pmd_t _pmd, old_pmd; 2795 unsigned long addr; 2796 pte_t *pte; 2797 int i; 2798 2799 /* 2800 * Leave pmd empty until pte is filled note that it is fine to delay 2801 * notification until mmu_notifier_invalidate_range_end() as we are 2802 * replacing a zero pmd write protected page with a zero pte write 2803 * protected page. 2804 * 2805 * See Documentation/mm/mmu_notifier.rst 2806 */ 2807 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); 2808 2809 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2810 pmd_populate(mm, &_pmd, pgtable); 2811 2812 pte = pte_offset_map(&_pmd, haddr); 2813 VM_BUG_ON(!pte); 2814 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2815 pte_t entry; 2816 2817 entry = pfn_pte(my_zero_pfn(addr), vma->vm_page_prot); 2818 entry = pte_mkspecial(entry); 2819 if (pmd_uffd_wp(old_pmd)) 2820 entry = pte_mkuffd_wp(entry); 2821 VM_BUG_ON(!pte_none(ptep_get(pte))); 2822 set_pte_at(mm, addr, pte, entry); 2823 pte++; 2824 } 2825 pte_unmap(pte - 1); 2826 smp_wmb(); /* make pte visible before pmd */ 2827 pmd_populate(mm, pmd, pgtable); 2828 } 2829 2830 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 2831 unsigned long haddr, bool freeze) 2832 { 2833 struct mm_struct *mm = vma->vm_mm; 2834 struct folio *folio; 2835 struct page *page; 2836 pgtable_t pgtable; 2837 pmd_t old_pmd, _pmd; 2838 bool young, write, soft_dirty, pmd_migration = false, uffd_wp = false; 2839 bool anon_exclusive = false, dirty = false; 2840 unsigned long addr; 2841 pte_t *pte; 2842 int i; 2843 2844 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 2845 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2846 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 2847 VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd)); 2848 2849 count_vm_event(THP_SPLIT_PMD); 2850 2851 if (!vma_is_anonymous(vma)) { 2852 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); 2853 /* 2854 * We are going to unmap this huge page. So 2855 * just go ahead and zap it 2856 */ 2857 if (arch_needs_pgtable_deposit()) 2858 zap_deposited_table(mm, pmd); 2859 if (!vma_is_dax(vma) && vma_is_special_huge(vma)) 2860 return; 2861 if (unlikely(is_pmd_migration_entry(old_pmd))) { 2862 swp_entry_t entry; 2863 2864 entry = pmd_to_swp_entry(old_pmd); 2865 folio = pfn_swap_entry_folio(entry); 2866 } else if (is_huge_zero_pmd(old_pmd)) { 2867 return; 2868 } else { 2869 page = pmd_page(old_pmd); 2870 folio = page_folio(page); 2871 if (!folio_test_dirty(folio) && pmd_dirty(old_pmd)) 2872 folio_mark_dirty(folio); 2873 if (!folio_test_referenced(folio) && pmd_young(old_pmd)) 2874 folio_set_referenced(folio); 2875 folio_remove_rmap_pmd(folio, page, vma); 2876 folio_put(folio); 2877 } 2878 add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR); 2879 return; 2880 } 2881 2882 if (is_huge_zero_pmd(*pmd)) { 2883 /* 2884 * FIXME: Do we want to invalidate secondary mmu by calling 2885 * mmu_notifier_arch_invalidate_secondary_tlbs() see comments below 2886 * inside __split_huge_pmd() ? 2887 * 2888 * We are going from a zero huge page write protected to zero 2889 * small page also write protected so it does not seems useful 2890 * to invalidate secondary mmu at this time. 2891 */ 2892 return __split_huge_zero_page_pmd(vma, haddr, pmd); 2893 } 2894 2895 pmd_migration = is_pmd_migration_entry(*pmd); 2896 if (unlikely(pmd_migration)) { 2897 swp_entry_t entry; 2898 2899 old_pmd = *pmd; 2900 entry = pmd_to_swp_entry(old_pmd); 2901 page = pfn_swap_entry_to_page(entry); 2902 write = is_writable_migration_entry(entry); 2903 if (PageAnon(page)) 2904 anon_exclusive = is_readable_exclusive_migration_entry(entry); 2905 young = is_migration_entry_young(entry); 2906 dirty = is_migration_entry_dirty(entry); 2907 soft_dirty = pmd_swp_soft_dirty(old_pmd); 2908 uffd_wp = pmd_swp_uffd_wp(old_pmd); 2909 } else { 2910 /* 2911 * Up to this point the pmd is present and huge and userland has 2912 * the whole access to the hugepage during the split (which 2913 * happens in place). If we overwrite the pmd with the not-huge 2914 * version pointing to the pte here (which of course we could if 2915 * all CPUs were bug free), userland could trigger a small page 2916 * size TLB miss on the small sized TLB while the hugepage TLB 2917 * entry is still established in the huge TLB. Some CPU doesn't 2918 * like that. See 2919 * http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum 2920 * 383 on page 105. Intel should be safe but is also warns that 2921 * it's only safe if the permission and cache attributes of the 2922 * two entries loaded in the two TLB is identical (which should 2923 * be the case here). But it is generally safer to never allow 2924 * small and huge TLB entries for the same virtual address to be 2925 * loaded simultaneously. So instead of doing "pmd_populate(); 2926 * flush_pmd_tlb_range();" we first mark the current pmd 2927 * notpresent (atomically because here the pmd_trans_huge must 2928 * remain set at all times on the pmd until the split is 2929 * complete for this pmd), then we flush the SMP TLB and finally 2930 * we write the non-huge version of the pmd entry with 2931 * pmd_populate. 2932 */ 2933 old_pmd = pmdp_invalidate(vma, haddr, pmd); 2934 page = pmd_page(old_pmd); 2935 folio = page_folio(page); 2936 if (pmd_dirty(old_pmd)) { 2937 dirty = true; 2938 folio_set_dirty(folio); 2939 } 2940 write = pmd_write(old_pmd); 2941 young = pmd_young(old_pmd); 2942 soft_dirty = pmd_soft_dirty(old_pmd); 2943 uffd_wp = pmd_uffd_wp(old_pmd); 2944 2945 VM_WARN_ON_FOLIO(!folio_ref_count(folio), folio); 2946 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); 2947 2948 /* 2949 * Without "freeze", we'll simply split the PMD, propagating the 2950 * PageAnonExclusive() flag for each PTE by setting it for 2951 * each subpage -- no need to (temporarily) clear. 2952 * 2953 * With "freeze" we want to replace mapped pages by 2954 * migration entries right away. This is only possible if we 2955 * managed to clear PageAnonExclusive() -- see 2956 * set_pmd_migration_entry(). 2957 * 2958 * In case we cannot clear PageAnonExclusive(), split the PMD 2959 * only and let try_to_migrate_one() fail later. 2960 * 2961 * See folio_try_share_anon_rmap_pmd(): invalidate PMD first. 2962 */ 2963 anon_exclusive = PageAnonExclusive(page); 2964 if (freeze && anon_exclusive && 2965 folio_try_share_anon_rmap_pmd(folio, page)) 2966 freeze = false; 2967 if (!freeze) { 2968 rmap_t rmap_flags = RMAP_NONE; 2969 2970 folio_ref_add(folio, HPAGE_PMD_NR - 1); 2971 if (anon_exclusive) 2972 rmap_flags |= RMAP_EXCLUSIVE; 2973 folio_add_anon_rmap_ptes(folio, page, HPAGE_PMD_NR, 2974 vma, haddr, rmap_flags); 2975 } 2976 } 2977 2978 /* 2979 * Withdraw the table only after we mark the pmd entry invalid. 2980 * This's critical for some architectures (Power). 2981 */ 2982 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2983 pmd_populate(mm, &_pmd, pgtable); 2984 2985 pte = pte_offset_map(&_pmd, haddr); 2986 VM_BUG_ON(!pte); 2987 2988 /* 2989 * Note that NUMA hinting access restrictions are not transferred to 2990 * avoid any possibility of altering permissions across VMAs. 2991 */ 2992 if (freeze || pmd_migration) { 2993 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2994 pte_t entry; 2995 swp_entry_t swp_entry; 2996 2997 if (write) 2998 swp_entry = make_writable_migration_entry( 2999 page_to_pfn(page + i)); 3000 else if (anon_exclusive) 3001 swp_entry = make_readable_exclusive_migration_entry( 3002 page_to_pfn(page + i)); 3003 else 3004 swp_entry = make_readable_migration_entry( 3005 page_to_pfn(page + i)); 3006 if (young) 3007 swp_entry = make_migration_entry_young(swp_entry); 3008 if (dirty) 3009 swp_entry = make_migration_entry_dirty(swp_entry); 3010 entry = swp_entry_to_pte(swp_entry); 3011 if (soft_dirty) 3012 entry = pte_swp_mksoft_dirty(entry); 3013 if (uffd_wp) 3014 entry = pte_swp_mkuffd_wp(entry); 3015 3016 VM_WARN_ON(!pte_none(ptep_get(pte + i))); 3017 set_pte_at(mm, addr, pte + i, entry); 3018 } 3019 } else { 3020 pte_t entry; 3021 3022 entry = mk_pte(page, READ_ONCE(vma->vm_page_prot)); 3023 if (write) 3024 entry = pte_mkwrite(entry, vma); 3025 if (!young) 3026 entry = pte_mkold(entry); 3027 /* NOTE: this may set soft-dirty too on some archs */ 3028 if (dirty) 3029 entry = pte_mkdirty(entry); 3030 if (soft_dirty) 3031 entry = pte_mksoft_dirty(entry); 3032 if (uffd_wp) 3033 entry = pte_mkuffd_wp(entry); 3034 3035 for (i = 0; i < HPAGE_PMD_NR; i++) 3036 VM_WARN_ON(!pte_none(ptep_get(pte + i))); 3037 3038 set_ptes(mm, haddr, pte, entry, HPAGE_PMD_NR); 3039 } 3040 pte_unmap(pte); 3041 3042 if (!pmd_migration) 3043 folio_remove_rmap_pmd(folio, page, vma); 3044 if (freeze) 3045 put_page(page); 3046 3047 smp_wmb(); /* make pte visible before pmd */ 3048 pmd_populate(mm, pmd, pgtable); 3049 } 3050 3051 void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, 3052 pmd_t *pmd, bool freeze) 3053 { 3054 VM_WARN_ON_ONCE(!IS_ALIGNED(address, HPAGE_PMD_SIZE)); 3055 if (pmd_trans_huge(*pmd) || is_pmd_migration_entry(*pmd)) 3056 __split_huge_pmd_locked(vma, pmd, address, freeze); 3057 } 3058 3059 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 3060 unsigned long address, bool freeze) 3061 { 3062 spinlock_t *ptl; 3063 struct mmu_notifier_range range; 3064 3065 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 3066 address & HPAGE_PMD_MASK, 3067 (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE); 3068 mmu_notifier_invalidate_range_start(&range); 3069 ptl = pmd_lock(vma->vm_mm, pmd); 3070 split_huge_pmd_locked(vma, range.start, pmd, freeze); 3071 spin_unlock(ptl); 3072 mmu_notifier_invalidate_range_end(&range); 3073 } 3074 3075 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, 3076 bool freeze) 3077 { 3078 pmd_t *pmd = mm_find_pmd(vma->vm_mm, address); 3079 3080 if (!pmd) 3081 return; 3082 3083 __split_huge_pmd(vma, pmd, address, freeze); 3084 } 3085 3086 static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address) 3087 { 3088 /* 3089 * If the new address isn't hpage aligned and it could previously 3090 * contain an hugepage: check if we need to split an huge pmd. 3091 */ 3092 if (!IS_ALIGNED(address, HPAGE_PMD_SIZE) && 3093 range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE), 3094 ALIGN(address, HPAGE_PMD_SIZE))) 3095 split_huge_pmd_address(vma, address, false); 3096 } 3097 3098 void vma_adjust_trans_huge(struct vm_area_struct *vma, 3099 unsigned long start, 3100 unsigned long end, 3101 struct vm_area_struct *next) 3102 { 3103 /* Check if we need to split start first. */ 3104 split_huge_pmd_if_needed(vma, start); 3105 3106 /* Check if we need to split end next. */ 3107 split_huge_pmd_if_needed(vma, end); 3108 3109 /* If we're incrementing next->vm_start, we might need to split it. */ 3110 if (next) 3111 split_huge_pmd_if_needed(next, end); 3112 } 3113 3114 static void unmap_folio(struct folio *folio) 3115 { 3116 enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SYNC | 3117 TTU_BATCH_FLUSH; 3118 3119 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 3120 3121 if (folio_test_pmd_mappable(folio)) 3122 ttu_flags |= TTU_SPLIT_HUGE_PMD; 3123 3124 /* 3125 * Anon pages need migration entries to preserve them, but file 3126 * pages can simply be left unmapped, then faulted back on demand. 3127 * If that is ever changed (perhaps for mlock), update remap_page(). 3128 */ 3129 if (folio_test_anon(folio)) 3130 try_to_migrate(folio, ttu_flags); 3131 else 3132 try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK); 3133 3134 try_to_unmap_flush(); 3135 } 3136 3137 static bool __discard_anon_folio_pmd_locked(struct vm_area_struct *vma, 3138 unsigned long addr, pmd_t *pmdp, 3139 struct folio *folio) 3140 { 3141 struct mm_struct *mm = vma->vm_mm; 3142 int ref_count, map_count; 3143 pmd_t orig_pmd = *pmdp; 3144 3145 if (pmd_dirty(orig_pmd)) 3146 folio_set_dirty(folio); 3147 if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { 3148 folio_set_swapbacked(folio); 3149 return false; 3150 } 3151 3152 orig_pmd = pmdp_huge_clear_flush(vma, addr, pmdp); 3153 3154 /* 3155 * Syncing against concurrent GUP-fast: 3156 * - clear PMD; barrier; read refcount 3157 * - inc refcount; barrier; read PMD 3158 */ 3159 smp_mb(); 3160 3161 ref_count = folio_ref_count(folio); 3162 map_count = folio_mapcount(folio); 3163 3164 /* 3165 * Order reads for folio refcount and dirty flag 3166 * (see comments in __remove_mapping()). 3167 */ 3168 smp_rmb(); 3169 3170 /* 3171 * If the folio or its PMD is redirtied at this point, or if there 3172 * are unexpected references, we will give up to discard this folio 3173 * and remap it. 3174 * 3175 * The only folio refs must be one from isolation plus the rmap(s). 3176 */ 3177 if (pmd_dirty(orig_pmd)) 3178 folio_set_dirty(folio); 3179 if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { 3180 folio_set_swapbacked(folio); 3181 set_pmd_at(mm, addr, pmdp, orig_pmd); 3182 return false; 3183 } 3184 3185 if (ref_count != map_count + 1) { 3186 set_pmd_at(mm, addr, pmdp, orig_pmd); 3187 return false; 3188 } 3189 3190 folio_remove_rmap_pmd(folio, pmd_page(orig_pmd), vma); 3191 zap_deposited_table(mm, pmdp); 3192 add_mm_counter(mm, MM_ANONPAGES, -HPAGE_PMD_NR); 3193 if (vma->vm_flags & VM_LOCKED) 3194 mlock_drain_local(); 3195 folio_put(folio); 3196 3197 return true; 3198 } 3199 3200 bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr, 3201 pmd_t *pmdp, struct folio *folio) 3202 { 3203 VM_WARN_ON_FOLIO(!folio_test_pmd_mappable(folio), folio); 3204 VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); 3205 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); 3206 VM_WARN_ON_FOLIO(folio_test_swapbacked(folio), folio); 3207 VM_WARN_ON_ONCE(!IS_ALIGNED(addr, HPAGE_PMD_SIZE)); 3208 3209 return __discard_anon_folio_pmd_locked(vma, addr, pmdp, folio); 3210 } 3211 3212 static void remap_page(struct folio *folio, unsigned long nr, int flags) 3213 { 3214 int i = 0; 3215 3216 /* If unmap_folio() uses try_to_migrate() on file, remove this check */ 3217 if (!folio_test_anon(folio)) 3218 return; 3219 for (;;) { 3220 remove_migration_ptes(folio, folio, RMP_LOCKED | flags); 3221 i += folio_nr_pages(folio); 3222 if (i >= nr) 3223 break; 3224 folio = folio_next(folio); 3225 } 3226 } 3227 3228 static void lru_add_split_folio(struct folio *folio, struct folio *new_folio, 3229 struct lruvec *lruvec, struct list_head *list) 3230 { 3231 VM_BUG_ON_FOLIO(folio_test_lru(new_folio), folio); 3232 lockdep_assert_held(&lruvec->lru_lock); 3233 3234 if (list) { 3235 /* page reclaim is reclaiming a huge page */ 3236 VM_WARN_ON(folio_test_lru(folio)); 3237 folio_get(new_folio); 3238 list_add_tail(&new_folio->lru, list); 3239 } else { 3240 /* head is still on lru (and we have it frozen) */ 3241 VM_WARN_ON(!folio_test_lru(folio)); 3242 if (folio_test_unevictable(folio)) 3243 new_folio->mlock_count = 0; 3244 else 3245 list_add_tail(&new_folio->lru, &folio->lru); 3246 folio_set_lru(new_folio); 3247 } 3248 } 3249 3250 /* Racy check whether the huge page can be split */ 3251 bool can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins) 3252 { 3253 int extra_pins; 3254 3255 /* Additional pins from page cache */ 3256 if (folio_test_anon(folio)) 3257 extra_pins = folio_test_swapcache(folio) ? 3258 folio_nr_pages(folio) : 0; 3259 else 3260 extra_pins = folio_nr_pages(folio); 3261 if (pextra_pins) 3262 *pextra_pins = extra_pins; 3263 return folio_mapcount(folio) == folio_ref_count(folio) - extra_pins - 3264 caller_pins; 3265 } 3266 3267 static bool page_range_has_hwpoisoned(struct page *page, long nr_pages) 3268 { 3269 for (; nr_pages; page++, nr_pages--) 3270 if (PageHWPoison(page)) 3271 return true; 3272 return false; 3273 } 3274 3275 /* 3276 * It splits @folio into @new_order folios and copies the @folio metadata to 3277 * all the resulting folios. 3278 */ 3279 static void __split_folio_to_order(struct folio *folio, int old_order, 3280 int new_order) 3281 { 3282 /* Scan poisoned pages when split a poisoned folio to large folios */ 3283 const bool handle_hwpoison = folio_test_has_hwpoisoned(folio) && new_order; 3284 long new_nr_pages = 1 << new_order; 3285 long nr_pages = 1 << old_order; 3286 long i; 3287 3288 folio_clear_has_hwpoisoned(folio); 3289 3290 /* Check first new_nr_pages since the loop below skips them */ 3291 if (handle_hwpoison && 3292 page_range_has_hwpoisoned(folio_page(folio, 0), new_nr_pages)) 3293 folio_set_has_hwpoisoned(folio); 3294 /* 3295 * Skip the first new_nr_pages, since the new folio from them have all 3296 * the flags from the original folio. 3297 */ 3298 for (i = new_nr_pages; i < nr_pages; i += new_nr_pages) { 3299 struct page *new_head = &folio->page + i; 3300 /* 3301 * Careful: new_folio is not a "real" folio before we cleared PageTail. 3302 * Don't pass it around before clear_compound_head(). 3303 */ 3304 struct folio *new_folio = (struct folio *)new_head; 3305 3306 VM_BUG_ON_PAGE(atomic_read(&new_folio->_mapcount) != -1, new_head); 3307 3308 /* 3309 * Clone page flags before unfreezing refcount. 3310 * 3311 * After successful get_page_unless_zero() might follow flags change, 3312 * for example lock_page() which set PG_waiters. 3313 * 3314 * Note that for mapped sub-pages of an anonymous THP, 3315 * PG_anon_exclusive has been cleared in unmap_folio() and is stored in 3316 * the migration entry instead from where remap_page() will restore it. 3317 * We can still have PG_anon_exclusive set on effectively unmapped and 3318 * unreferenced sub-pages of an anonymous THP: we can simply drop 3319 * PG_anon_exclusive (-> PG_mappedtodisk) for these here. 3320 */ 3321 new_folio->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP; 3322 new_folio->flags.f |= (folio->flags.f & 3323 ((1L << PG_referenced) | 3324 (1L << PG_swapbacked) | 3325 (1L << PG_swapcache) | 3326 (1L << PG_mlocked) | 3327 (1L << PG_uptodate) | 3328 (1L << PG_active) | 3329 (1L << PG_workingset) | 3330 (1L << PG_locked) | 3331 (1L << PG_unevictable) | 3332 #ifdef CONFIG_ARCH_USES_PG_ARCH_2 3333 (1L << PG_arch_2) | 3334 #endif 3335 #ifdef CONFIG_ARCH_USES_PG_ARCH_3 3336 (1L << PG_arch_3) | 3337 #endif 3338 (1L << PG_dirty) | 3339 LRU_GEN_MASK | LRU_REFS_MASK)); 3340 3341 if (handle_hwpoison && 3342 page_range_has_hwpoisoned(new_head, new_nr_pages)) 3343 folio_set_has_hwpoisoned(new_folio); 3344 3345 new_folio->mapping = folio->mapping; 3346 new_folio->index = folio->index + i; 3347 3348 /* 3349 * page->private should not be set in tail pages. Fix up and warn once 3350 * if private is unexpectedly set. 3351 */ 3352 if (unlikely(new_folio->private)) { 3353 VM_WARN_ON_ONCE_PAGE(true, new_head); 3354 new_folio->private = NULL; 3355 } 3356 3357 if (folio_test_swapcache(folio)) 3358 new_folio->swap.val = folio->swap.val + i; 3359 3360 /* Page flags must be visible before we make the page non-compound. */ 3361 smp_wmb(); 3362 3363 /* 3364 * Clear PageTail before unfreezing page refcount. 3365 * 3366 * After successful get_page_unless_zero() might follow put_page() 3367 * which needs correct compound_head(). 3368 */ 3369 clear_compound_head(new_head); 3370 if (new_order) { 3371 prep_compound_page(new_head, new_order); 3372 folio_set_large_rmappable(new_folio); 3373 } 3374 3375 if (folio_test_young(folio)) 3376 folio_set_young(new_folio); 3377 if (folio_test_idle(folio)) 3378 folio_set_idle(new_folio); 3379 #ifdef CONFIG_MEMCG 3380 new_folio->memcg_data = folio->memcg_data; 3381 #endif 3382 3383 folio_xchg_last_cpupid(new_folio, folio_last_cpupid(folio)); 3384 } 3385 3386 if (new_order) 3387 folio_set_order(folio, new_order); 3388 else 3389 ClearPageCompound(&folio->page); 3390 } 3391 3392 /* 3393 * It splits an unmapped @folio to lower order smaller folios in two ways. 3394 * @folio: the to-be-split folio 3395 * @new_order: the smallest order of the after split folios (since buddy 3396 * allocator like split generates folios with orders from @folio's 3397 * order - 1 to new_order). 3398 * @split_at: in buddy allocator like split, the folio containing @split_at 3399 * will be split until its order becomes @new_order. 3400 * @xas: xa_state pointing to folio->mapping->i_pages and locked by caller 3401 * @mapping: @folio->mapping 3402 * @uniform_split: if the split is uniform or not (buddy allocator like split) 3403 * 3404 * 3405 * 1. uniform split: the given @folio into multiple @new_order small folios, 3406 * where all small folios have the same order. This is done when 3407 * uniform_split is true. 3408 * 2. buddy allocator like (non-uniform) split: the given @folio is split into 3409 * half and one of the half (containing the given page) is split into half 3410 * until the given @page's order becomes @new_order. This is done when 3411 * uniform_split is false. 3412 * 3413 * The high level flow for these two methods are: 3414 * 1. uniform split: a single __split_folio_to_order() is called to split the 3415 * @folio into @new_order, then we traverse all the resulting folios one by 3416 * one in PFN ascending order and perform stats, unfreeze, adding to list, 3417 * and file mapping index operations. 3418 * 2. non-uniform split: in general, folio_order - @new_order calls to 3419 * __split_folio_to_order() are made in a for loop to split the @folio 3420 * to one lower order at a time. The resulting small folios are processed 3421 * like what is done during the traversal in 1, except the one containing 3422 * @page, which is split in next for loop. 3423 * 3424 * After splitting, the caller's folio reference will be transferred to the 3425 * folio containing @page. The caller needs to unlock and/or free after-split 3426 * folios if necessary. 3427 * 3428 * For !uniform_split, when -ENOMEM is returned, the original folio might be 3429 * split. The caller needs to check the input folio. 3430 */ 3431 static int __split_unmapped_folio(struct folio *folio, int new_order, 3432 struct page *split_at, struct xa_state *xas, 3433 struct address_space *mapping, bool uniform_split) 3434 { 3435 int order = folio_order(folio); 3436 int start_order = uniform_split ? new_order : order - 1; 3437 bool stop_split = false; 3438 struct folio *next; 3439 int split_order; 3440 int ret = 0; 3441 3442 if (folio_test_anon(folio)) 3443 mod_mthp_stat(order, MTHP_STAT_NR_ANON, -1); 3444 3445 /* 3446 * split to new_order one order at a time. For uniform split, 3447 * folio is split to new_order directly. 3448 */ 3449 for (split_order = start_order; 3450 split_order >= new_order && !stop_split; 3451 split_order--) { 3452 struct folio *end_folio = folio_next(folio); 3453 int old_order = folio_order(folio); 3454 struct folio *new_folio; 3455 3456 /* order-1 anonymous folio is not supported */ 3457 if (folio_test_anon(folio) && split_order == 1) 3458 continue; 3459 if (uniform_split && split_order != new_order) 3460 continue; 3461 3462 if (mapping) { 3463 /* 3464 * uniform split has xas_split_alloc() called before 3465 * irq is disabled to allocate enough memory, whereas 3466 * non-uniform split can handle ENOMEM. 3467 */ 3468 if (uniform_split) 3469 xas_split(xas, folio, old_order); 3470 else { 3471 xas_set_order(xas, folio->index, split_order); 3472 xas_try_split(xas, folio, old_order); 3473 if (xas_error(xas)) { 3474 ret = xas_error(xas); 3475 stop_split = true; 3476 } 3477 } 3478 } 3479 3480 if (!stop_split) { 3481 folio_split_memcg_refs(folio, old_order, split_order); 3482 split_page_owner(&folio->page, old_order, split_order); 3483 pgalloc_tag_split(folio, old_order, split_order); 3484 3485 __split_folio_to_order(folio, old_order, split_order); 3486 } 3487 3488 /* 3489 * Iterate through after-split folios and update folio stats. 3490 * But in buddy allocator like split, the folio 3491 * containing the specified page is skipped until its order 3492 * is new_order, since the folio will be worked on in next 3493 * iteration. 3494 */ 3495 for (new_folio = folio; new_folio != end_folio; new_folio = next) { 3496 next = folio_next(new_folio); 3497 /* 3498 * for buddy allocator like split, new_folio containing 3499 * @split_at page could be split again, thus do not 3500 * change stats yet. Wait until new_folio's order is 3501 * @new_order or stop_split is set to true by the above 3502 * xas_split() failure. 3503 */ 3504 if (new_folio == page_folio(split_at)) { 3505 folio = new_folio; 3506 if (split_order != new_order && !stop_split) 3507 continue; 3508 } 3509 if (folio_test_anon(new_folio)) 3510 mod_mthp_stat(folio_order(new_folio), 3511 MTHP_STAT_NR_ANON, 1); 3512 } 3513 } 3514 3515 return ret; 3516 } 3517 3518 bool non_uniform_split_supported(struct folio *folio, unsigned int new_order, 3519 bool warns) 3520 { 3521 if (folio_test_anon(folio)) { 3522 /* order-1 is not supported for anonymous THP. */ 3523 VM_WARN_ONCE(warns && new_order == 1, 3524 "Cannot split to order-1 folio"); 3525 return new_order != 1; 3526 } else if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && 3527 !mapping_large_folio_support(folio->mapping)) { 3528 /* 3529 * No split if the file system does not support large folio. 3530 * Note that we might still have THPs in such mappings due to 3531 * CONFIG_READ_ONLY_THP_FOR_FS. But in that case, the mapping 3532 * does not actually support large folios properly. 3533 */ 3534 VM_WARN_ONCE(warns, 3535 "Cannot split file folio to non-0 order"); 3536 return false; 3537 } 3538 3539 /* Only swapping a whole PMD-mapped folio is supported */ 3540 if (folio_test_swapcache(folio)) { 3541 VM_WARN_ONCE(warns, 3542 "Cannot split swapcache folio to non-0 order"); 3543 return false; 3544 } 3545 3546 return true; 3547 } 3548 3549 /* See comments in non_uniform_split_supported() */ 3550 bool uniform_split_supported(struct folio *folio, unsigned int new_order, 3551 bool warns) 3552 { 3553 if (folio_test_anon(folio)) { 3554 VM_WARN_ONCE(warns && new_order == 1, 3555 "Cannot split to order-1 folio"); 3556 return new_order != 1; 3557 } else if (new_order) { 3558 if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && 3559 !mapping_large_folio_support(folio->mapping)) { 3560 VM_WARN_ONCE(warns, 3561 "Cannot split file folio to non-0 order"); 3562 return false; 3563 } 3564 } 3565 3566 if (new_order && folio_test_swapcache(folio)) { 3567 VM_WARN_ONCE(warns, 3568 "Cannot split swapcache folio to non-0 order"); 3569 return false; 3570 } 3571 3572 return true; 3573 } 3574 3575 /* 3576 * __folio_split: split a folio at @split_at to a @new_order folio 3577 * @folio: folio to split 3578 * @new_order: the order of the new folio 3579 * @split_at: a page within the new folio 3580 * @lock_at: a page within @folio to be left locked to caller 3581 * @list: after-split folios will be put on it if non NULL 3582 * @uniform_split: perform uniform split or not (non-uniform split) 3583 * 3584 * It calls __split_unmapped_folio() to perform uniform and non-uniform split. 3585 * It is in charge of checking whether the split is supported or not and 3586 * preparing @folio for __split_unmapped_folio(). 3587 * 3588 * After splitting, the after-split folio containing @lock_at remains locked 3589 * and others are unlocked: 3590 * 1. for uniform split, @lock_at points to one of @folio's subpages; 3591 * 2. for buddy allocator like (non-uniform) split, @lock_at points to @folio. 3592 * 3593 * return: 0: successful, <0 failed (if -ENOMEM is returned, @folio might be 3594 * split but not to @new_order, the caller needs to check) 3595 */ 3596 static int __folio_split(struct folio *folio, unsigned int new_order, 3597 struct page *split_at, struct page *lock_at, 3598 struct list_head *list, bool uniform_split) 3599 { 3600 struct deferred_split *ds_queue = get_deferred_split_queue(folio); 3601 XA_STATE(xas, &folio->mapping->i_pages, folio->index); 3602 struct folio *end_folio = folio_next(folio); 3603 bool is_anon = folio_test_anon(folio); 3604 struct address_space *mapping = NULL; 3605 struct anon_vma *anon_vma = NULL; 3606 int order = folio_order(folio); 3607 struct folio *new_folio, *next; 3608 int nr_shmem_dropped = 0; 3609 int remap_flags = 0; 3610 int extra_pins, ret; 3611 pgoff_t end; 3612 bool is_hzp; 3613 3614 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); 3615 VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio); 3616 3617 if (folio != page_folio(split_at) || folio != page_folio(lock_at)) 3618 return -EINVAL; 3619 3620 if (new_order >= folio_order(folio)) 3621 return -EINVAL; 3622 3623 if (uniform_split && !uniform_split_supported(folio, new_order, true)) 3624 return -EINVAL; 3625 3626 if (!uniform_split && 3627 !non_uniform_split_supported(folio, new_order, true)) 3628 return -EINVAL; 3629 3630 is_hzp = is_huge_zero_folio(folio); 3631 if (is_hzp) { 3632 pr_warn_ratelimited("Called split_huge_page for huge zero page\n"); 3633 return -EBUSY; 3634 } 3635 3636 if (folio_test_writeback(folio)) 3637 return -EBUSY; 3638 3639 if (is_anon) { 3640 /* 3641 * The caller does not necessarily hold an mmap_lock that would 3642 * prevent the anon_vma disappearing so we first we take a 3643 * reference to it and then lock the anon_vma for write. This 3644 * is similar to folio_lock_anon_vma_read except the write lock 3645 * is taken to serialise against parallel split or collapse 3646 * operations. 3647 */ 3648 anon_vma = folio_get_anon_vma(folio); 3649 if (!anon_vma) { 3650 ret = -EBUSY; 3651 goto out; 3652 } 3653 mapping = NULL; 3654 anon_vma_lock_write(anon_vma); 3655 } else { 3656 unsigned int min_order; 3657 gfp_t gfp; 3658 3659 mapping = folio->mapping; 3660 3661 /* Truncated ? */ 3662 /* 3663 * TODO: add support for large shmem folio in swap cache. 3664 * When shmem is in swap cache, mapping is NULL and 3665 * folio_test_swapcache() is true. 3666 */ 3667 if (!mapping) { 3668 ret = -EBUSY; 3669 goto out; 3670 } 3671 3672 min_order = mapping_min_folio_order(folio->mapping); 3673 if (new_order < min_order) { 3674 ret = -EINVAL; 3675 goto out; 3676 } 3677 3678 gfp = current_gfp_context(mapping_gfp_mask(mapping) & 3679 GFP_RECLAIM_MASK); 3680 3681 if (!filemap_release_folio(folio, gfp)) { 3682 ret = -EBUSY; 3683 goto out; 3684 } 3685 3686 if (uniform_split) { 3687 xas_set_order(&xas, folio->index, new_order); 3688 xas_split_alloc(&xas, folio, folio_order(folio), gfp); 3689 if (xas_error(&xas)) { 3690 ret = xas_error(&xas); 3691 goto out; 3692 } 3693 } 3694 3695 anon_vma = NULL; 3696 i_mmap_lock_read(mapping); 3697 3698 /* 3699 *__split_unmapped_folio() may need to trim off pages beyond 3700 * EOF: but on 32-bit, i_size_read() takes an irq-unsafe 3701 * seqlock, which cannot be nested inside the page tree lock. 3702 * So note end now: i_size itself may be changed at any moment, 3703 * but folio lock is good enough to serialize the trimming. 3704 */ 3705 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); 3706 if (shmem_mapping(mapping)) 3707 end = shmem_fallocend(mapping->host, end); 3708 } 3709 3710 /* 3711 * Racy check if we can split the page, before unmap_folio() will 3712 * split PMDs 3713 */ 3714 if (!can_split_folio(folio, 1, &extra_pins)) { 3715 ret = -EAGAIN; 3716 goto out_unlock; 3717 } 3718 3719 unmap_folio(folio); 3720 3721 /* block interrupt reentry in xa_lock and spinlock */ 3722 local_irq_disable(); 3723 if (mapping) { 3724 /* 3725 * Check if the folio is present in page cache. 3726 * We assume all tail are present too, if folio is there. 3727 */ 3728 xas_lock(&xas); 3729 xas_reset(&xas); 3730 if (xas_load(&xas) != folio) { 3731 ret = -EAGAIN; 3732 goto fail; 3733 } 3734 } 3735 3736 /* Prevent deferred_split_scan() touching ->_refcount */ 3737 spin_lock(&ds_queue->split_queue_lock); 3738 if (folio_ref_freeze(folio, 1 + extra_pins)) { 3739 struct swap_cluster_info *ci = NULL; 3740 struct lruvec *lruvec; 3741 int expected_refs; 3742 3743 if (folio_order(folio) > 1 && 3744 !list_empty(&folio->_deferred_list)) { 3745 ds_queue->split_queue_len--; 3746 if (folio_test_partially_mapped(folio)) { 3747 folio_clear_partially_mapped(folio); 3748 mod_mthp_stat(folio_order(folio), 3749 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 3750 } 3751 /* 3752 * Reinitialize page_deferred_list after removing the 3753 * page from the split_queue, otherwise a subsequent 3754 * split will see list corruption when checking the 3755 * page_deferred_list. 3756 */ 3757 list_del_init(&folio->_deferred_list); 3758 } 3759 spin_unlock(&ds_queue->split_queue_lock); 3760 if (mapping) { 3761 int nr = folio_nr_pages(folio); 3762 3763 if (folio_test_pmd_mappable(folio) && 3764 new_order < HPAGE_PMD_ORDER) { 3765 if (folio_test_swapbacked(folio)) { 3766 __lruvec_stat_mod_folio(folio, 3767 NR_SHMEM_THPS, -nr); 3768 } else { 3769 __lruvec_stat_mod_folio(folio, 3770 NR_FILE_THPS, -nr); 3771 filemap_nr_thps_dec(mapping); 3772 } 3773 } 3774 } 3775 3776 if (folio_test_swapcache(folio)) { 3777 if (mapping) { 3778 VM_WARN_ON_ONCE_FOLIO(mapping, folio); 3779 ret = -EINVAL; 3780 goto fail; 3781 } 3782 3783 ci = swap_cluster_get_and_lock(folio); 3784 } 3785 3786 /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ 3787 lruvec = folio_lruvec_lock(folio); 3788 3789 ret = __split_unmapped_folio(folio, new_order, split_at, &xas, 3790 mapping, uniform_split); 3791 3792 /* 3793 * Unfreeze after-split folios and put them back to the right 3794 * list. @folio should be kept frozon until page cache 3795 * entries are updated with all the other after-split folios 3796 * to prevent others seeing stale page cache entries. 3797 * As a result, new_folio starts from the next folio of 3798 * @folio. 3799 */ 3800 for (new_folio = folio_next(folio); new_folio != end_folio; 3801 new_folio = next) { 3802 unsigned long nr_pages = folio_nr_pages(new_folio); 3803 3804 next = folio_next(new_folio); 3805 3806 expected_refs = folio_expected_ref_count(new_folio) + 1; 3807 folio_ref_unfreeze(new_folio, expected_refs); 3808 3809 lru_add_split_folio(folio, new_folio, lruvec, list); 3810 3811 /* 3812 * Anonymous folio with swap cache. 3813 * NOTE: shmem in swap cache is not supported yet. 3814 */ 3815 if (ci) { 3816 __swap_cache_replace_folio(ci, folio, new_folio); 3817 continue; 3818 } 3819 3820 /* Anonymous folio without swap cache */ 3821 if (!mapping) 3822 continue; 3823 3824 /* Add the new folio to the page cache. */ 3825 if (new_folio->index < end) { 3826 __xa_store(&mapping->i_pages, new_folio->index, 3827 new_folio, 0); 3828 continue; 3829 } 3830 3831 /* Drop folio beyond EOF: ->index >= end */ 3832 if (shmem_mapping(mapping)) 3833 nr_shmem_dropped += nr_pages; 3834 else if (folio_test_clear_dirty(new_folio)) 3835 folio_account_cleaned( 3836 new_folio, inode_to_wb(mapping->host)); 3837 __filemap_remove_folio(new_folio, NULL); 3838 folio_put_refs(new_folio, nr_pages); 3839 } 3840 /* 3841 * Unfreeze @folio only after all page cache entries, which 3842 * used to point to it, have been updated with new folios. 3843 * Otherwise, a parallel folio_try_get() can grab @folio 3844 * and its caller can see stale page cache entries. 3845 */ 3846 expected_refs = folio_expected_ref_count(folio) + 1; 3847 folio_ref_unfreeze(folio, expected_refs); 3848 3849 unlock_page_lruvec(lruvec); 3850 3851 if (ci) 3852 swap_cluster_unlock(ci); 3853 } else { 3854 spin_unlock(&ds_queue->split_queue_lock); 3855 ret = -EAGAIN; 3856 } 3857 fail: 3858 if (mapping) 3859 xas_unlock(&xas); 3860 3861 local_irq_enable(); 3862 3863 if (nr_shmem_dropped) 3864 shmem_uncharge(mapping->host, nr_shmem_dropped); 3865 3866 if (!ret && is_anon) 3867 remap_flags = RMP_USE_SHARED_ZEROPAGE; 3868 remap_page(folio, 1 << order, remap_flags); 3869 3870 /* 3871 * Unlock all after-split folios except the one containing 3872 * @lock_at page. If @folio is not split, it will be kept locked. 3873 */ 3874 for (new_folio = folio; new_folio != end_folio; new_folio = next) { 3875 next = folio_next(new_folio); 3876 if (new_folio == page_folio(lock_at)) 3877 continue; 3878 3879 folio_unlock(new_folio); 3880 /* 3881 * Subpages may be freed if there wasn't any mapping 3882 * like if add_to_swap() is running on a lru page that 3883 * had its mapping zapped. And freeing these pages 3884 * requires taking the lru_lock so we do the put_page 3885 * of the tail pages after the split is complete. 3886 */ 3887 free_folio_and_swap_cache(new_folio); 3888 } 3889 3890 out_unlock: 3891 if (anon_vma) { 3892 anon_vma_unlock_write(anon_vma); 3893 put_anon_vma(anon_vma); 3894 } 3895 if (mapping) 3896 i_mmap_unlock_read(mapping); 3897 out: 3898 xas_destroy(&xas); 3899 if (order == HPAGE_PMD_ORDER) 3900 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); 3901 count_mthp_stat(order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED); 3902 return ret; 3903 } 3904 3905 /* 3906 * This function splits a large folio into smaller folios of order @new_order. 3907 * @page can point to any page of the large folio to split. The split operation 3908 * does not change the position of @page. 3909 * 3910 * Prerequisites: 3911 * 3912 * 1) The caller must hold a reference on the @page's owning folio, also known 3913 * as the large folio. 3914 * 3915 * 2) The large folio must be locked. 3916 * 3917 * 3) The folio must not be pinned. Any unexpected folio references, including 3918 * GUP pins, will result in the folio not getting split; instead, the caller 3919 * will receive an -EAGAIN. 3920 * 3921 * 4) @new_order > 1, usually. Splitting to order-1 anonymous folios is not 3922 * supported for non-file-backed folios, because folio->_deferred_list, which 3923 * is used by partially mapped folios, is stored in subpage 2, but an order-1 3924 * folio only has subpages 0 and 1. File-backed order-1 folios are supported, 3925 * since they do not use _deferred_list. 3926 * 3927 * After splitting, the caller's folio reference will be transferred to @page, 3928 * resulting in a raised refcount of @page after this call. The other pages may 3929 * be freed if they are not mapped. 3930 * 3931 * If @list is null, tail pages will be added to LRU list, otherwise, to @list. 3932 * 3933 * Pages in @new_order will inherit the mapping, flags, and so on from the 3934 * huge page. 3935 * 3936 * Returns 0 if the huge page was split successfully. 3937 * 3938 * Returns -EAGAIN if the folio has unexpected reference (e.g., GUP) or if 3939 * the folio was concurrently removed from the page cache. 3940 * 3941 * Returns -EBUSY when trying to split the huge zeropage, if the folio is 3942 * under writeback, if fs-specific folio metadata cannot currently be 3943 * released, or if some unexpected race happened (e.g., anon VMA disappeared, 3944 * truncation). 3945 * 3946 * Callers should ensure that the order respects the address space mapping 3947 * min-order if one is set for non-anonymous folios. 3948 * 3949 * Returns -EINVAL when trying to split to an order that is incompatible 3950 * with the folio. Splitting to order 0 is compatible with all folios. 3951 */ 3952 int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, 3953 unsigned int new_order) 3954 { 3955 struct folio *folio = page_folio(page); 3956 3957 return __folio_split(folio, new_order, &folio->page, page, list, true); 3958 } 3959 3960 /* 3961 * folio_split: split a folio at @split_at to a @new_order folio 3962 * @folio: folio to split 3963 * @new_order: the order of the new folio 3964 * @split_at: a page within the new folio 3965 * 3966 * return: 0: successful, <0 failed (if -ENOMEM is returned, @folio might be 3967 * split but not to @new_order, the caller needs to check) 3968 * 3969 * It has the same prerequisites and returns as 3970 * split_huge_page_to_list_to_order(). 3971 * 3972 * Split a folio at @split_at to a new_order folio, leave the 3973 * remaining subpages of the original folio as large as possible. For example, 3974 * in the case of splitting an order-9 folio at its third order-3 subpages to 3975 * an order-3 folio, there are 2^(9-3)=64 order-3 subpages in the order-9 folio. 3976 * After the split, there will be a group of folios with different orders and 3977 * the new folio containing @split_at is marked in bracket: 3978 * [order-4, {order-3}, order-3, order-5, order-6, order-7, order-8]. 3979 * 3980 * After split, folio is left locked for caller. 3981 */ 3982 int folio_split(struct folio *folio, unsigned int new_order, 3983 struct page *split_at, struct list_head *list) 3984 { 3985 return __folio_split(folio, new_order, split_at, &folio->page, list, 3986 false); 3987 } 3988 3989 int min_order_for_split(struct folio *folio) 3990 { 3991 if (folio_test_anon(folio)) 3992 return 0; 3993 3994 if (!folio->mapping) { 3995 if (folio_test_pmd_mappable(folio)) 3996 count_vm_event(THP_SPLIT_PAGE_FAILED); 3997 return -EBUSY; 3998 } 3999 4000 return mapping_min_folio_order(folio->mapping); 4001 } 4002 4003 int split_folio_to_list(struct folio *folio, struct list_head *list) 4004 { 4005 return split_huge_page_to_list_to_order(&folio->page, list, 0); 4006 } 4007 4008 /* 4009 * __folio_unqueue_deferred_split() is not to be called directly: 4010 * the folio_unqueue_deferred_split() inline wrapper in mm/internal.h 4011 * limits its calls to those folios which may have a _deferred_list for 4012 * queueing THP splits, and that list is (racily observed to be) non-empty. 4013 * 4014 * It is unsafe to call folio_unqueue_deferred_split() until folio refcount is 4015 * zero: because even when split_queue_lock is held, a non-empty _deferred_list 4016 * might be in use on deferred_split_scan()'s unlocked on-stack list. 4017 * 4018 * If memory cgroups are enabled, split_queue_lock is in the mem_cgroup: it is 4019 * therefore important to unqueue deferred split before changing folio memcg. 4020 */ 4021 bool __folio_unqueue_deferred_split(struct folio *folio) 4022 { 4023 struct deferred_split *ds_queue; 4024 unsigned long flags; 4025 bool unqueued = false; 4026 4027 WARN_ON_ONCE(folio_ref_count(folio)); 4028 WARN_ON_ONCE(!mem_cgroup_disabled() && !folio_memcg(folio)); 4029 4030 ds_queue = get_deferred_split_queue(folio); 4031 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 4032 if (!list_empty(&folio->_deferred_list)) { 4033 ds_queue->split_queue_len--; 4034 if (folio_test_partially_mapped(folio)) { 4035 folio_clear_partially_mapped(folio); 4036 mod_mthp_stat(folio_order(folio), 4037 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 4038 } 4039 list_del_init(&folio->_deferred_list); 4040 unqueued = true; 4041 } 4042 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 4043 4044 return unqueued; /* useful for debug warnings */ 4045 } 4046 4047 /* partially_mapped=false won't clear PG_partially_mapped folio flag */ 4048 void deferred_split_folio(struct folio *folio, bool partially_mapped) 4049 { 4050 struct deferred_split *ds_queue = get_deferred_split_queue(folio); 4051 #ifdef CONFIG_MEMCG 4052 struct mem_cgroup *memcg = folio_memcg(folio); 4053 #endif 4054 unsigned long flags; 4055 4056 /* 4057 * Order 1 folios have no space for a deferred list, but we also 4058 * won't waste much memory by not adding them to the deferred list. 4059 */ 4060 if (folio_order(folio) <= 1) 4061 return; 4062 4063 if (!partially_mapped && !split_underused_thp) 4064 return; 4065 4066 /* 4067 * Exclude swapcache: originally to avoid a corrupt deferred split 4068 * queue. Nowadays that is fully prevented by memcg1_swapout(); 4069 * but if page reclaim is already handling the same folio, it is 4070 * unnecessary to handle it again in the shrinker, so excluding 4071 * swapcache here may still be a useful optimization. 4072 */ 4073 if (folio_test_swapcache(folio)) 4074 return; 4075 4076 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 4077 if (partially_mapped) { 4078 if (!folio_test_partially_mapped(folio)) { 4079 folio_set_partially_mapped(folio); 4080 if (folio_test_pmd_mappable(folio)) 4081 count_vm_event(THP_DEFERRED_SPLIT_PAGE); 4082 count_mthp_stat(folio_order(folio), MTHP_STAT_SPLIT_DEFERRED); 4083 mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, 1); 4084 4085 } 4086 } else { 4087 /* partially mapped folios cannot become non-partially mapped */ 4088 VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio); 4089 } 4090 if (list_empty(&folio->_deferred_list)) { 4091 list_add_tail(&folio->_deferred_list, &ds_queue->split_queue); 4092 ds_queue->split_queue_len++; 4093 #ifdef CONFIG_MEMCG 4094 if (memcg) 4095 set_shrinker_bit(memcg, folio_nid(folio), 4096 deferred_split_shrinker->id); 4097 #endif 4098 } 4099 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 4100 } 4101 4102 static unsigned long deferred_split_count(struct shrinker *shrink, 4103 struct shrink_control *sc) 4104 { 4105 struct pglist_data *pgdata = NODE_DATA(sc->nid); 4106 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 4107 4108 #ifdef CONFIG_MEMCG 4109 if (sc->memcg) 4110 ds_queue = &sc->memcg->deferred_split_queue; 4111 #endif 4112 return READ_ONCE(ds_queue->split_queue_len); 4113 } 4114 4115 static bool thp_underused(struct folio *folio) 4116 { 4117 int num_zero_pages = 0, num_filled_pages = 0; 4118 int i; 4119 4120 if (khugepaged_max_ptes_none == HPAGE_PMD_NR - 1) 4121 return false; 4122 4123 if (folio_contain_hwpoisoned_page(folio)) 4124 return false; 4125 4126 for (i = 0; i < folio_nr_pages(folio); i++) { 4127 if (pages_identical(folio_page(folio, i), ZERO_PAGE(0))) { 4128 if (++num_zero_pages > khugepaged_max_ptes_none) 4129 return true; 4130 } else { 4131 /* 4132 * Another path for early exit once the number 4133 * of non-zero filled pages exceeds threshold. 4134 */ 4135 if (++num_filled_pages >= HPAGE_PMD_NR - khugepaged_max_ptes_none) 4136 return false; 4137 } 4138 } 4139 return false; 4140 } 4141 4142 static unsigned long deferred_split_scan(struct shrinker *shrink, 4143 struct shrink_control *sc) 4144 { 4145 struct pglist_data *pgdata = NODE_DATA(sc->nid); 4146 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 4147 unsigned long flags; 4148 LIST_HEAD(list); 4149 struct folio *folio, *next, *prev = NULL; 4150 int split = 0, removed = 0; 4151 4152 #ifdef CONFIG_MEMCG 4153 if (sc->memcg) 4154 ds_queue = &sc->memcg->deferred_split_queue; 4155 #endif 4156 4157 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 4158 /* Take pin on all head pages to avoid freeing them under us */ 4159 list_for_each_entry_safe(folio, next, &ds_queue->split_queue, 4160 _deferred_list) { 4161 if (folio_try_get(folio)) { 4162 list_move(&folio->_deferred_list, &list); 4163 } else { 4164 /* We lost race with folio_put() */ 4165 if (folio_test_partially_mapped(folio)) { 4166 folio_clear_partially_mapped(folio); 4167 mod_mthp_stat(folio_order(folio), 4168 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 4169 } 4170 list_del_init(&folio->_deferred_list); 4171 ds_queue->split_queue_len--; 4172 } 4173 if (!--sc->nr_to_scan) 4174 break; 4175 } 4176 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 4177 4178 list_for_each_entry_safe(folio, next, &list, _deferred_list) { 4179 bool did_split = false; 4180 bool underused = false; 4181 4182 if (!folio_test_partially_mapped(folio)) { 4183 /* 4184 * See try_to_map_unused_to_zeropage(): we cannot 4185 * optimize zero-filled pages after splitting an 4186 * mlocked folio. 4187 */ 4188 if (folio_test_mlocked(folio)) 4189 goto next; 4190 underused = thp_underused(folio); 4191 if (!underused) 4192 goto next; 4193 } 4194 if (!folio_trylock(folio)) 4195 goto next; 4196 if (!split_folio(folio)) { 4197 did_split = true; 4198 if (underused) 4199 count_vm_event(THP_UNDERUSED_SPLIT_PAGE); 4200 split++; 4201 } 4202 folio_unlock(folio); 4203 next: 4204 /* 4205 * split_folio() removes folio from list on success. 4206 * Only add back to the queue if folio is partially mapped. 4207 * If thp_underused returns false, or if split_folio fails 4208 * in the case it was underused, then consider it used and 4209 * don't add it back to split_queue. 4210 */ 4211 if (did_split) { 4212 ; /* folio already removed from list */ 4213 } else if (!folio_test_partially_mapped(folio)) { 4214 list_del_init(&folio->_deferred_list); 4215 removed++; 4216 } else { 4217 /* 4218 * That unlocked list_del_init() above would be unsafe, 4219 * unless its folio is separated from any earlier folios 4220 * left on the list (which may be concurrently unqueued) 4221 * by one safe folio with refcount still raised. 4222 */ 4223 swap(folio, prev); 4224 } 4225 if (folio) 4226 folio_put(folio); 4227 } 4228 4229 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 4230 list_splice_tail(&list, &ds_queue->split_queue); 4231 ds_queue->split_queue_len -= removed; 4232 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 4233 4234 if (prev) 4235 folio_put(prev); 4236 4237 /* 4238 * Stop shrinker if we didn't split any page, but the queue is empty. 4239 * This can happen if pages were freed under us. 4240 */ 4241 if (!split && list_empty(&ds_queue->split_queue)) 4242 return SHRINK_STOP; 4243 return split; 4244 } 4245 4246 #ifdef CONFIG_DEBUG_FS 4247 static void split_huge_pages_all(void) 4248 { 4249 struct zone *zone; 4250 struct page *page; 4251 struct folio *folio; 4252 unsigned long pfn, max_zone_pfn; 4253 unsigned long total = 0, split = 0; 4254 4255 pr_debug("Split all THPs\n"); 4256 for_each_zone(zone) { 4257 if (!managed_zone(zone)) 4258 continue; 4259 max_zone_pfn = zone_end_pfn(zone); 4260 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 4261 int nr_pages; 4262 4263 page = pfn_to_online_page(pfn); 4264 if (!page || PageTail(page)) 4265 continue; 4266 folio = page_folio(page); 4267 if (!folio_try_get(folio)) 4268 continue; 4269 4270 if (unlikely(page_folio(page) != folio)) 4271 goto next; 4272 4273 if (zone != folio_zone(folio)) 4274 goto next; 4275 4276 if (!folio_test_large(folio) 4277 || folio_test_hugetlb(folio) 4278 || !folio_test_lru(folio)) 4279 goto next; 4280 4281 total++; 4282 folio_lock(folio); 4283 nr_pages = folio_nr_pages(folio); 4284 if (!split_folio(folio)) 4285 split++; 4286 pfn += nr_pages - 1; 4287 folio_unlock(folio); 4288 next: 4289 folio_put(folio); 4290 cond_resched(); 4291 } 4292 } 4293 4294 pr_debug("%lu of %lu THP split\n", split, total); 4295 } 4296 4297 static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma) 4298 { 4299 return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) || 4300 is_vm_hugetlb_page(vma); 4301 } 4302 4303 static int split_huge_pages_pid(int pid, unsigned long vaddr_start, 4304 unsigned long vaddr_end, unsigned int new_order, 4305 long in_folio_offset) 4306 { 4307 int ret = 0; 4308 struct task_struct *task; 4309 struct mm_struct *mm; 4310 unsigned long total = 0, split = 0; 4311 unsigned long addr; 4312 4313 vaddr_start &= PAGE_MASK; 4314 vaddr_end &= PAGE_MASK; 4315 4316 task = find_get_task_by_vpid(pid); 4317 if (!task) { 4318 ret = -ESRCH; 4319 goto out; 4320 } 4321 4322 /* Find the mm_struct */ 4323 mm = get_task_mm(task); 4324 put_task_struct(task); 4325 4326 if (!mm) { 4327 ret = -EINVAL; 4328 goto out; 4329 } 4330 4331 pr_debug("Split huge pages in pid: %d, vaddr: [0x%lx - 0x%lx], new_order: %u, in_folio_offset: %ld\n", 4332 pid, vaddr_start, vaddr_end, new_order, in_folio_offset); 4333 4334 mmap_read_lock(mm); 4335 /* 4336 * always increase addr by PAGE_SIZE, since we could have a PTE page 4337 * table filled with PTE-mapped THPs, each of which is distinct. 4338 */ 4339 for (addr = vaddr_start; addr < vaddr_end; addr += PAGE_SIZE) { 4340 struct vm_area_struct *vma = vma_lookup(mm, addr); 4341 struct folio_walk fw; 4342 struct folio *folio; 4343 struct address_space *mapping; 4344 unsigned int target_order = new_order; 4345 4346 if (!vma) 4347 break; 4348 4349 /* skip special VMA and hugetlb VMA */ 4350 if (vma_not_suitable_for_thp_split(vma)) { 4351 addr = vma->vm_end; 4352 continue; 4353 } 4354 4355 folio = folio_walk_start(&fw, vma, addr, 0); 4356 if (!folio) 4357 continue; 4358 4359 if (!is_transparent_hugepage(folio)) 4360 goto next; 4361 4362 if (!folio_test_anon(folio)) { 4363 mapping = folio->mapping; 4364 target_order = max(new_order, 4365 mapping_min_folio_order(mapping)); 4366 } 4367 4368 if (target_order >= folio_order(folio)) 4369 goto next; 4370 4371 total++; 4372 /* 4373 * For folios with private, split_huge_page_to_list_to_order() 4374 * will try to drop it before split and then check if the folio 4375 * can be split or not. So skip the check here. 4376 */ 4377 if (!folio_test_private(folio) && 4378 !can_split_folio(folio, 0, NULL)) 4379 goto next; 4380 4381 if (!folio_trylock(folio)) 4382 goto next; 4383 folio_get(folio); 4384 folio_walk_end(&fw, vma); 4385 4386 if (!folio_test_anon(folio) && folio->mapping != mapping) 4387 goto unlock; 4388 4389 if (in_folio_offset < 0 || 4390 in_folio_offset >= folio_nr_pages(folio)) { 4391 if (!split_folio_to_order(folio, target_order)) 4392 split++; 4393 } else { 4394 struct page *split_at = folio_page(folio, 4395 in_folio_offset); 4396 if (!folio_split(folio, target_order, split_at, NULL)) 4397 split++; 4398 } 4399 4400 unlock: 4401 4402 folio_unlock(folio); 4403 folio_put(folio); 4404 4405 cond_resched(); 4406 continue; 4407 next: 4408 folio_walk_end(&fw, vma); 4409 cond_resched(); 4410 } 4411 mmap_read_unlock(mm); 4412 mmput(mm); 4413 4414 pr_debug("%lu of %lu THP split\n", split, total); 4415 4416 out: 4417 return ret; 4418 } 4419 4420 static int split_huge_pages_in_file(const char *file_path, pgoff_t off_start, 4421 pgoff_t off_end, unsigned int new_order, 4422 long in_folio_offset) 4423 { 4424 struct filename *file; 4425 struct file *candidate; 4426 struct address_space *mapping; 4427 int ret = -EINVAL; 4428 pgoff_t index; 4429 int nr_pages = 1; 4430 unsigned long total = 0, split = 0; 4431 unsigned int min_order; 4432 unsigned int target_order; 4433 4434 file = getname_kernel(file_path); 4435 if (IS_ERR(file)) 4436 return ret; 4437 4438 candidate = file_open_name(file, O_RDONLY, 0); 4439 if (IS_ERR(candidate)) 4440 goto out; 4441 4442 pr_debug("split file-backed THPs in file: %s, page offset: [0x%lx - 0x%lx], new_order: %u, in_folio_offset: %ld\n", 4443 file_path, off_start, off_end, new_order, in_folio_offset); 4444 4445 mapping = candidate->f_mapping; 4446 min_order = mapping_min_folio_order(mapping); 4447 target_order = max(new_order, min_order); 4448 4449 for (index = off_start; index < off_end; index += nr_pages) { 4450 struct folio *folio = filemap_get_folio(mapping, index); 4451 4452 nr_pages = 1; 4453 if (IS_ERR(folio)) 4454 continue; 4455 4456 if (!folio_test_large(folio)) 4457 goto next; 4458 4459 total++; 4460 nr_pages = folio_nr_pages(folio); 4461 4462 if (target_order >= folio_order(folio)) 4463 goto next; 4464 4465 if (!folio_trylock(folio)) 4466 goto next; 4467 4468 if (folio->mapping != mapping) 4469 goto unlock; 4470 4471 if (in_folio_offset < 0 || in_folio_offset >= nr_pages) { 4472 if (!split_folio_to_order(folio, target_order)) 4473 split++; 4474 } else { 4475 struct page *split_at = folio_page(folio, 4476 in_folio_offset); 4477 if (!folio_split(folio, target_order, split_at, NULL)) 4478 split++; 4479 } 4480 4481 unlock: 4482 folio_unlock(folio); 4483 next: 4484 folio_put(folio); 4485 cond_resched(); 4486 } 4487 4488 filp_close(candidate, NULL); 4489 ret = 0; 4490 4491 pr_debug("%lu of %lu file-backed THP split\n", split, total); 4492 out: 4493 putname(file); 4494 return ret; 4495 } 4496 4497 #define MAX_INPUT_BUF_SZ 255 4498 4499 static ssize_t split_huge_pages_write(struct file *file, const char __user *buf, 4500 size_t count, loff_t *ppops) 4501 { 4502 static DEFINE_MUTEX(split_debug_mutex); 4503 ssize_t ret; 4504 /* 4505 * hold pid, start_vaddr, end_vaddr, new_order or 4506 * file_path, off_start, off_end, new_order 4507 */ 4508 char input_buf[MAX_INPUT_BUF_SZ]; 4509 int pid; 4510 unsigned long vaddr_start, vaddr_end; 4511 unsigned int new_order = 0; 4512 long in_folio_offset = -1; 4513 4514 ret = mutex_lock_interruptible(&split_debug_mutex); 4515 if (ret) 4516 return ret; 4517 4518 ret = -EFAULT; 4519 4520 memset(input_buf, 0, MAX_INPUT_BUF_SZ); 4521 if (copy_from_user(input_buf, buf, min_t(size_t, count, MAX_INPUT_BUF_SZ))) 4522 goto out; 4523 4524 input_buf[MAX_INPUT_BUF_SZ - 1] = '\0'; 4525 4526 if (input_buf[0] == '/') { 4527 char *tok; 4528 char *tok_buf = input_buf; 4529 char file_path[MAX_INPUT_BUF_SZ]; 4530 pgoff_t off_start = 0, off_end = 0; 4531 size_t input_len = strlen(input_buf); 4532 4533 tok = strsep(&tok_buf, ","); 4534 if (tok && tok_buf) { 4535 strscpy(file_path, tok); 4536 } else { 4537 ret = -EINVAL; 4538 goto out; 4539 } 4540 4541 ret = sscanf(tok_buf, "0x%lx,0x%lx,%d,%ld", &off_start, &off_end, 4542 &new_order, &in_folio_offset); 4543 if (ret != 2 && ret != 3 && ret != 4) { 4544 ret = -EINVAL; 4545 goto out; 4546 } 4547 ret = split_huge_pages_in_file(file_path, off_start, off_end, 4548 new_order, in_folio_offset); 4549 if (!ret) 4550 ret = input_len; 4551 4552 goto out; 4553 } 4554 4555 ret = sscanf(input_buf, "%d,0x%lx,0x%lx,%d,%ld", &pid, &vaddr_start, 4556 &vaddr_end, &new_order, &in_folio_offset); 4557 if (ret == 1 && pid == 1) { 4558 split_huge_pages_all(); 4559 ret = strlen(input_buf); 4560 goto out; 4561 } else if (ret != 3 && ret != 4 && ret != 5) { 4562 ret = -EINVAL; 4563 goto out; 4564 } 4565 4566 ret = split_huge_pages_pid(pid, vaddr_start, vaddr_end, new_order, 4567 in_folio_offset); 4568 if (!ret) 4569 ret = strlen(input_buf); 4570 out: 4571 mutex_unlock(&split_debug_mutex); 4572 return ret; 4573 4574 } 4575 4576 static const struct file_operations split_huge_pages_fops = { 4577 .owner = THIS_MODULE, 4578 .write = split_huge_pages_write, 4579 }; 4580 4581 static int __init split_huge_pages_debugfs(void) 4582 { 4583 debugfs_create_file("split_huge_pages", 0200, NULL, NULL, 4584 &split_huge_pages_fops); 4585 return 0; 4586 } 4587 late_initcall(split_huge_pages_debugfs); 4588 #endif 4589 4590 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 4591 int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 4592 struct page *page) 4593 { 4594 struct folio *folio = page_folio(page); 4595 struct vm_area_struct *vma = pvmw->vma; 4596 struct mm_struct *mm = vma->vm_mm; 4597 unsigned long address = pvmw->address; 4598 bool anon_exclusive; 4599 pmd_t pmdval; 4600 swp_entry_t entry; 4601 pmd_t pmdswp; 4602 4603 if (!(pvmw->pmd && !pvmw->pte)) 4604 return 0; 4605 4606 flush_cache_range(vma, address, address + HPAGE_PMD_SIZE); 4607 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); 4608 4609 /* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. */ 4610 anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(page); 4611 if (anon_exclusive && folio_try_share_anon_rmap_pmd(folio, page)) { 4612 set_pmd_at(mm, address, pvmw->pmd, pmdval); 4613 return -EBUSY; 4614 } 4615 4616 if (pmd_dirty(pmdval)) 4617 folio_mark_dirty(folio); 4618 if (pmd_write(pmdval)) 4619 entry = make_writable_migration_entry(page_to_pfn(page)); 4620 else if (anon_exclusive) 4621 entry = make_readable_exclusive_migration_entry(page_to_pfn(page)); 4622 else 4623 entry = make_readable_migration_entry(page_to_pfn(page)); 4624 if (pmd_young(pmdval)) 4625 entry = make_migration_entry_young(entry); 4626 if (pmd_dirty(pmdval)) 4627 entry = make_migration_entry_dirty(entry); 4628 pmdswp = swp_entry_to_pmd(entry); 4629 if (pmd_soft_dirty(pmdval)) 4630 pmdswp = pmd_swp_mksoft_dirty(pmdswp); 4631 if (pmd_uffd_wp(pmdval)) 4632 pmdswp = pmd_swp_mkuffd_wp(pmdswp); 4633 set_pmd_at(mm, address, pvmw->pmd, pmdswp); 4634 folio_remove_rmap_pmd(folio, page, vma); 4635 folio_put(folio); 4636 trace_set_migration_pmd(address, pmd_val(pmdswp)); 4637 4638 return 0; 4639 } 4640 4641 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) 4642 { 4643 struct folio *folio = page_folio(new); 4644 struct vm_area_struct *vma = pvmw->vma; 4645 struct mm_struct *mm = vma->vm_mm; 4646 unsigned long address = pvmw->address; 4647 unsigned long haddr = address & HPAGE_PMD_MASK; 4648 pmd_t pmde; 4649 swp_entry_t entry; 4650 4651 if (!(pvmw->pmd && !pvmw->pte)) 4652 return; 4653 4654 entry = pmd_to_swp_entry(*pvmw->pmd); 4655 folio_get(folio); 4656 pmde = folio_mk_pmd(folio, READ_ONCE(vma->vm_page_prot)); 4657 if (pmd_swp_soft_dirty(*pvmw->pmd)) 4658 pmde = pmd_mksoft_dirty(pmde); 4659 if (is_writable_migration_entry(entry)) 4660 pmde = pmd_mkwrite(pmde, vma); 4661 if (pmd_swp_uffd_wp(*pvmw->pmd)) 4662 pmde = pmd_mkuffd_wp(pmde); 4663 if (!is_migration_entry_young(entry)) 4664 pmde = pmd_mkold(pmde); 4665 /* NOTE: this may contain setting soft-dirty on some archs */ 4666 if (folio_test_dirty(folio) && is_migration_entry_dirty(entry)) 4667 pmde = pmd_mkdirty(pmde); 4668 4669 if (folio_test_anon(folio)) { 4670 rmap_t rmap_flags = RMAP_NONE; 4671 4672 if (!is_readable_migration_entry(entry)) 4673 rmap_flags |= RMAP_EXCLUSIVE; 4674 4675 folio_add_anon_rmap_pmd(folio, new, vma, haddr, rmap_flags); 4676 } else { 4677 folio_add_file_rmap_pmd(folio, new, vma); 4678 } 4679 VM_BUG_ON(pmd_write(pmde) && folio_test_anon(folio) && !PageAnonExclusive(new)); 4680 set_pmd_at(mm, haddr, pvmw->pmd, pmde); 4681 4682 /* No need to invalidate - it was non-present before */ 4683 update_mmu_cache_pmd(vma, address, pvmw->pmd); 4684 trace_remove_migration_pmd(address, pmd_val(pmde)); 4685 } 4686 #endif 4687