1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2009 Red Hat, Inc. 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/mm.h> 9 #include <linux/sched.h> 10 #include <linux/sched/mm.h> 11 #include <linux/sched/numa_balancing.h> 12 #include <linux/highmem.h> 13 #include <linux/hugetlb.h> 14 #include <linux/mmu_notifier.h> 15 #include <linux/rmap.h> 16 #include <linux/swap.h> 17 #include <linux/shrinker.h> 18 #include <linux/mm_inline.h> 19 #include <linux/swapops.h> 20 #include <linux/backing-dev.h> 21 #include <linux/dax.h> 22 #include <linux/mm_types.h> 23 #include <linux/khugepaged.h> 24 #include <linux/freezer.h> 25 #include <linux/mman.h> 26 #include <linux/memremap.h> 27 #include <linux/pagemap.h> 28 #include <linux/debugfs.h> 29 #include <linux/migrate.h> 30 #include <linux/hashtable.h> 31 #include <linux/userfaultfd_k.h> 32 #include <linux/page_idle.h> 33 #include <linux/shmem_fs.h> 34 #include <linux/oom.h> 35 #include <linux/numa.h> 36 #include <linux/page_owner.h> 37 #include <linux/sched/sysctl.h> 38 #include <linux/memory-tiers.h> 39 #include <linux/compat.h> 40 #include <linux/pgalloc.h> 41 #include <linux/pgalloc_tag.h> 42 #include <linux/pagewalk.h> 43 44 #include <asm/tlb.h> 45 #include "internal.h" 46 #include "swap.h" 47 48 #define CREATE_TRACE_POINTS 49 #include <trace/events/thp.h> 50 51 /* 52 * By default, transparent hugepage support is disabled in order to avoid 53 * risking an increased memory footprint for applications that are not 54 * guaranteed to benefit from it. When transparent hugepage support is 55 * enabled, it is for all mappings, and khugepaged scans all mappings. 56 * Defrag is invoked by khugepaged hugepage allocations and by page faults 57 * for all hugepage allocations. 58 */ 59 unsigned long transparent_hugepage_flags __read_mostly = 60 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS 61 (1<<TRANSPARENT_HUGEPAGE_FLAG)| 62 #endif 63 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE 64 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| 65 #endif 66 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)| 67 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| 68 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 69 70 static struct shrinker *deferred_split_shrinker; 71 static unsigned long deferred_split_count(struct shrinker *shrink, 72 struct shrink_control *sc); 73 static unsigned long deferred_split_scan(struct shrinker *shrink, 74 struct shrink_control *sc); 75 static bool split_underused_thp = true; 76 77 static atomic_t huge_zero_refcount; 78 struct folio *huge_zero_folio __read_mostly; 79 unsigned long huge_zero_pfn __read_mostly = ~0UL; 80 unsigned long huge_anon_orders_always __read_mostly; 81 unsigned long huge_anon_orders_madvise __read_mostly; 82 unsigned long huge_anon_orders_inherit __read_mostly; 83 static bool anon_orders_configured __initdata; 84 85 static inline bool file_thp_enabled(struct vm_area_struct *vma) 86 { 87 struct inode *inode; 88 89 if (!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS)) 90 return false; 91 92 if (!vma->vm_file) 93 return false; 94 95 inode = file_inode(vma->vm_file); 96 97 return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); 98 } 99 100 unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, 101 vm_flags_t vm_flags, 102 enum tva_type type, 103 unsigned long orders) 104 { 105 const bool smaps = type == TVA_SMAPS; 106 const bool in_pf = type == TVA_PAGEFAULT; 107 const bool forced_collapse = type == TVA_FORCED_COLLAPSE; 108 unsigned long supported_orders; 109 110 /* Check the intersection of requested and supported orders. */ 111 if (vma_is_anonymous(vma)) 112 supported_orders = THP_ORDERS_ALL_ANON; 113 else if (vma_is_special_huge(vma)) 114 supported_orders = THP_ORDERS_ALL_SPECIAL; 115 else 116 supported_orders = THP_ORDERS_ALL_FILE_DEFAULT; 117 118 orders &= supported_orders; 119 if (!orders) 120 return 0; 121 122 if (!vma->vm_mm) /* vdso */ 123 return 0; 124 125 if (thp_disabled_by_hw() || vma_thp_disabled(vma, vm_flags, forced_collapse)) 126 return 0; 127 128 /* khugepaged doesn't collapse DAX vma, but page fault is fine. */ 129 if (vma_is_dax(vma)) 130 return in_pf ? orders : 0; 131 132 /* 133 * khugepaged special VMA and hugetlb VMA. 134 * Must be checked after dax since some dax mappings may have 135 * VM_MIXEDMAP set. 136 */ 137 if (!in_pf && !smaps && (vm_flags & VM_NO_KHUGEPAGED)) 138 return 0; 139 140 /* 141 * Check alignment for file vma and size for both file and anon vma by 142 * filtering out the unsuitable orders. 143 * 144 * Skip the check for page fault. Huge fault does the check in fault 145 * handlers. 146 */ 147 if (!in_pf) { 148 int order = highest_order(orders); 149 unsigned long addr; 150 151 while (orders) { 152 addr = vma->vm_end - (PAGE_SIZE << order); 153 if (thp_vma_suitable_order(vma, addr, order)) 154 break; 155 order = next_order(&orders, order); 156 } 157 158 if (!orders) 159 return 0; 160 } 161 162 /* 163 * Enabled via shmem mount options or sysfs settings. 164 * Must be done before hugepage flags check since shmem has its 165 * own flags. 166 */ 167 if (!in_pf && shmem_file(vma->vm_file)) 168 return orders & shmem_allowable_huge_orders(file_inode(vma->vm_file), 169 vma, vma->vm_pgoff, 0, 170 forced_collapse); 171 172 if (!vma_is_anonymous(vma)) { 173 /* 174 * Enforce THP collapse requirements as necessary. Anonymous vmas 175 * were already handled in thp_vma_allowable_orders(). 176 */ 177 if (!forced_collapse && 178 (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) && 179 !hugepage_global_always()))) 180 return 0; 181 182 /* 183 * Trust that ->huge_fault() handlers know what they are doing 184 * in fault path. 185 */ 186 if (((in_pf || smaps)) && vma->vm_ops->huge_fault) 187 return orders; 188 /* Only regular file is valid in collapse path */ 189 if (((!in_pf || smaps)) && file_thp_enabled(vma)) 190 return orders; 191 return 0; 192 } 193 194 if (vma_is_temporary_stack(vma)) 195 return 0; 196 197 /* 198 * THPeligible bit of smaps should show 1 for proper VMAs even 199 * though anon_vma is not initialized yet. 200 * 201 * Allow page fault since anon_vma may be not initialized until 202 * the first page fault. 203 */ 204 if (!vma->anon_vma) 205 return (smaps || in_pf) ? orders : 0; 206 207 return orders; 208 } 209 210 static bool get_huge_zero_folio(void) 211 { 212 struct folio *zero_folio; 213 retry: 214 if (likely(atomic_inc_not_zero(&huge_zero_refcount))) 215 return true; 216 217 zero_folio = folio_alloc((GFP_TRANSHUGE | __GFP_ZERO | __GFP_ZEROTAGS) & 218 ~__GFP_MOVABLE, 219 HPAGE_PMD_ORDER); 220 if (!zero_folio) { 221 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); 222 return false; 223 } 224 /* Ensure zero folio won't have large_rmappable flag set. */ 225 folio_clear_large_rmappable(zero_folio); 226 preempt_disable(); 227 if (cmpxchg(&huge_zero_folio, NULL, zero_folio)) { 228 preempt_enable(); 229 folio_put(zero_folio); 230 goto retry; 231 } 232 WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio)); 233 234 /* We take additional reference here. It will be put back by shrinker */ 235 atomic_set(&huge_zero_refcount, 2); 236 preempt_enable(); 237 count_vm_event(THP_ZERO_PAGE_ALLOC); 238 return true; 239 } 240 241 static void put_huge_zero_folio(void) 242 { 243 /* 244 * Counter should never go to zero here. Only shrinker can put 245 * last reference. 246 */ 247 BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); 248 } 249 250 struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) 251 { 252 if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) 253 return huge_zero_folio; 254 255 if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm)) 256 return READ_ONCE(huge_zero_folio); 257 258 if (!get_huge_zero_folio()) 259 return NULL; 260 261 if (mm_flags_test_and_set(MMF_HUGE_ZERO_FOLIO, mm)) 262 put_huge_zero_folio(); 263 264 return READ_ONCE(huge_zero_folio); 265 } 266 267 void mm_put_huge_zero_folio(struct mm_struct *mm) 268 { 269 if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) 270 return; 271 272 if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm)) 273 put_huge_zero_folio(); 274 } 275 276 static unsigned long shrink_huge_zero_folio_count(struct shrinker *shrink, 277 struct shrink_control *sc) 278 { 279 /* we can free zero page only if last reference remains */ 280 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; 281 } 282 283 static unsigned long shrink_huge_zero_folio_scan(struct shrinker *shrink, 284 struct shrink_control *sc) 285 { 286 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { 287 struct folio *zero_folio = xchg(&huge_zero_folio, NULL); 288 BUG_ON(zero_folio == NULL); 289 WRITE_ONCE(huge_zero_pfn, ~0UL); 290 folio_put(zero_folio); 291 return HPAGE_PMD_NR; 292 } 293 294 return 0; 295 } 296 297 static struct shrinker *huge_zero_folio_shrinker; 298 299 #ifdef CONFIG_SYSFS 300 static ssize_t enabled_show(struct kobject *kobj, 301 struct kobj_attribute *attr, char *buf) 302 { 303 const char *output; 304 305 if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags)) 306 output = "[always] madvise never"; 307 else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 308 &transparent_hugepage_flags)) 309 output = "always [madvise] never"; 310 else 311 output = "always madvise [never]"; 312 313 return sysfs_emit(buf, "%s\n", output); 314 } 315 316 static ssize_t enabled_store(struct kobject *kobj, 317 struct kobj_attribute *attr, 318 const char *buf, size_t count) 319 { 320 ssize_t ret = count; 321 322 if (sysfs_streq(buf, "always")) { 323 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 324 set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 325 } else if (sysfs_streq(buf, "madvise")) { 326 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 327 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 328 } else if (sysfs_streq(buf, "never")) { 329 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 330 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 331 } else 332 ret = -EINVAL; 333 334 if (ret > 0) { 335 int err = start_stop_khugepaged(); 336 if (err) 337 ret = err; 338 } 339 return ret; 340 } 341 342 static struct kobj_attribute enabled_attr = __ATTR_RW(enabled); 343 344 ssize_t single_hugepage_flag_show(struct kobject *kobj, 345 struct kobj_attribute *attr, char *buf, 346 enum transparent_hugepage_flag flag) 347 { 348 return sysfs_emit(buf, "%d\n", 349 !!test_bit(flag, &transparent_hugepage_flags)); 350 } 351 352 ssize_t single_hugepage_flag_store(struct kobject *kobj, 353 struct kobj_attribute *attr, 354 const char *buf, size_t count, 355 enum transparent_hugepage_flag flag) 356 { 357 unsigned long value; 358 int ret; 359 360 ret = kstrtoul(buf, 10, &value); 361 if (ret < 0) 362 return ret; 363 if (value > 1) 364 return -EINVAL; 365 366 if (value) 367 set_bit(flag, &transparent_hugepage_flags); 368 else 369 clear_bit(flag, &transparent_hugepage_flags); 370 371 return count; 372 } 373 374 static ssize_t defrag_show(struct kobject *kobj, 375 struct kobj_attribute *attr, char *buf) 376 { 377 const char *output; 378 379 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, 380 &transparent_hugepage_flags)) 381 output = "[always] defer defer+madvise madvise never"; 382 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, 383 &transparent_hugepage_flags)) 384 output = "always [defer] defer+madvise madvise never"; 385 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, 386 &transparent_hugepage_flags)) 387 output = "always defer [defer+madvise] madvise never"; 388 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, 389 &transparent_hugepage_flags)) 390 output = "always defer defer+madvise [madvise] never"; 391 else 392 output = "always defer defer+madvise madvise [never]"; 393 394 return sysfs_emit(buf, "%s\n", output); 395 } 396 397 static ssize_t defrag_store(struct kobject *kobj, 398 struct kobj_attribute *attr, 399 const char *buf, size_t count) 400 { 401 if (sysfs_streq(buf, "always")) { 402 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 403 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 404 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 405 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 406 } else if (sysfs_streq(buf, "defer+madvise")) { 407 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 408 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 409 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 410 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 411 } else if (sysfs_streq(buf, "defer")) { 412 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 413 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 414 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 415 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 416 } else if (sysfs_streq(buf, "madvise")) { 417 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 418 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 419 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 420 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 421 } else if (sysfs_streq(buf, "never")) { 422 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 423 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 424 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 425 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 426 } else 427 return -EINVAL; 428 429 return count; 430 } 431 static struct kobj_attribute defrag_attr = __ATTR_RW(defrag); 432 433 static ssize_t use_zero_page_show(struct kobject *kobj, 434 struct kobj_attribute *attr, char *buf) 435 { 436 return single_hugepage_flag_show(kobj, attr, buf, 437 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 438 } 439 static ssize_t use_zero_page_store(struct kobject *kobj, 440 struct kobj_attribute *attr, const char *buf, size_t count) 441 { 442 return single_hugepage_flag_store(kobj, attr, buf, count, 443 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 444 } 445 static struct kobj_attribute use_zero_page_attr = __ATTR_RW(use_zero_page); 446 447 static ssize_t hpage_pmd_size_show(struct kobject *kobj, 448 struct kobj_attribute *attr, char *buf) 449 { 450 return sysfs_emit(buf, "%lu\n", HPAGE_PMD_SIZE); 451 } 452 static struct kobj_attribute hpage_pmd_size_attr = 453 __ATTR_RO(hpage_pmd_size); 454 455 static ssize_t split_underused_thp_show(struct kobject *kobj, 456 struct kobj_attribute *attr, char *buf) 457 { 458 return sysfs_emit(buf, "%d\n", split_underused_thp); 459 } 460 461 static ssize_t split_underused_thp_store(struct kobject *kobj, 462 struct kobj_attribute *attr, 463 const char *buf, size_t count) 464 { 465 int err = kstrtobool(buf, &split_underused_thp); 466 467 if (err < 0) 468 return err; 469 470 return count; 471 } 472 473 static struct kobj_attribute split_underused_thp_attr = __ATTR( 474 shrink_underused, 0644, split_underused_thp_show, split_underused_thp_store); 475 476 static struct attribute *hugepage_attr[] = { 477 &enabled_attr.attr, 478 &defrag_attr.attr, 479 &use_zero_page_attr.attr, 480 &hpage_pmd_size_attr.attr, 481 #ifdef CONFIG_SHMEM 482 &shmem_enabled_attr.attr, 483 #endif 484 &split_underused_thp_attr.attr, 485 NULL, 486 }; 487 488 static const struct attribute_group hugepage_attr_group = { 489 .attrs = hugepage_attr, 490 }; 491 492 static void hugepage_exit_sysfs(struct kobject *hugepage_kobj); 493 static void thpsize_release(struct kobject *kobj); 494 static DEFINE_SPINLOCK(huge_anon_orders_lock); 495 static LIST_HEAD(thpsize_list); 496 497 static ssize_t anon_enabled_show(struct kobject *kobj, 498 struct kobj_attribute *attr, char *buf) 499 { 500 int order = to_thpsize(kobj)->order; 501 const char *output; 502 503 if (test_bit(order, &huge_anon_orders_always)) 504 output = "[always] inherit madvise never"; 505 else if (test_bit(order, &huge_anon_orders_inherit)) 506 output = "always [inherit] madvise never"; 507 else if (test_bit(order, &huge_anon_orders_madvise)) 508 output = "always inherit [madvise] never"; 509 else 510 output = "always inherit madvise [never]"; 511 512 return sysfs_emit(buf, "%s\n", output); 513 } 514 515 static ssize_t anon_enabled_store(struct kobject *kobj, 516 struct kobj_attribute *attr, 517 const char *buf, size_t count) 518 { 519 int order = to_thpsize(kobj)->order; 520 ssize_t ret = count; 521 522 if (sysfs_streq(buf, "always")) { 523 spin_lock(&huge_anon_orders_lock); 524 clear_bit(order, &huge_anon_orders_inherit); 525 clear_bit(order, &huge_anon_orders_madvise); 526 set_bit(order, &huge_anon_orders_always); 527 spin_unlock(&huge_anon_orders_lock); 528 } else if (sysfs_streq(buf, "inherit")) { 529 spin_lock(&huge_anon_orders_lock); 530 clear_bit(order, &huge_anon_orders_always); 531 clear_bit(order, &huge_anon_orders_madvise); 532 set_bit(order, &huge_anon_orders_inherit); 533 spin_unlock(&huge_anon_orders_lock); 534 } else if (sysfs_streq(buf, "madvise")) { 535 spin_lock(&huge_anon_orders_lock); 536 clear_bit(order, &huge_anon_orders_always); 537 clear_bit(order, &huge_anon_orders_inherit); 538 set_bit(order, &huge_anon_orders_madvise); 539 spin_unlock(&huge_anon_orders_lock); 540 } else if (sysfs_streq(buf, "never")) { 541 spin_lock(&huge_anon_orders_lock); 542 clear_bit(order, &huge_anon_orders_always); 543 clear_bit(order, &huge_anon_orders_inherit); 544 clear_bit(order, &huge_anon_orders_madvise); 545 spin_unlock(&huge_anon_orders_lock); 546 } else 547 ret = -EINVAL; 548 549 if (ret > 0) { 550 int err; 551 552 err = start_stop_khugepaged(); 553 if (err) 554 ret = err; 555 } 556 return ret; 557 } 558 559 static struct kobj_attribute anon_enabled_attr = 560 __ATTR(enabled, 0644, anon_enabled_show, anon_enabled_store); 561 562 static struct attribute *anon_ctrl_attrs[] = { 563 &anon_enabled_attr.attr, 564 NULL, 565 }; 566 567 static const struct attribute_group anon_ctrl_attr_grp = { 568 .attrs = anon_ctrl_attrs, 569 }; 570 571 static struct attribute *file_ctrl_attrs[] = { 572 #ifdef CONFIG_SHMEM 573 &thpsize_shmem_enabled_attr.attr, 574 #endif 575 NULL, 576 }; 577 578 static const struct attribute_group file_ctrl_attr_grp = { 579 .attrs = file_ctrl_attrs, 580 }; 581 582 static struct attribute *any_ctrl_attrs[] = { 583 NULL, 584 }; 585 586 static const struct attribute_group any_ctrl_attr_grp = { 587 .attrs = any_ctrl_attrs, 588 }; 589 590 static const struct kobj_type thpsize_ktype = { 591 .release = &thpsize_release, 592 .sysfs_ops = &kobj_sysfs_ops, 593 }; 594 595 DEFINE_PER_CPU(struct mthp_stat, mthp_stats) = {{{0}}}; 596 597 static unsigned long sum_mthp_stat(int order, enum mthp_stat_item item) 598 { 599 unsigned long sum = 0; 600 int cpu; 601 602 for_each_possible_cpu(cpu) { 603 struct mthp_stat *this = &per_cpu(mthp_stats, cpu); 604 605 sum += this->stats[order][item]; 606 } 607 608 return sum; 609 } 610 611 #define DEFINE_MTHP_STAT_ATTR(_name, _index) \ 612 static ssize_t _name##_show(struct kobject *kobj, \ 613 struct kobj_attribute *attr, char *buf) \ 614 { \ 615 int order = to_thpsize(kobj)->order; \ 616 \ 617 return sysfs_emit(buf, "%lu\n", sum_mthp_stat(order, _index)); \ 618 } \ 619 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 620 621 DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC); 622 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK); 623 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); 624 DEFINE_MTHP_STAT_ATTR(zswpout, MTHP_STAT_ZSWPOUT); 625 DEFINE_MTHP_STAT_ATTR(swpin, MTHP_STAT_SWPIN); 626 DEFINE_MTHP_STAT_ATTR(swpin_fallback, MTHP_STAT_SWPIN_FALLBACK); 627 DEFINE_MTHP_STAT_ATTR(swpin_fallback_charge, MTHP_STAT_SWPIN_FALLBACK_CHARGE); 628 DEFINE_MTHP_STAT_ATTR(swpout, MTHP_STAT_SWPOUT); 629 DEFINE_MTHP_STAT_ATTR(swpout_fallback, MTHP_STAT_SWPOUT_FALLBACK); 630 #ifdef CONFIG_SHMEM 631 DEFINE_MTHP_STAT_ATTR(shmem_alloc, MTHP_STAT_SHMEM_ALLOC); 632 DEFINE_MTHP_STAT_ATTR(shmem_fallback, MTHP_STAT_SHMEM_FALLBACK); 633 DEFINE_MTHP_STAT_ATTR(shmem_fallback_charge, MTHP_STAT_SHMEM_FALLBACK_CHARGE); 634 #endif 635 DEFINE_MTHP_STAT_ATTR(split, MTHP_STAT_SPLIT); 636 DEFINE_MTHP_STAT_ATTR(split_failed, MTHP_STAT_SPLIT_FAILED); 637 DEFINE_MTHP_STAT_ATTR(split_deferred, MTHP_STAT_SPLIT_DEFERRED); 638 DEFINE_MTHP_STAT_ATTR(nr_anon, MTHP_STAT_NR_ANON); 639 DEFINE_MTHP_STAT_ATTR(nr_anon_partially_mapped, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED); 640 641 static struct attribute *anon_stats_attrs[] = { 642 &anon_fault_alloc_attr.attr, 643 &anon_fault_fallback_attr.attr, 644 &anon_fault_fallback_charge_attr.attr, 645 #ifndef CONFIG_SHMEM 646 &zswpout_attr.attr, 647 &swpin_attr.attr, 648 &swpin_fallback_attr.attr, 649 &swpin_fallback_charge_attr.attr, 650 &swpout_attr.attr, 651 &swpout_fallback_attr.attr, 652 #endif 653 &split_deferred_attr.attr, 654 &nr_anon_attr.attr, 655 &nr_anon_partially_mapped_attr.attr, 656 NULL, 657 }; 658 659 static struct attribute_group anon_stats_attr_grp = { 660 .name = "stats", 661 .attrs = anon_stats_attrs, 662 }; 663 664 static struct attribute *file_stats_attrs[] = { 665 #ifdef CONFIG_SHMEM 666 &shmem_alloc_attr.attr, 667 &shmem_fallback_attr.attr, 668 &shmem_fallback_charge_attr.attr, 669 #endif 670 NULL, 671 }; 672 673 static struct attribute_group file_stats_attr_grp = { 674 .name = "stats", 675 .attrs = file_stats_attrs, 676 }; 677 678 static struct attribute *any_stats_attrs[] = { 679 #ifdef CONFIG_SHMEM 680 &zswpout_attr.attr, 681 &swpin_attr.attr, 682 &swpin_fallback_attr.attr, 683 &swpin_fallback_charge_attr.attr, 684 &swpout_attr.attr, 685 &swpout_fallback_attr.attr, 686 #endif 687 &split_attr.attr, 688 &split_failed_attr.attr, 689 NULL, 690 }; 691 692 static struct attribute_group any_stats_attr_grp = { 693 .name = "stats", 694 .attrs = any_stats_attrs, 695 }; 696 697 static int sysfs_add_group(struct kobject *kobj, 698 const struct attribute_group *grp) 699 { 700 int ret = -ENOENT; 701 702 /* 703 * If the group is named, try to merge first, assuming the subdirectory 704 * was already created. This avoids the warning emitted by 705 * sysfs_create_group() if the directory already exists. 706 */ 707 if (grp->name) 708 ret = sysfs_merge_group(kobj, grp); 709 if (ret) 710 ret = sysfs_create_group(kobj, grp); 711 712 return ret; 713 } 714 715 static struct thpsize *thpsize_create(int order, struct kobject *parent) 716 { 717 unsigned long size = (PAGE_SIZE << order) / SZ_1K; 718 struct thpsize *thpsize; 719 int ret = -ENOMEM; 720 721 thpsize = kzalloc(sizeof(*thpsize), GFP_KERNEL); 722 if (!thpsize) 723 goto err; 724 725 thpsize->order = order; 726 727 ret = kobject_init_and_add(&thpsize->kobj, &thpsize_ktype, parent, 728 "hugepages-%lukB", size); 729 if (ret) { 730 kfree(thpsize); 731 goto err; 732 } 733 734 735 ret = sysfs_add_group(&thpsize->kobj, &any_ctrl_attr_grp); 736 if (ret) 737 goto err_put; 738 739 ret = sysfs_add_group(&thpsize->kobj, &any_stats_attr_grp); 740 if (ret) 741 goto err_put; 742 743 if (BIT(order) & THP_ORDERS_ALL_ANON) { 744 ret = sysfs_add_group(&thpsize->kobj, &anon_ctrl_attr_grp); 745 if (ret) 746 goto err_put; 747 748 ret = sysfs_add_group(&thpsize->kobj, &anon_stats_attr_grp); 749 if (ret) 750 goto err_put; 751 } 752 753 if (BIT(order) & THP_ORDERS_ALL_FILE_DEFAULT) { 754 ret = sysfs_add_group(&thpsize->kobj, &file_ctrl_attr_grp); 755 if (ret) 756 goto err_put; 757 758 ret = sysfs_add_group(&thpsize->kobj, &file_stats_attr_grp); 759 if (ret) 760 goto err_put; 761 } 762 763 return thpsize; 764 err_put: 765 kobject_put(&thpsize->kobj); 766 err: 767 return ERR_PTR(ret); 768 } 769 770 static void thpsize_release(struct kobject *kobj) 771 { 772 kfree(to_thpsize(kobj)); 773 } 774 775 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj) 776 { 777 int err; 778 struct thpsize *thpsize; 779 unsigned long orders; 780 int order; 781 782 /* 783 * Default to setting PMD-sized THP to inherit the global setting and 784 * disable all other sizes. powerpc's PMD_ORDER isn't a compile-time 785 * constant so we have to do this here. 786 */ 787 if (!anon_orders_configured) 788 huge_anon_orders_inherit = BIT(PMD_ORDER); 789 790 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj); 791 if (unlikely(!*hugepage_kobj)) { 792 pr_err("failed to create transparent hugepage kobject\n"); 793 return -ENOMEM; 794 } 795 796 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group); 797 if (err) { 798 pr_err("failed to register transparent hugepage group\n"); 799 goto delete_obj; 800 } 801 802 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group); 803 if (err) { 804 pr_err("failed to register transparent hugepage group\n"); 805 goto remove_hp_group; 806 } 807 808 orders = THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_FILE_DEFAULT; 809 order = highest_order(orders); 810 while (orders) { 811 thpsize = thpsize_create(order, *hugepage_kobj); 812 if (IS_ERR(thpsize)) { 813 pr_err("failed to create thpsize for order %d\n", order); 814 err = PTR_ERR(thpsize); 815 goto remove_all; 816 } 817 list_add(&thpsize->node, &thpsize_list); 818 order = next_order(&orders, order); 819 } 820 821 return 0; 822 823 remove_all: 824 hugepage_exit_sysfs(*hugepage_kobj); 825 return err; 826 remove_hp_group: 827 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group); 828 delete_obj: 829 kobject_put(*hugepage_kobj); 830 return err; 831 } 832 833 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj) 834 { 835 struct thpsize *thpsize, *tmp; 836 837 list_for_each_entry_safe(thpsize, tmp, &thpsize_list, node) { 838 list_del(&thpsize->node); 839 kobject_put(&thpsize->kobj); 840 } 841 842 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group); 843 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group); 844 kobject_put(hugepage_kobj); 845 } 846 #else 847 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj) 848 { 849 return 0; 850 } 851 852 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) 853 { 854 } 855 #endif /* CONFIG_SYSFS */ 856 857 static int __init thp_shrinker_init(void) 858 { 859 deferred_split_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE | 860 SHRINKER_MEMCG_AWARE | 861 SHRINKER_NONSLAB, 862 "thp-deferred_split"); 863 if (!deferred_split_shrinker) 864 return -ENOMEM; 865 866 deferred_split_shrinker->count_objects = deferred_split_count; 867 deferred_split_shrinker->scan_objects = deferred_split_scan; 868 shrinker_register(deferred_split_shrinker); 869 870 if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) { 871 /* 872 * Bump the reference of the huge_zero_folio and do not 873 * initialize the shrinker. 874 * 875 * huge_zero_folio will always be NULL on failure. We assume 876 * that get_huge_zero_folio() will most likely not fail as 877 * thp_shrinker_init() is invoked early on during boot. 878 */ 879 if (!get_huge_zero_folio()) 880 pr_warn("Allocating persistent huge zero folio failed\n"); 881 return 0; 882 } 883 884 huge_zero_folio_shrinker = shrinker_alloc(0, "thp-zero"); 885 if (!huge_zero_folio_shrinker) { 886 shrinker_free(deferred_split_shrinker); 887 return -ENOMEM; 888 } 889 890 huge_zero_folio_shrinker->count_objects = shrink_huge_zero_folio_count; 891 huge_zero_folio_shrinker->scan_objects = shrink_huge_zero_folio_scan; 892 shrinker_register(huge_zero_folio_shrinker); 893 894 return 0; 895 } 896 897 static void __init thp_shrinker_exit(void) 898 { 899 shrinker_free(huge_zero_folio_shrinker); 900 shrinker_free(deferred_split_shrinker); 901 } 902 903 static int __init hugepage_init(void) 904 { 905 int err; 906 struct kobject *hugepage_kobj; 907 908 if (!has_transparent_hugepage()) { 909 transparent_hugepage_flags = 1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED; 910 return -EINVAL; 911 } 912 913 /* 914 * hugepages can't be allocated by the buddy allocator 915 */ 916 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER > MAX_PAGE_ORDER); 917 918 err = hugepage_init_sysfs(&hugepage_kobj); 919 if (err) 920 goto err_sysfs; 921 922 err = khugepaged_init(); 923 if (err) 924 goto err_slab; 925 926 err = thp_shrinker_init(); 927 if (err) 928 goto err_shrinker; 929 930 /* 931 * By default disable transparent hugepages on smaller systems, 932 * where the extra memory used could hurt more than TLB overhead 933 * is likely to save. The admin can still enable it through /sys. 934 */ 935 if (totalram_pages() < MB_TO_PAGES(512)) { 936 transparent_hugepage_flags = 0; 937 return 0; 938 } 939 940 err = start_stop_khugepaged(); 941 if (err) 942 goto err_khugepaged; 943 944 return 0; 945 err_khugepaged: 946 thp_shrinker_exit(); 947 err_shrinker: 948 khugepaged_destroy(); 949 err_slab: 950 hugepage_exit_sysfs(hugepage_kobj); 951 err_sysfs: 952 return err; 953 } 954 subsys_initcall(hugepage_init); 955 956 static int __init setup_transparent_hugepage(char *str) 957 { 958 int ret = 0; 959 if (!str) 960 goto out; 961 if (!strcmp(str, "always")) { 962 set_bit(TRANSPARENT_HUGEPAGE_FLAG, 963 &transparent_hugepage_flags); 964 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 965 &transparent_hugepage_flags); 966 ret = 1; 967 } else if (!strcmp(str, "madvise")) { 968 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 969 &transparent_hugepage_flags); 970 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 971 &transparent_hugepage_flags); 972 ret = 1; 973 } else if (!strcmp(str, "never")) { 974 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 975 &transparent_hugepage_flags); 976 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 977 &transparent_hugepage_flags); 978 ret = 1; 979 } 980 out: 981 if (!ret) 982 pr_warn("transparent_hugepage= cannot parse, ignored\n"); 983 return ret; 984 } 985 __setup("transparent_hugepage=", setup_transparent_hugepage); 986 987 static char str_dup[PAGE_SIZE] __initdata; 988 static int __init setup_thp_anon(char *str) 989 { 990 char *token, *range, *policy, *subtoken; 991 unsigned long always, inherit, madvise; 992 char *start_size, *end_size; 993 int start, end, nr; 994 char *p; 995 996 if (!str || strlen(str) + 1 > PAGE_SIZE) 997 goto err; 998 strscpy(str_dup, str); 999 1000 always = huge_anon_orders_always; 1001 madvise = huge_anon_orders_madvise; 1002 inherit = huge_anon_orders_inherit; 1003 p = str_dup; 1004 while ((token = strsep(&p, ";")) != NULL) { 1005 range = strsep(&token, ":"); 1006 policy = token; 1007 1008 if (!policy) 1009 goto err; 1010 1011 while ((subtoken = strsep(&range, ",")) != NULL) { 1012 if (strchr(subtoken, '-')) { 1013 start_size = strsep(&subtoken, "-"); 1014 end_size = subtoken; 1015 1016 start = get_order_from_str(start_size, THP_ORDERS_ALL_ANON); 1017 end = get_order_from_str(end_size, THP_ORDERS_ALL_ANON); 1018 } else { 1019 start_size = end_size = subtoken; 1020 start = end = get_order_from_str(subtoken, 1021 THP_ORDERS_ALL_ANON); 1022 } 1023 1024 if (start == -EINVAL) { 1025 pr_err("invalid size %s in thp_anon boot parameter\n", start_size); 1026 goto err; 1027 } 1028 1029 if (end == -EINVAL) { 1030 pr_err("invalid size %s in thp_anon boot parameter\n", end_size); 1031 goto err; 1032 } 1033 1034 if (start < 0 || end < 0 || start > end) 1035 goto err; 1036 1037 nr = end - start + 1; 1038 if (!strcmp(policy, "always")) { 1039 bitmap_set(&always, start, nr); 1040 bitmap_clear(&inherit, start, nr); 1041 bitmap_clear(&madvise, start, nr); 1042 } else if (!strcmp(policy, "madvise")) { 1043 bitmap_set(&madvise, start, nr); 1044 bitmap_clear(&inherit, start, nr); 1045 bitmap_clear(&always, start, nr); 1046 } else if (!strcmp(policy, "inherit")) { 1047 bitmap_set(&inherit, start, nr); 1048 bitmap_clear(&madvise, start, nr); 1049 bitmap_clear(&always, start, nr); 1050 } else if (!strcmp(policy, "never")) { 1051 bitmap_clear(&inherit, start, nr); 1052 bitmap_clear(&madvise, start, nr); 1053 bitmap_clear(&always, start, nr); 1054 } else { 1055 pr_err("invalid policy %s in thp_anon boot parameter\n", policy); 1056 goto err; 1057 } 1058 } 1059 } 1060 1061 huge_anon_orders_always = always; 1062 huge_anon_orders_madvise = madvise; 1063 huge_anon_orders_inherit = inherit; 1064 anon_orders_configured = true; 1065 return 1; 1066 1067 err: 1068 pr_warn("thp_anon=%s: error parsing string, ignoring setting\n", str); 1069 return 0; 1070 } 1071 __setup("thp_anon=", setup_thp_anon); 1072 1073 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 1074 { 1075 if (likely(vma->vm_flags & VM_WRITE)) 1076 pmd = pmd_mkwrite(pmd, vma); 1077 return pmd; 1078 } 1079 1080 #ifdef CONFIG_MEMCG 1081 static inline 1082 struct deferred_split *get_deferred_split_queue(struct folio *folio) 1083 { 1084 struct mem_cgroup *memcg = folio_memcg(folio); 1085 struct pglist_data *pgdat = NODE_DATA(folio_nid(folio)); 1086 1087 if (memcg) 1088 return &memcg->deferred_split_queue; 1089 else 1090 return &pgdat->deferred_split_queue; 1091 } 1092 #else 1093 static inline 1094 struct deferred_split *get_deferred_split_queue(struct folio *folio) 1095 { 1096 struct pglist_data *pgdat = NODE_DATA(folio_nid(folio)); 1097 1098 return &pgdat->deferred_split_queue; 1099 } 1100 #endif 1101 1102 static inline bool is_transparent_hugepage(const struct folio *folio) 1103 { 1104 if (!folio_test_large(folio)) 1105 return false; 1106 1107 return is_huge_zero_folio(folio) || 1108 folio_test_large_rmappable(folio); 1109 } 1110 1111 static unsigned long __thp_get_unmapped_area(struct file *filp, 1112 unsigned long addr, unsigned long len, 1113 loff_t off, unsigned long flags, unsigned long size, 1114 vm_flags_t vm_flags) 1115 { 1116 loff_t off_end = off + len; 1117 loff_t off_align = round_up(off, size); 1118 unsigned long len_pad, ret, off_sub; 1119 1120 if (!IS_ENABLED(CONFIG_64BIT) || in_compat_syscall()) 1121 return 0; 1122 1123 if (off_end <= off_align || (off_end - off_align) < size) 1124 return 0; 1125 1126 len_pad = len + size; 1127 if (len_pad < len || (off + len_pad) < off) 1128 return 0; 1129 1130 ret = mm_get_unmapped_area_vmflags(filp, addr, len_pad, 1131 off >> PAGE_SHIFT, flags, vm_flags); 1132 1133 /* 1134 * The failure might be due to length padding. The caller will retry 1135 * without the padding. 1136 */ 1137 if (IS_ERR_VALUE(ret)) 1138 return 0; 1139 1140 /* 1141 * Do not try to align to THP boundary if allocation at the address 1142 * hint succeeds. 1143 */ 1144 if (ret == addr) 1145 return addr; 1146 1147 off_sub = (off - ret) & (size - 1); 1148 1149 if (mm_flags_test(MMF_TOPDOWN, current->mm) && !off_sub) 1150 return ret + size; 1151 1152 ret += off_sub; 1153 return ret; 1154 } 1155 1156 unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, 1157 unsigned long len, unsigned long pgoff, unsigned long flags, 1158 vm_flags_t vm_flags) 1159 { 1160 unsigned long ret; 1161 loff_t off = (loff_t)pgoff << PAGE_SHIFT; 1162 1163 ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, vm_flags); 1164 if (ret) 1165 return ret; 1166 1167 return mm_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 1168 vm_flags); 1169 } 1170 1171 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, 1172 unsigned long len, unsigned long pgoff, unsigned long flags) 1173 { 1174 return thp_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0); 1175 } 1176 EXPORT_SYMBOL_GPL(thp_get_unmapped_area); 1177 1178 static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma, 1179 unsigned long addr) 1180 { 1181 gfp_t gfp = vma_thp_gfp_mask(vma); 1182 const int order = HPAGE_PMD_ORDER; 1183 struct folio *folio; 1184 1185 folio = vma_alloc_folio(gfp, order, vma, addr & HPAGE_PMD_MASK); 1186 1187 if (unlikely(!folio)) { 1188 count_vm_event(THP_FAULT_FALLBACK); 1189 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK); 1190 return NULL; 1191 } 1192 1193 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 1194 if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) { 1195 folio_put(folio); 1196 count_vm_event(THP_FAULT_FALLBACK); 1197 count_vm_event(THP_FAULT_FALLBACK_CHARGE); 1198 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK); 1199 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); 1200 return NULL; 1201 } 1202 folio_throttle_swaprate(folio, gfp); 1203 1204 /* 1205 * When a folio is not zeroed during allocation (__GFP_ZERO not used) 1206 * or user folios require special handling, folio_zero_user() is used to 1207 * make sure that the page corresponding to the faulting address will be 1208 * hot in the cache after zeroing. 1209 */ 1210 if (user_alloc_needs_zeroing()) 1211 folio_zero_user(folio, addr); 1212 /* 1213 * The memory barrier inside __folio_mark_uptodate makes sure that 1214 * folio_zero_user writes become visible before the set_pmd_at() 1215 * write. 1216 */ 1217 __folio_mark_uptodate(folio); 1218 return folio; 1219 } 1220 1221 static void map_anon_folio_pmd(struct folio *folio, pmd_t *pmd, 1222 struct vm_area_struct *vma, unsigned long haddr) 1223 { 1224 pmd_t entry; 1225 1226 entry = folio_mk_pmd(folio, vma->vm_page_prot); 1227 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1228 folio_add_new_anon_rmap(folio, vma, haddr, RMAP_EXCLUSIVE); 1229 folio_add_lru_vma(folio, vma); 1230 set_pmd_at(vma->vm_mm, haddr, pmd, entry); 1231 update_mmu_cache_pmd(vma, haddr, pmd); 1232 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1233 count_vm_event(THP_FAULT_ALLOC); 1234 count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_ALLOC); 1235 count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); 1236 } 1237 1238 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf) 1239 { 1240 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1241 struct vm_area_struct *vma = vmf->vma; 1242 struct folio *folio; 1243 pgtable_t pgtable; 1244 vm_fault_t ret = 0; 1245 1246 folio = vma_alloc_anon_folio_pmd(vma, vmf->address); 1247 if (unlikely(!folio)) 1248 return VM_FAULT_FALLBACK; 1249 1250 pgtable = pte_alloc_one(vma->vm_mm); 1251 if (unlikely(!pgtable)) { 1252 ret = VM_FAULT_OOM; 1253 goto release; 1254 } 1255 1256 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1257 if (unlikely(!pmd_none(*vmf->pmd))) { 1258 goto unlock_release; 1259 } else { 1260 ret = check_stable_address_space(vma->vm_mm); 1261 if (ret) 1262 goto unlock_release; 1263 1264 /* Deliver the page fault to userland */ 1265 if (userfaultfd_missing(vma)) { 1266 spin_unlock(vmf->ptl); 1267 folio_put(folio); 1268 pte_free(vma->vm_mm, pgtable); 1269 ret = handle_userfault(vmf, VM_UFFD_MISSING); 1270 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 1271 return ret; 1272 } 1273 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); 1274 map_anon_folio_pmd(folio, vmf->pmd, vma, haddr); 1275 mm_inc_nr_ptes(vma->vm_mm); 1276 deferred_split_folio(folio, false); 1277 spin_unlock(vmf->ptl); 1278 } 1279 1280 return 0; 1281 unlock_release: 1282 spin_unlock(vmf->ptl); 1283 release: 1284 if (pgtable) 1285 pte_free(vma->vm_mm, pgtable); 1286 folio_put(folio); 1287 return ret; 1288 1289 } 1290 1291 /* 1292 * always: directly stall for all thp allocations 1293 * defer: wake kswapd and fail if not immediately available 1294 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise 1295 * fail if not immediately available 1296 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately 1297 * available 1298 * never: never stall for any thp allocation 1299 */ 1300 gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma) 1301 { 1302 const bool vma_madvised = vma && (vma->vm_flags & VM_HUGEPAGE); 1303 1304 /* Always do synchronous compaction */ 1305 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) 1306 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY); 1307 1308 /* Kick kcompactd and fail quickly */ 1309 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) 1310 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM; 1311 1312 /* Synchronous compaction if madvised, otherwise kick kcompactd */ 1313 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags)) 1314 return GFP_TRANSHUGE_LIGHT | 1315 (vma_madvised ? __GFP_DIRECT_RECLAIM : 1316 __GFP_KSWAPD_RECLAIM); 1317 1318 /* Only do synchronous compaction if madvised */ 1319 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags)) 1320 return GFP_TRANSHUGE_LIGHT | 1321 (vma_madvised ? __GFP_DIRECT_RECLAIM : 0); 1322 1323 return GFP_TRANSHUGE_LIGHT; 1324 } 1325 1326 /* Caller must hold page table lock. */ 1327 static void set_huge_zero_folio(pgtable_t pgtable, struct mm_struct *mm, 1328 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, 1329 struct folio *zero_folio) 1330 { 1331 pmd_t entry; 1332 entry = folio_mk_pmd(zero_folio, vma->vm_page_prot); 1333 entry = pmd_mkspecial(entry); 1334 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1335 set_pmd_at(mm, haddr, pmd, entry); 1336 mm_inc_nr_ptes(mm); 1337 } 1338 1339 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) 1340 { 1341 struct vm_area_struct *vma = vmf->vma; 1342 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1343 vm_fault_t ret; 1344 1345 if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER)) 1346 return VM_FAULT_FALLBACK; 1347 ret = vmf_anon_prepare(vmf); 1348 if (ret) 1349 return ret; 1350 khugepaged_enter_vma(vma, vma->vm_flags); 1351 1352 if (!(vmf->flags & FAULT_FLAG_WRITE) && 1353 !mm_forbids_zeropage(vma->vm_mm) && 1354 transparent_hugepage_use_zero_page()) { 1355 pgtable_t pgtable; 1356 struct folio *zero_folio; 1357 vm_fault_t ret; 1358 1359 pgtable = pte_alloc_one(vma->vm_mm); 1360 if (unlikely(!pgtable)) 1361 return VM_FAULT_OOM; 1362 zero_folio = mm_get_huge_zero_folio(vma->vm_mm); 1363 if (unlikely(!zero_folio)) { 1364 pte_free(vma->vm_mm, pgtable); 1365 count_vm_event(THP_FAULT_FALLBACK); 1366 return VM_FAULT_FALLBACK; 1367 } 1368 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1369 ret = 0; 1370 if (pmd_none(*vmf->pmd)) { 1371 ret = check_stable_address_space(vma->vm_mm); 1372 if (ret) { 1373 spin_unlock(vmf->ptl); 1374 pte_free(vma->vm_mm, pgtable); 1375 } else if (userfaultfd_missing(vma)) { 1376 spin_unlock(vmf->ptl); 1377 pte_free(vma->vm_mm, pgtable); 1378 ret = handle_userfault(vmf, VM_UFFD_MISSING); 1379 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 1380 } else { 1381 set_huge_zero_folio(pgtable, vma->vm_mm, vma, 1382 haddr, vmf->pmd, zero_folio); 1383 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1384 spin_unlock(vmf->ptl); 1385 } 1386 } else { 1387 spin_unlock(vmf->ptl); 1388 pte_free(vma->vm_mm, pgtable); 1389 } 1390 return ret; 1391 } 1392 1393 return __do_huge_pmd_anonymous_page(vmf); 1394 } 1395 1396 struct folio_or_pfn { 1397 union { 1398 struct folio *folio; 1399 unsigned long pfn; 1400 }; 1401 bool is_folio; 1402 }; 1403 1404 static vm_fault_t insert_pmd(struct vm_area_struct *vma, unsigned long addr, 1405 pmd_t *pmd, struct folio_or_pfn fop, pgprot_t prot, 1406 bool write) 1407 { 1408 struct mm_struct *mm = vma->vm_mm; 1409 pgtable_t pgtable = NULL; 1410 spinlock_t *ptl; 1411 pmd_t entry; 1412 1413 if (addr < vma->vm_start || addr >= vma->vm_end) 1414 return VM_FAULT_SIGBUS; 1415 1416 if (arch_needs_pgtable_deposit()) { 1417 pgtable = pte_alloc_one(vma->vm_mm); 1418 if (!pgtable) 1419 return VM_FAULT_OOM; 1420 } 1421 1422 ptl = pmd_lock(mm, pmd); 1423 if (!pmd_none(*pmd)) { 1424 const unsigned long pfn = fop.is_folio ? folio_pfn(fop.folio) : 1425 fop.pfn; 1426 1427 if (write) { 1428 if (pmd_pfn(*pmd) != pfn) { 1429 WARN_ON_ONCE(!is_huge_zero_pmd(*pmd)); 1430 goto out_unlock; 1431 } 1432 entry = pmd_mkyoung(*pmd); 1433 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1434 if (pmdp_set_access_flags(vma, addr, pmd, entry, 1)) 1435 update_mmu_cache_pmd(vma, addr, pmd); 1436 } 1437 goto out_unlock; 1438 } 1439 1440 if (fop.is_folio) { 1441 entry = folio_mk_pmd(fop.folio, vma->vm_page_prot); 1442 1443 if (is_huge_zero_folio(fop.folio)) { 1444 entry = pmd_mkspecial(entry); 1445 } else { 1446 folio_get(fop.folio); 1447 folio_add_file_rmap_pmd(fop.folio, &fop.folio->page, vma); 1448 add_mm_counter(mm, mm_counter_file(fop.folio), HPAGE_PMD_NR); 1449 } 1450 } else { 1451 entry = pmd_mkhuge(pfn_pmd(fop.pfn, prot)); 1452 entry = pmd_mkspecial(entry); 1453 } 1454 if (write) { 1455 entry = pmd_mkyoung(pmd_mkdirty(entry)); 1456 entry = maybe_pmd_mkwrite(entry, vma); 1457 } 1458 1459 if (pgtable) { 1460 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1461 mm_inc_nr_ptes(mm); 1462 pgtable = NULL; 1463 } 1464 1465 set_pmd_at(mm, addr, pmd, entry); 1466 update_mmu_cache_pmd(vma, addr, pmd); 1467 1468 out_unlock: 1469 spin_unlock(ptl); 1470 if (pgtable) 1471 pte_free(mm, pgtable); 1472 return VM_FAULT_NOPAGE; 1473 } 1474 1475 /** 1476 * vmf_insert_pfn_pmd - insert a pmd size pfn 1477 * @vmf: Structure describing the fault 1478 * @pfn: pfn to insert 1479 * @write: whether it's a write fault 1480 * 1481 * Insert a pmd size pfn. See vmf_insert_pfn() for additional info. 1482 * 1483 * Return: vm_fault_t value. 1484 */ 1485 vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, unsigned long pfn, 1486 bool write) 1487 { 1488 unsigned long addr = vmf->address & PMD_MASK; 1489 struct vm_area_struct *vma = vmf->vma; 1490 pgprot_t pgprot = vma->vm_page_prot; 1491 struct folio_or_pfn fop = { 1492 .pfn = pfn, 1493 }; 1494 1495 /* 1496 * If we had pmd_special, we could avoid all these restrictions, 1497 * but we need to be consistent with PTEs and architectures that 1498 * can't support a 'special' bit. 1499 */ 1500 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 1501 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1502 (VM_PFNMAP|VM_MIXEDMAP)); 1503 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1504 1505 pfnmap_setup_cachemode_pfn(pfn, &pgprot); 1506 1507 return insert_pmd(vma, addr, vmf->pmd, fop, pgprot, write); 1508 } 1509 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd); 1510 1511 vm_fault_t vmf_insert_folio_pmd(struct vm_fault *vmf, struct folio *folio, 1512 bool write) 1513 { 1514 struct vm_area_struct *vma = vmf->vma; 1515 unsigned long addr = vmf->address & PMD_MASK; 1516 struct folio_or_pfn fop = { 1517 .folio = folio, 1518 .is_folio = true, 1519 }; 1520 1521 if (WARN_ON_ONCE(folio_order(folio) != PMD_ORDER)) 1522 return VM_FAULT_SIGBUS; 1523 1524 return insert_pmd(vma, addr, vmf->pmd, fop, vma->vm_page_prot, write); 1525 } 1526 EXPORT_SYMBOL_GPL(vmf_insert_folio_pmd); 1527 1528 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1529 static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma) 1530 { 1531 if (likely(vma->vm_flags & VM_WRITE)) 1532 pud = pud_mkwrite(pud); 1533 return pud; 1534 } 1535 1536 static vm_fault_t insert_pud(struct vm_area_struct *vma, unsigned long addr, 1537 pud_t *pud, struct folio_or_pfn fop, pgprot_t prot, bool write) 1538 { 1539 struct mm_struct *mm = vma->vm_mm; 1540 spinlock_t *ptl; 1541 pud_t entry; 1542 1543 if (addr < vma->vm_start || addr >= vma->vm_end) 1544 return VM_FAULT_SIGBUS; 1545 1546 ptl = pud_lock(mm, pud); 1547 if (!pud_none(*pud)) { 1548 const unsigned long pfn = fop.is_folio ? folio_pfn(fop.folio) : 1549 fop.pfn; 1550 1551 if (write) { 1552 if (WARN_ON_ONCE(pud_pfn(*pud) != pfn)) 1553 goto out_unlock; 1554 entry = pud_mkyoung(*pud); 1555 entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma); 1556 if (pudp_set_access_flags(vma, addr, pud, entry, 1)) 1557 update_mmu_cache_pud(vma, addr, pud); 1558 } 1559 goto out_unlock; 1560 } 1561 1562 if (fop.is_folio) { 1563 entry = folio_mk_pud(fop.folio, vma->vm_page_prot); 1564 1565 folio_get(fop.folio); 1566 folio_add_file_rmap_pud(fop.folio, &fop.folio->page, vma); 1567 add_mm_counter(mm, mm_counter_file(fop.folio), HPAGE_PUD_NR); 1568 } else { 1569 entry = pud_mkhuge(pfn_pud(fop.pfn, prot)); 1570 entry = pud_mkspecial(entry); 1571 } 1572 if (write) { 1573 entry = pud_mkyoung(pud_mkdirty(entry)); 1574 entry = maybe_pud_mkwrite(entry, vma); 1575 } 1576 set_pud_at(mm, addr, pud, entry); 1577 update_mmu_cache_pud(vma, addr, pud); 1578 out_unlock: 1579 spin_unlock(ptl); 1580 return VM_FAULT_NOPAGE; 1581 } 1582 1583 /** 1584 * vmf_insert_pfn_pud - insert a pud size pfn 1585 * @vmf: Structure describing the fault 1586 * @pfn: pfn to insert 1587 * @write: whether it's a write fault 1588 * 1589 * Insert a pud size pfn. See vmf_insert_pfn() for additional info. 1590 * 1591 * Return: vm_fault_t value. 1592 */ 1593 vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, unsigned long pfn, 1594 bool write) 1595 { 1596 unsigned long addr = vmf->address & PUD_MASK; 1597 struct vm_area_struct *vma = vmf->vma; 1598 pgprot_t pgprot = vma->vm_page_prot; 1599 struct folio_or_pfn fop = { 1600 .pfn = pfn, 1601 }; 1602 1603 /* 1604 * If we had pud_special, we could avoid all these restrictions, 1605 * but we need to be consistent with PTEs and architectures that 1606 * can't support a 'special' bit. 1607 */ 1608 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 1609 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1610 (VM_PFNMAP|VM_MIXEDMAP)); 1611 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1612 1613 pfnmap_setup_cachemode_pfn(pfn, &pgprot); 1614 1615 return insert_pud(vma, addr, vmf->pud, fop, pgprot, write); 1616 } 1617 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud); 1618 1619 /** 1620 * vmf_insert_folio_pud - insert a pud size folio mapped by a pud entry 1621 * @vmf: Structure describing the fault 1622 * @folio: folio to insert 1623 * @write: whether it's a write fault 1624 * 1625 * Return: vm_fault_t value. 1626 */ 1627 vm_fault_t vmf_insert_folio_pud(struct vm_fault *vmf, struct folio *folio, 1628 bool write) 1629 { 1630 struct vm_area_struct *vma = vmf->vma; 1631 unsigned long addr = vmf->address & PUD_MASK; 1632 struct folio_or_pfn fop = { 1633 .folio = folio, 1634 .is_folio = true, 1635 }; 1636 1637 if (WARN_ON_ONCE(folio_order(folio) != PUD_ORDER)) 1638 return VM_FAULT_SIGBUS; 1639 1640 return insert_pud(vma, addr, vmf->pud, fop, vma->vm_page_prot, write); 1641 } 1642 EXPORT_SYMBOL_GPL(vmf_insert_folio_pud); 1643 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1644 1645 void touch_pmd(struct vm_area_struct *vma, unsigned long addr, 1646 pmd_t *pmd, bool write) 1647 { 1648 pmd_t _pmd; 1649 1650 _pmd = pmd_mkyoung(*pmd); 1651 if (write) 1652 _pmd = pmd_mkdirty(_pmd); 1653 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, 1654 pmd, _pmd, write)) 1655 update_mmu_cache_pmd(vma, addr, pmd); 1656 } 1657 1658 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1659 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 1660 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) 1661 { 1662 spinlock_t *dst_ptl, *src_ptl; 1663 struct page *src_page; 1664 struct folio *src_folio; 1665 pmd_t pmd; 1666 pgtable_t pgtable = NULL; 1667 int ret = -ENOMEM; 1668 1669 pmd = pmdp_get_lockless(src_pmd); 1670 if (unlikely(pmd_present(pmd) && pmd_special(pmd) && 1671 !is_huge_zero_pmd(pmd))) { 1672 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1673 src_ptl = pmd_lockptr(src_mm, src_pmd); 1674 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1675 /* 1676 * No need to recheck the pmd, it can't change with write 1677 * mmap lock held here. 1678 * 1679 * Meanwhile, making sure it's not a CoW VMA with writable 1680 * mapping, otherwise it means either the anon page wrongly 1681 * applied special bit, or we made the PRIVATE mapping be 1682 * able to wrongly write to the backend MMIO. 1683 */ 1684 VM_WARN_ON_ONCE(is_cow_mapping(src_vma->vm_flags) && pmd_write(pmd)); 1685 goto set_pmd; 1686 } 1687 1688 /* Skip if can be re-fill on fault */ 1689 if (!vma_is_anonymous(dst_vma)) 1690 return 0; 1691 1692 pgtable = pte_alloc_one(dst_mm); 1693 if (unlikely(!pgtable)) 1694 goto out; 1695 1696 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1697 src_ptl = pmd_lockptr(src_mm, src_pmd); 1698 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1699 1700 ret = -EAGAIN; 1701 pmd = *src_pmd; 1702 1703 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 1704 if (unlikely(is_swap_pmd(pmd))) { 1705 swp_entry_t entry = pmd_to_swp_entry(pmd); 1706 1707 VM_BUG_ON(!is_pmd_migration_entry(pmd)); 1708 if (!is_readable_migration_entry(entry)) { 1709 entry = make_readable_migration_entry( 1710 swp_offset(entry)); 1711 pmd = swp_entry_to_pmd(entry); 1712 if (pmd_swp_soft_dirty(*src_pmd)) 1713 pmd = pmd_swp_mksoft_dirty(pmd); 1714 if (pmd_swp_uffd_wp(*src_pmd)) 1715 pmd = pmd_swp_mkuffd_wp(pmd); 1716 set_pmd_at(src_mm, addr, src_pmd, pmd); 1717 } 1718 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1719 mm_inc_nr_ptes(dst_mm); 1720 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1721 if (!userfaultfd_wp(dst_vma)) 1722 pmd = pmd_swp_clear_uffd_wp(pmd); 1723 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1724 ret = 0; 1725 goto out_unlock; 1726 } 1727 #endif 1728 1729 if (unlikely(!pmd_trans_huge(pmd))) { 1730 pte_free(dst_mm, pgtable); 1731 goto out_unlock; 1732 } 1733 /* 1734 * When page table lock is held, the huge zero pmd should not be 1735 * under splitting since we don't split the page itself, only pmd to 1736 * a page table. 1737 */ 1738 if (is_huge_zero_pmd(pmd)) { 1739 /* 1740 * mm_get_huge_zero_folio() will never allocate a new 1741 * folio here, since we already have a zero page to 1742 * copy. It just takes a reference. 1743 */ 1744 mm_get_huge_zero_folio(dst_mm); 1745 goto out_zero_page; 1746 } 1747 1748 src_page = pmd_page(pmd); 1749 VM_BUG_ON_PAGE(!PageHead(src_page), src_page); 1750 src_folio = page_folio(src_page); 1751 1752 folio_get(src_folio); 1753 if (unlikely(folio_try_dup_anon_rmap_pmd(src_folio, src_page, dst_vma, src_vma))) { 1754 /* Page maybe pinned: split and retry the fault on PTEs. */ 1755 folio_put(src_folio); 1756 pte_free(dst_mm, pgtable); 1757 spin_unlock(src_ptl); 1758 spin_unlock(dst_ptl); 1759 __split_huge_pmd(src_vma, src_pmd, addr, false); 1760 return -EAGAIN; 1761 } 1762 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1763 out_zero_page: 1764 mm_inc_nr_ptes(dst_mm); 1765 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1766 pmdp_set_wrprotect(src_mm, addr, src_pmd); 1767 if (!userfaultfd_wp(dst_vma)) 1768 pmd = pmd_clear_uffd_wp(pmd); 1769 pmd = pmd_wrprotect(pmd); 1770 set_pmd: 1771 pmd = pmd_mkold(pmd); 1772 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1773 1774 ret = 0; 1775 out_unlock: 1776 spin_unlock(src_ptl); 1777 spin_unlock(dst_ptl); 1778 out: 1779 return ret; 1780 } 1781 1782 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1783 void touch_pud(struct vm_area_struct *vma, unsigned long addr, 1784 pud_t *pud, bool write) 1785 { 1786 pud_t _pud; 1787 1788 _pud = pud_mkyoung(*pud); 1789 if (write) 1790 _pud = pud_mkdirty(_pud); 1791 if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK, 1792 pud, _pud, write)) 1793 update_mmu_cache_pud(vma, addr, pud); 1794 } 1795 1796 int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1797 pud_t *dst_pud, pud_t *src_pud, unsigned long addr, 1798 struct vm_area_struct *vma) 1799 { 1800 spinlock_t *dst_ptl, *src_ptl; 1801 pud_t pud; 1802 int ret; 1803 1804 dst_ptl = pud_lock(dst_mm, dst_pud); 1805 src_ptl = pud_lockptr(src_mm, src_pud); 1806 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1807 1808 ret = -EAGAIN; 1809 pud = *src_pud; 1810 if (unlikely(!pud_trans_huge(pud))) 1811 goto out_unlock; 1812 1813 /* 1814 * TODO: once we support anonymous pages, use 1815 * folio_try_dup_anon_rmap_*() and split if duplicating fails. 1816 */ 1817 if (is_cow_mapping(vma->vm_flags) && pud_write(pud)) { 1818 pudp_set_wrprotect(src_mm, addr, src_pud); 1819 pud = pud_wrprotect(pud); 1820 } 1821 pud = pud_mkold(pud); 1822 set_pud_at(dst_mm, addr, dst_pud, pud); 1823 1824 ret = 0; 1825 out_unlock: 1826 spin_unlock(src_ptl); 1827 spin_unlock(dst_ptl); 1828 return ret; 1829 } 1830 1831 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud) 1832 { 1833 bool write = vmf->flags & FAULT_FLAG_WRITE; 1834 1835 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); 1836 if (unlikely(!pud_same(*vmf->pud, orig_pud))) 1837 goto unlock; 1838 1839 touch_pud(vmf->vma, vmf->address, vmf->pud, write); 1840 unlock: 1841 spin_unlock(vmf->ptl); 1842 } 1843 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1844 1845 void huge_pmd_set_accessed(struct vm_fault *vmf) 1846 { 1847 bool write = vmf->flags & FAULT_FLAG_WRITE; 1848 1849 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); 1850 if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd))) 1851 goto unlock; 1852 1853 touch_pmd(vmf->vma, vmf->address, vmf->pmd, write); 1854 1855 unlock: 1856 spin_unlock(vmf->ptl); 1857 } 1858 1859 static vm_fault_t do_huge_zero_wp_pmd(struct vm_fault *vmf) 1860 { 1861 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1862 struct vm_area_struct *vma = vmf->vma; 1863 struct mmu_notifier_range range; 1864 struct folio *folio; 1865 vm_fault_t ret = 0; 1866 1867 folio = vma_alloc_anon_folio_pmd(vma, vmf->address); 1868 if (unlikely(!folio)) 1869 return VM_FAULT_FALLBACK; 1870 1871 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, haddr, 1872 haddr + HPAGE_PMD_SIZE); 1873 mmu_notifier_invalidate_range_start(&range); 1874 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1875 if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) 1876 goto release; 1877 ret = check_stable_address_space(vma->vm_mm); 1878 if (ret) 1879 goto release; 1880 (void)pmdp_huge_clear_flush(vma, haddr, vmf->pmd); 1881 map_anon_folio_pmd(folio, vmf->pmd, vma, haddr); 1882 goto unlock; 1883 release: 1884 folio_put(folio); 1885 unlock: 1886 spin_unlock(vmf->ptl); 1887 mmu_notifier_invalidate_range_end(&range); 1888 return ret; 1889 } 1890 1891 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) 1892 { 1893 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; 1894 struct vm_area_struct *vma = vmf->vma; 1895 struct folio *folio; 1896 struct page *page; 1897 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1898 pmd_t orig_pmd = vmf->orig_pmd; 1899 1900 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); 1901 VM_BUG_ON_VMA(!vma->anon_vma, vma); 1902 1903 if (is_huge_zero_pmd(orig_pmd)) { 1904 vm_fault_t ret = do_huge_zero_wp_pmd(vmf); 1905 1906 if (!(ret & VM_FAULT_FALLBACK)) 1907 return ret; 1908 1909 /* Fallback to splitting PMD if THP cannot be allocated */ 1910 goto fallback; 1911 } 1912 1913 spin_lock(vmf->ptl); 1914 1915 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1916 spin_unlock(vmf->ptl); 1917 return 0; 1918 } 1919 1920 page = pmd_page(orig_pmd); 1921 folio = page_folio(page); 1922 VM_BUG_ON_PAGE(!PageHead(page), page); 1923 1924 /* Early check when only holding the PT lock. */ 1925 if (PageAnonExclusive(page)) 1926 goto reuse; 1927 1928 if (!folio_trylock(folio)) { 1929 folio_get(folio); 1930 spin_unlock(vmf->ptl); 1931 folio_lock(folio); 1932 spin_lock(vmf->ptl); 1933 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1934 spin_unlock(vmf->ptl); 1935 folio_unlock(folio); 1936 folio_put(folio); 1937 return 0; 1938 } 1939 folio_put(folio); 1940 } 1941 1942 /* Recheck after temporarily dropping the PT lock. */ 1943 if (PageAnonExclusive(page)) { 1944 folio_unlock(folio); 1945 goto reuse; 1946 } 1947 1948 /* 1949 * See do_wp_page(): we can only reuse the folio exclusively if 1950 * there are no additional references. Note that we always drain 1951 * the LRU cache immediately after adding a THP. 1952 */ 1953 if (folio_ref_count(folio) > 1954 1 + folio_test_swapcache(folio) * folio_nr_pages(folio)) 1955 goto unlock_fallback; 1956 if (folio_test_swapcache(folio)) 1957 folio_free_swap(folio); 1958 if (folio_ref_count(folio) == 1) { 1959 pmd_t entry; 1960 1961 folio_move_anon_rmap(folio, vma); 1962 SetPageAnonExclusive(page); 1963 folio_unlock(folio); 1964 reuse: 1965 if (unlikely(unshare)) { 1966 spin_unlock(vmf->ptl); 1967 return 0; 1968 } 1969 entry = pmd_mkyoung(orig_pmd); 1970 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1971 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) 1972 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1973 spin_unlock(vmf->ptl); 1974 return 0; 1975 } 1976 1977 unlock_fallback: 1978 folio_unlock(folio); 1979 spin_unlock(vmf->ptl); 1980 fallback: 1981 __split_huge_pmd(vma, vmf->pmd, vmf->address, false); 1982 return VM_FAULT_FALLBACK; 1983 } 1984 1985 static inline bool can_change_pmd_writable(struct vm_area_struct *vma, 1986 unsigned long addr, pmd_t pmd) 1987 { 1988 struct page *page; 1989 1990 if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) 1991 return false; 1992 1993 /* Don't touch entries that are not even readable (NUMA hinting). */ 1994 if (pmd_protnone(pmd)) 1995 return false; 1996 1997 /* Do we need write faults for softdirty tracking? */ 1998 if (pmd_needs_soft_dirty_wp(vma, pmd)) 1999 return false; 2000 2001 /* Do we need write faults for uffd-wp tracking? */ 2002 if (userfaultfd_huge_pmd_wp(vma, pmd)) 2003 return false; 2004 2005 if (!(vma->vm_flags & VM_SHARED)) { 2006 /* See can_change_pte_writable(). */ 2007 page = vm_normal_page_pmd(vma, addr, pmd); 2008 return page && PageAnon(page) && PageAnonExclusive(page); 2009 } 2010 2011 /* See can_change_pte_writable(). */ 2012 return pmd_dirty(pmd); 2013 } 2014 2015 /* NUMA hinting page fault entry point for trans huge pmds */ 2016 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) 2017 { 2018 struct vm_area_struct *vma = vmf->vma; 2019 struct folio *folio; 2020 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 2021 int nid = NUMA_NO_NODE; 2022 int target_nid, last_cpupid; 2023 pmd_t pmd, old_pmd; 2024 bool writable = false; 2025 int flags = 0; 2026 2027 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 2028 old_pmd = pmdp_get(vmf->pmd); 2029 2030 if (unlikely(!pmd_same(old_pmd, vmf->orig_pmd))) { 2031 spin_unlock(vmf->ptl); 2032 return 0; 2033 } 2034 2035 pmd = pmd_modify(old_pmd, vma->vm_page_prot); 2036 2037 /* 2038 * Detect now whether the PMD could be writable; this information 2039 * is only valid while holding the PT lock. 2040 */ 2041 writable = pmd_write(pmd); 2042 if (!writable && vma_wants_manual_pte_write_upgrade(vma) && 2043 can_change_pmd_writable(vma, vmf->address, pmd)) 2044 writable = true; 2045 2046 folio = vm_normal_folio_pmd(vma, haddr, pmd); 2047 if (!folio) 2048 goto out_map; 2049 2050 nid = folio_nid(folio); 2051 2052 target_nid = numa_migrate_check(folio, vmf, haddr, &flags, writable, 2053 &last_cpupid); 2054 if (target_nid == NUMA_NO_NODE) 2055 goto out_map; 2056 if (migrate_misplaced_folio_prepare(folio, vma, target_nid)) { 2057 flags |= TNF_MIGRATE_FAIL; 2058 goto out_map; 2059 } 2060 /* The folio is isolated and isolation code holds a folio reference. */ 2061 spin_unlock(vmf->ptl); 2062 writable = false; 2063 2064 if (!migrate_misplaced_folio(folio, target_nid)) { 2065 flags |= TNF_MIGRATED; 2066 nid = target_nid; 2067 task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags); 2068 return 0; 2069 } 2070 2071 flags |= TNF_MIGRATE_FAIL; 2072 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 2073 if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) { 2074 spin_unlock(vmf->ptl); 2075 return 0; 2076 } 2077 out_map: 2078 /* Restore the PMD */ 2079 pmd = pmd_modify(pmdp_get(vmf->pmd), vma->vm_page_prot); 2080 pmd = pmd_mkyoung(pmd); 2081 if (writable) 2082 pmd = pmd_mkwrite(pmd, vma); 2083 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); 2084 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 2085 spin_unlock(vmf->ptl); 2086 2087 if (nid != NUMA_NO_NODE) 2088 task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags); 2089 return 0; 2090 } 2091 2092 /* 2093 * Return true if we do MADV_FREE successfully on entire pmd page. 2094 * Otherwise, return false. 2095 */ 2096 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2097 pmd_t *pmd, unsigned long addr, unsigned long next) 2098 { 2099 spinlock_t *ptl; 2100 pmd_t orig_pmd; 2101 struct folio *folio; 2102 struct mm_struct *mm = tlb->mm; 2103 bool ret = false; 2104 2105 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2106 2107 ptl = pmd_trans_huge_lock(pmd, vma); 2108 if (!ptl) 2109 goto out_unlocked; 2110 2111 orig_pmd = *pmd; 2112 if (is_huge_zero_pmd(orig_pmd)) 2113 goto out; 2114 2115 if (unlikely(!pmd_present(orig_pmd))) { 2116 VM_BUG_ON(thp_migration_supported() && 2117 !is_pmd_migration_entry(orig_pmd)); 2118 goto out; 2119 } 2120 2121 folio = pmd_folio(orig_pmd); 2122 /* 2123 * If other processes are mapping this folio, we couldn't discard 2124 * the folio unless they all do MADV_FREE so let's skip the folio. 2125 */ 2126 if (folio_maybe_mapped_shared(folio)) 2127 goto out; 2128 2129 if (!folio_trylock(folio)) 2130 goto out; 2131 2132 /* 2133 * If user want to discard part-pages of THP, split it so MADV_FREE 2134 * will deactivate only them. 2135 */ 2136 if (next - addr != HPAGE_PMD_SIZE) { 2137 folio_get(folio); 2138 spin_unlock(ptl); 2139 split_folio(folio); 2140 folio_unlock(folio); 2141 folio_put(folio); 2142 goto out_unlocked; 2143 } 2144 2145 if (folio_test_dirty(folio)) 2146 folio_clear_dirty(folio); 2147 folio_unlock(folio); 2148 2149 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) { 2150 pmdp_invalidate(vma, addr, pmd); 2151 orig_pmd = pmd_mkold(orig_pmd); 2152 orig_pmd = pmd_mkclean(orig_pmd); 2153 2154 set_pmd_at(mm, addr, pmd, orig_pmd); 2155 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 2156 } 2157 2158 folio_mark_lazyfree(folio); 2159 ret = true; 2160 out: 2161 spin_unlock(ptl); 2162 out_unlocked: 2163 return ret; 2164 } 2165 2166 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd) 2167 { 2168 pgtable_t pgtable; 2169 2170 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2171 pte_free(mm, pgtable); 2172 mm_dec_nr_ptes(mm); 2173 } 2174 2175 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2176 pmd_t *pmd, unsigned long addr) 2177 { 2178 pmd_t orig_pmd; 2179 spinlock_t *ptl; 2180 2181 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2182 2183 ptl = __pmd_trans_huge_lock(pmd, vma); 2184 if (!ptl) 2185 return 0; 2186 /* 2187 * For architectures like ppc64 we look at deposited pgtable 2188 * when calling pmdp_huge_get_and_clear. So do the 2189 * pgtable_trans_huge_withdraw after finishing pmdp related 2190 * operations. 2191 */ 2192 orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd, 2193 tlb->fullmm); 2194 arch_check_zapped_pmd(vma, orig_pmd); 2195 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 2196 if (!vma_is_dax(vma) && vma_is_special_huge(vma)) { 2197 if (arch_needs_pgtable_deposit()) 2198 zap_deposited_table(tlb->mm, pmd); 2199 spin_unlock(ptl); 2200 } else if (is_huge_zero_pmd(orig_pmd)) { 2201 if (!vma_is_dax(vma) || arch_needs_pgtable_deposit()) 2202 zap_deposited_table(tlb->mm, pmd); 2203 spin_unlock(ptl); 2204 } else { 2205 struct folio *folio = NULL; 2206 int flush_needed = 1; 2207 2208 if (pmd_present(orig_pmd)) { 2209 struct page *page = pmd_page(orig_pmd); 2210 2211 folio = page_folio(page); 2212 folio_remove_rmap_pmd(folio, page, vma); 2213 WARN_ON_ONCE(folio_mapcount(folio) < 0); 2214 VM_BUG_ON_PAGE(!PageHead(page), page); 2215 } else if (thp_migration_supported()) { 2216 swp_entry_t entry; 2217 2218 VM_BUG_ON(!is_pmd_migration_entry(orig_pmd)); 2219 entry = pmd_to_swp_entry(orig_pmd); 2220 folio = pfn_swap_entry_folio(entry); 2221 flush_needed = 0; 2222 } else 2223 WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!"); 2224 2225 if (folio_test_anon(folio)) { 2226 zap_deposited_table(tlb->mm, pmd); 2227 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); 2228 } else { 2229 if (arch_needs_pgtable_deposit()) 2230 zap_deposited_table(tlb->mm, pmd); 2231 add_mm_counter(tlb->mm, mm_counter_file(folio), 2232 -HPAGE_PMD_NR); 2233 2234 /* 2235 * Use flush_needed to indicate whether the PMD entry 2236 * is present, instead of checking pmd_present() again. 2237 */ 2238 if (flush_needed && pmd_young(orig_pmd) && 2239 likely(vma_has_recency(vma))) 2240 folio_mark_accessed(folio); 2241 } 2242 2243 spin_unlock(ptl); 2244 if (flush_needed) 2245 tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE); 2246 } 2247 return 1; 2248 } 2249 2250 #ifndef pmd_move_must_withdraw 2251 static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl, 2252 spinlock_t *old_pmd_ptl, 2253 struct vm_area_struct *vma) 2254 { 2255 /* 2256 * With split pmd lock we also need to move preallocated 2257 * PTE page table if new_pmd is on different PMD page table. 2258 * 2259 * We also don't deposit and withdraw tables for file pages. 2260 */ 2261 return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma); 2262 } 2263 #endif 2264 2265 static pmd_t move_soft_dirty_pmd(pmd_t pmd) 2266 { 2267 #ifdef CONFIG_MEM_SOFT_DIRTY 2268 if (unlikely(is_pmd_migration_entry(pmd))) 2269 pmd = pmd_swp_mksoft_dirty(pmd); 2270 else if (pmd_present(pmd)) 2271 pmd = pmd_mksoft_dirty(pmd); 2272 #endif 2273 return pmd; 2274 } 2275 2276 static pmd_t clear_uffd_wp_pmd(pmd_t pmd) 2277 { 2278 if (pmd_present(pmd)) 2279 pmd = pmd_clear_uffd_wp(pmd); 2280 else if (is_swap_pmd(pmd)) 2281 pmd = pmd_swp_clear_uffd_wp(pmd); 2282 2283 return pmd; 2284 } 2285 2286 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, 2287 unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd) 2288 { 2289 spinlock_t *old_ptl, *new_ptl; 2290 pmd_t pmd; 2291 struct mm_struct *mm = vma->vm_mm; 2292 bool force_flush = false; 2293 2294 /* 2295 * The destination pmd shouldn't be established, free_pgtables() 2296 * should have released it; but move_page_tables() might have already 2297 * inserted a page table, if racing against shmem/file collapse. 2298 */ 2299 if (!pmd_none(*new_pmd)) { 2300 VM_BUG_ON(pmd_trans_huge(*new_pmd)); 2301 return false; 2302 } 2303 2304 /* 2305 * We don't have to worry about the ordering of src and dst 2306 * ptlocks because exclusive mmap_lock prevents deadlock. 2307 */ 2308 old_ptl = __pmd_trans_huge_lock(old_pmd, vma); 2309 if (old_ptl) { 2310 new_ptl = pmd_lockptr(mm, new_pmd); 2311 if (new_ptl != old_ptl) 2312 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 2313 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); 2314 if (pmd_present(pmd)) 2315 force_flush = true; 2316 VM_BUG_ON(!pmd_none(*new_pmd)); 2317 2318 if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) { 2319 pgtable_t pgtable; 2320 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); 2321 pgtable_trans_huge_deposit(mm, new_pmd, pgtable); 2322 } 2323 pmd = move_soft_dirty_pmd(pmd); 2324 if (vma_has_uffd_without_event_remap(vma)) 2325 pmd = clear_uffd_wp_pmd(pmd); 2326 set_pmd_at(mm, new_addr, new_pmd, pmd); 2327 if (force_flush) 2328 flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE); 2329 if (new_ptl != old_ptl) 2330 spin_unlock(new_ptl); 2331 spin_unlock(old_ptl); 2332 return true; 2333 } 2334 return false; 2335 } 2336 2337 /* 2338 * Returns 2339 * - 0 if PMD could not be locked 2340 * - 1 if PMD was locked but protections unchanged and TLB flush unnecessary 2341 * or if prot_numa but THP migration is not supported 2342 * - HPAGE_PMD_NR if protections changed and TLB flush necessary 2343 */ 2344 int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2345 pmd_t *pmd, unsigned long addr, pgprot_t newprot, 2346 unsigned long cp_flags) 2347 { 2348 struct mm_struct *mm = vma->vm_mm; 2349 spinlock_t *ptl; 2350 pmd_t oldpmd, entry; 2351 bool prot_numa = cp_flags & MM_CP_PROT_NUMA; 2352 bool uffd_wp = cp_flags & MM_CP_UFFD_WP; 2353 bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; 2354 int ret = 1; 2355 2356 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2357 2358 if (prot_numa && !thp_migration_supported()) 2359 return 1; 2360 2361 ptl = __pmd_trans_huge_lock(pmd, vma); 2362 if (!ptl) 2363 return 0; 2364 2365 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 2366 if (is_swap_pmd(*pmd)) { 2367 swp_entry_t entry = pmd_to_swp_entry(*pmd); 2368 struct folio *folio = pfn_swap_entry_folio(entry); 2369 pmd_t newpmd; 2370 2371 VM_BUG_ON(!is_pmd_migration_entry(*pmd)); 2372 if (is_writable_migration_entry(entry)) { 2373 /* 2374 * A protection check is difficult so 2375 * just be safe and disable write 2376 */ 2377 if (folio_test_anon(folio)) 2378 entry = make_readable_exclusive_migration_entry(swp_offset(entry)); 2379 else 2380 entry = make_readable_migration_entry(swp_offset(entry)); 2381 newpmd = swp_entry_to_pmd(entry); 2382 if (pmd_swp_soft_dirty(*pmd)) 2383 newpmd = pmd_swp_mksoft_dirty(newpmd); 2384 } else { 2385 newpmd = *pmd; 2386 } 2387 2388 if (uffd_wp) 2389 newpmd = pmd_swp_mkuffd_wp(newpmd); 2390 else if (uffd_wp_resolve) 2391 newpmd = pmd_swp_clear_uffd_wp(newpmd); 2392 if (!pmd_same(*pmd, newpmd)) 2393 set_pmd_at(mm, addr, pmd, newpmd); 2394 goto unlock; 2395 } 2396 #endif 2397 2398 if (prot_numa) { 2399 2400 /* 2401 * Avoid trapping faults against the zero page. The read-only 2402 * data is likely to be read-cached on the local CPU and 2403 * local/remote hits to the zero page are not interesting. 2404 */ 2405 if (is_huge_zero_pmd(*pmd)) 2406 goto unlock; 2407 2408 if (pmd_protnone(*pmd)) 2409 goto unlock; 2410 2411 if (!folio_can_map_prot_numa(pmd_folio(*pmd), vma, 2412 vma_is_single_threaded_private(vma))) 2413 goto unlock; 2414 } 2415 /* 2416 * In case prot_numa, we are under mmap_read_lock(mm). It's critical 2417 * to not clear pmd intermittently to avoid race with MADV_DONTNEED 2418 * which is also under mmap_read_lock(mm): 2419 * 2420 * CPU0: CPU1: 2421 * change_huge_pmd(prot_numa=1) 2422 * pmdp_huge_get_and_clear_notify() 2423 * madvise_dontneed() 2424 * zap_pmd_range() 2425 * pmd_trans_huge(*pmd) == 0 (without ptl) 2426 * // skip the pmd 2427 * set_pmd_at(); 2428 * // pmd is re-established 2429 * 2430 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it 2431 * which may break userspace. 2432 * 2433 * pmdp_invalidate_ad() is required to make sure we don't miss 2434 * dirty/young flags set by hardware. 2435 */ 2436 oldpmd = pmdp_invalidate_ad(vma, addr, pmd); 2437 2438 entry = pmd_modify(oldpmd, newprot); 2439 if (uffd_wp) 2440 entry = pmd_mkuffd_wp(entry); 2441 else if (uffd_wp_resolve) 2442 /* 2443 * Leave the write bit to be handled by PF interrupt 2444 * handler, then things like COW could be properly 2445 * handled. 2446 */ 2447 entry = pmd_clear_uffd_wp(entry); 2448 2449 /* See change_pte_range(). */ 2450 if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) && 2451 can_change_pmd_writable(vma, addr, entry)) 2452 entry = pmd_mkwrite(entry, vma); 2453 2454 ret = HPAGE_PMD_NR; 2455 set_pmd_at(mm, addr, pmd, entry); 2456 2457 if (huge_pmd_needs_flush(oldpmd, entry)) 2458 tlb_flush_pmd_range(tlb, addr, HPAGE_PMD_SIZE); 2459 unlock: 2460 spin_unlock(ptl); 2461 return ret; 2462 } 2463 2464 /* 2465 * Returns: 2466 * 2467 * - 0: if pud leaf changed from under us 2468 * - 1: if pud can be skipped 2469 * - HPAGE_PUD_NR: if pud was successfully processed 2470 */ 2471 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2472 int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 2473 pud_t *pudp, unsigned long addr, pgprot_t newprot, 2474 unsigned long cp_flags) 2475 { 2476 struct mm_struct *mm = vma->vm_mm; 2477 pud_t oldpud, entry; 2478 spinlock_t *ptl; 2479 2480 tlb_change_page_size(tlb, HPAGE_PUD_SIZE); 2481 2482 /* NUMA balancing doesn't apply to dax */ 2483 if (cp_flags & MM_CP_PROT_NUMA) 2484 return 1; 2485 2486 /* 2487 * Huge entries on userfault-wp only works with anonymous, while we 2488 * don't have anonymous PUDs yet. 2489 */ 2490 if (WARN_ON_ONCE(cp_flags & MM_CP_UFFD_WP_ALL)) 2491 return 1; 2492 2493 ptl = __pud_trans_huge_lock(pudp, vma); 2494 if (!ptl) 2495 return 0; 2496 2497 /* 2498 * Can't clear PUD or it can race with concurrent zapping. See 2499 * change_huge_pmd(). 2500 */ 2501 oldpud = pudp_invalidate(vma, addr, pudp); 2502 entry = pud_modify(oldpud, newprot); 2503 set_pud_at(mm, addr, pudp, entry); 2504 tlb_flush_pud_range(tlb, addr, HPAGE_PUD_SIZE); 2505 2506 spin_unlock(ptl); 2507 return HPAGE_PUD_NR; 2508 } 2509 #endif 2510 2511 #ifdef CONFIG_USERFAULTFD 2512 /* 2513 * The PT lock for src_pmd and dst_vma/src_vma (for reading) are locked by 2514 * the caller, but it must return after releasing the page_table_lock. 2515 * Just move the page from src_pmd to dst_pmd if possible. 2516 * Return zero if succeeded in moving the page, -EAGAIN if it needs to be 2517 * repeated by the caller, or other errors in case of failure. 2518 */ 2519 int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval, 2520 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, 2521 unsigned long dst_addr, unsigned long src_addr) 2522 { 2523 pmd_t _dst_pmd, src_pmdval; 2524 struct page *src_page; 2525 struct folio *src_folio; 2526 spinlock_t *src_ptl, *dst_ptl; 2527 pgtable_t src_pgtable; 2528 struct mmu_notifier_range range; 2529 int err = 0; 2530 2531 src_pmdval = *src_pmd; 2532 src_ptl = pmd_lockptr(mm, src_pmd); 2533 2534 lockdep_assert_held(src_ptl); 2535 vma_assert_locked(src_vma); 2536 vma_assert_locked(dst_vma); 2537 2538 /* Sanity checks before the operation */ 2539 if (WARN_ON_ONCE(!pmd_none(dst_pmdval)) || WARN_ON_ONCE(src_addr & ~HPAGE_PMD_MASK) || 2540 WARN_ON_ONCE(dst_addr & ~HPAGE_PMD_MASK)) { 2541 spin_unlock(src_ptl); 2542 return -EINVAL; 2543 } 2544 2545 if (!pmd_trans_huge(src_pmdval)) { 2546 spin_unlock(src_ptl); 2547 if (is_pmd_migration_entry(src_pmdval)) { 2548 pmd_migration_entry_wait(mm, &src_pmdval); 2549 return -EAGAIN; 2550 } 2551 return -ENOENT; 2552 } 2553 2554 src_page = pmd_page(src_pmdval); 2555 2556 if (!is_huge_zero_pmd(src_pmdval)) { 2557 if (unlikely(!PageAnonExclusive(src_page))) { 2558 spin_unlock(src_ptl); 2559 return -EBUSY; 2560 } 2561 2562 src_folio = page_folio(src_page); 2563 folio_get(src_folio); 2564 } else 2565 src_folio = NULL; 2566 2567 spin_unlock(src_ptl); 2568 2569 flush_cache_range(src_vma, src_addr, src_addr + HPAGE_PMD_SIZE); 2570 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, src_addr, 2571 src_addr + HPAGE_PMD_SIZE); 2572 mmu_notifier_invalidate_range_start(&range); 2573 2574 if (src_folio) 2575 folio_lock(src_folio); 2576 2577 dst_ptl = pmd_lockptr(mm, dst_pmd); 2578 double_pt_lock(src_ptl, dst_ptl); 2579 if (unlikely(!pmd_same(*src_pmd, src_pmdval) || 2580 !pmd_same(*dst_pmd, dst_pmdval))) { 2581 err = -EAGAIN; 2582 goto unlock_ptls; 2583 } 2584 if (src_folio) { 2585 if (folio_maybe_dma_pinned(src_folio) || 2586 !PageAnonExclusive(&src_folio->page)) { 2587 err = -EBUSY; 2588 goto unlock_ptls; 2589 } 2590 2591 if (WARN_ON_ONCE(!folio_test_head(src_folio)) || 2592 WARN_ON_ONCE(!folio_test_anon(src_folio))) { 2593 err = -EBUSY; 2594 goto unlock_ptls; 2595 } 2596 2597 src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); 2598 /* Folio got pinned from under us. Put it back and fail the move. */ 2599 if (folio_maybe_dma_pinned(src_folio)) { 2600 set_pmd_at(mm, src_addr, src_pmd, src_pmdval); 2601 err = -EBUSY; 2602 goto unlock_ptls; 2603 } 2604 2605 folio_move_anon_rmap(src_folio, dst_vma); 2606 src_folio->index = linear_page_index(dst_vma, dst_addr); 2607 2608 _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); 2609 /* Follow mremap() behavior and treat the entry dirty after the move */ 2610 _dst_pmd = pmd_mkwrite(pmd_mkdirty(_dst_pmd), dst_vma); 2611 } else { 2612 src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); 2613 _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); 2614 } 2615 set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd); 2616 2617 src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd); 2618 pgtable_trans_huge_deposit(mm, dst_pmd, src_pgtable); 2619 unlock_ptls: 2620 double_pt_unlock(src_ptl, dst_ptl); 2621 /* unblock rmap walks */ 2622 if (src_folio) 2623 folio_unlock(src_folio); 2624 mmu_notifier_invalidate_range_end(&range); 2625 if (src_folio) 2626 folio_put(src_folio); 2627 return err; 2628 } 2629 #endif /* CONFIG_USERFAULTFD */ 2630 2631 /* 2632 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise. 2633 * 2634 * Note that if it returns page table lock pointer, this routine returns without 2635 * unlocking page table lock. So callers must unlock it. 2636 */ 2637 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) 2638 { 2639 spinlock_t *ptl; 2640 ptl = pmd_lock(vma->vm_mm, pmd); 2641 if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd))) 2642 return ptl; 2643 spin_unlock(ptl); 2644 return NULL; 2645 } 2646 2647 /* 2648 * Returns page table lock pointer if a given pud maps a thp, NULL otherwise. 2649 * 2650 * Note that if it returns page table lock pointer, this routine returns without 2651 * unlocking page table lock. So callers must unlock it. 2652 */ 2653 spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma) 2654 { 2655 spinlock_t *ptl; 2656 2657 ptl = pud_lock(vma->vm_mm, pud); 2658 if (likely(pud_trans_huge(*pud))) 2659 return ptl; 2660 spin_unlock(ptl); 2661 return NULL; 2662 } 2663 2664 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2665 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 2666 pud_t *pud, unsigned long addr) 2667 { 2668 spinlock_t *ptl; 2669 pud_t orig_pud; 2670 2671 ptl = __pud_trans_huge_lock(pud, vma); 2672 if (!ptl) 2673 return 0; 2674 2675 orig_pud = pudp_huge_get_and_clear_full(vma, addr, pud, tlb->fullmm); 2676 arch_check_zapped_pud(vma, orig_pud); 2677 tlb_remove_pud_tlb_entry(tlb, pud, addr); 2678 if (!vma_is_dax(vma) && vma_is_special_huge(vma)) { 2679 spin_unlock(ptl); 2680 /* No zero page support yet */ 2681 } else { 2682 struct page *page = NULL; 2683 struct folio *folio; 2684 2685 /* No support for anonymous PUD pages or migration yet */ 2686 VM_WARN_ON_ONCE(vma_is_anonymous(vma) || 2687 !pud_present(orig_pud)); 2688 2689 page = pud_page(orig_pud); 2690 folio = page_folio(page); 2691 folio_remove_rmap_pud(folio, page, vma); 2692 add_mm_counter(tlb->mm, mm_counter_file(folio), -HPAGE_PUD_NR); 2693 2694 spin_unlock(ptl); 2695 tlb_remove_page_size(tlb, page, HPAGE_PUD_SIZE); 2696 } 2697 return 1; 2698 } 2699 2700 static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud, 2701 unsigned long haddr) 2702 { 2703 struct folio *folio; 2704 struct page *page; 2705 pud_t old_pud; 2706 2707 VM_BUG_ON(haddr & ~HPAGE_PUD_MASK); 2708 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2709 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma); 2710 VM_BUG_ON(!pud_trans_huge(*pud)); 2711 2712 count_vm_event(THP_SPLIT_PUD); 2713 2714 old_pud = pudp_huge_clear_flush(vma, haddr, pud); 2715 2716 if (!vma_is_dax(vma)) 2717 return; 2718 2719 page = pud_page(old_pud); 2720 folio = page_folio(page); 2721 2722 if (!folio_test_dirty(folio) && pud_dirty(old_pud)) 2723 folio_mark_dirty(folio); 2724 if (!folio_test_referenced(folio) && pud_young(old_pud)) 2725 folio_set_referenced(folio); 2726 folio_remove_rmap_pud(folio, page, vma); 2727 folio_put(folio); 2728 add_mm_counter(vma->vm_mm, mm_counter_file(folio), 2729 -HPAGE_PUD_NR); 2730 } 2731 2732 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 2733 unsigned long address) 2734 { 2735 spinlock_t *ptl; 2736 struct mmu_notifier_range range; 2737 2738 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 2739 address & HPAGE_PUD_MASK, 2740 (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE); 2741 mmu_notifier_invalidate_range_start(&range); 2742 ptl = pud_lock(vma->vm_mm, pud); 2743 if (unlikely(!pud_trans_huge(*pud))) 2744 goto out; 2745 __split_huge_pud_locked(vma, pud, range.start); 2746 2747 out: 2748 spin_unlock(ptl); 2749 mmu_notifier_invalidate_range_end(&range); 2750 } 2751 #else 2752 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 2753 unsigned long address) 2754 { 2755 } 2756 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 2757 2758 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 2759 unsigned long haddr, pmd_t *pmd) 2760 { 2761 struct mm_struct *mm = vma->vm_mm; 2762 pgtable_t pgtable; 2763 pmd_t _pmd, old_pmd; 2764 unsigned long addr; 2765 pte_t *pte; 2766 int i; 2767 2768 /* 2769 * Leave pmd empty until pte is filled note that it is fine to delay 2770 * notification until mmu_notifier_invalidate_range_end() as we are 2771 * replacing a zero pmd write protected page with a zero pte write 2772 * protected page. 2773 * 2774 * See Documentation/mm/mmu_notifier.rst 2775 */ 2776 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); 2777 2778 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2779 pmd_populate(mm, &_pmd, pgtable); 2780 2781 pte = pte_offset_map(&_pmd, haddr); 2782 VM_BUG_ON(!pte); 2783 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2784 pte_t entry; 2785 2786 entry = pfn_pte(my_zero_pfn(addr), vma->vm_page_prot); 2787 entry = pte_mkspecial(entry); 2788 if (pmd_uffd_wp(old_pmd)) 2789 entry = pte_mkuffd_wp(entry); 2790 VM_BUG_ON(!pte_none(ptep_get(pte))); 2791 set_pte_at(mm, addr, pte, entry); 2792 pte++; 2793 } 2794 pte_unmap(pte - 1); 2795 smp_wmb(); /* make pte visible before pmd */ 2796 pmd_populate(mm, pmd, pgtable); 2797 } 2798 2799 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 2800 unsigned long haddr, bool freeze) 2801 { 2802 struct mm_struct *mm = vma->vm_mm; 2803 struct folio *folio; 2804 struct page *page; 2805 pgtable_t pgtable; 2806 pmd_t old_pmd, _pmd; 2807 bool young, write, soft_dirty, pmd_migration = false, uffd_wp = false; 2808 bool anon_exclusive = false, dirty = false; 2809 unsigned long addr; 2810 pte_t *pte; 2811 int i; 2812 2813 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 2814 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2815 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 2816 VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd)); 2817 2818 count_vm_event(THP_SPLIT_PMD); 2819 2820 if (!vma_is_anonymous(vma)) { 2821 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); 2822 /* 2823 * We are going to unmap this huge page. So 2824 * just go ahead and zap it 2825 */ 2826 if (arch_needs_pgtable_deposit()) 2827 zap_deposited_table(mm, pmd); 2828 if (!vma_is_dax(vma) && vma_is_special_huge(vma)) 2829 return; 2830 if (unlikely(is_pmd_migration_entry(old_pmd))) { 2831 swp_entry_t entry; 2832 2833 entry = pmd_to_swp_entry(old_pmd); 2834 folio = pfn_swap_entry_folio(entry); 2835 } else if (is_huge_zero_pmd(old_pmd)) { 2836 return; 2837 } else { 2838 page = pmd_page(old_pmd); 2839 folio = page_folio(page); 2840 if (!folio_test_dirty(folio) && pmd_dirty(old_pmd)) 2841 folio_mark_dirty(folio); 2842 if (!folio_test_referenced(folio) && pmd_young(old_pmd)) 2843 folio_set_referenced(folio); 2844 folio_remove_rmap_pmd(folio, page, vma); 2845 folio_put(folio); 2846 } 2847 add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR); 2848 return; 2849 } 2850 2851 if (is_huge_zero_pmd(*pmd)) { 2852 /* 2853 * FIXME: Do we want to invalidate secondary mmu by calling 2854 * mmu_notifier_arch_invalidate_secondary_tlbs() see comments below 2855 * inside __split_huge_pmd() ? 2856 * 2857 * We are going from a zero huge page write protected to zero 2858 * small page also write protected so it does not seems useful 2859 * to invalidate secondary mmu at this time. 2860 */ 2861 return __split_huge_zero_page_pmd(vma, haddr, pmd); 2862 } 2863 2864 pmd_migration = is_pmd_migration_entry(*pmd); 2865 if (unlikely(pmd_migration)) { 2866 swp_entry_t entry; 2867 2868 old_pmd = *pmd; 2869 entry = pmd_to_swp_entry(old_pmd); 2870 page = pfn_swap_entry_to_page(entry); 2871 write = is_writable_migration_entry(entry); 2872 if (PageAnon(page)) 2873 anon_exclusive = is_readable_exclusive_migration_entry(entry); 2874 young = is_migration_entry_young(entry); 2875 dirty = is_migration_entry_dirty(entry); 2876 soft_dirty = pmd_swp_soft_dirty(old_pmd); 2877 uffd_wp = pmd_swp_uffd_wp(old_pmd); 2878 } else { 2879 /* 2880 * Up to this point the pmd is present and huge and userland has 2881 * the whole access to the hugepage during the split (which 2882 * happens in place). If we overwrite the pmd with the not-huge 2883 * version pointing to the pte here (which of course we could if 2884 * all CPUs were bug free), userland could trigger a small page 2885 * size TLB miss on the small sized TLB while the hugepage TLB 2886 * entry is still established in the huge TLB. Some CPU doesn't 2887 * like that. See 2888 * http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum 2889 * 383 on page 105. Intel should be safe but is also warns that 2890 * it's only safe if the permission and cache attributes of the 2891 * two entries loaded in the two TLB is identical (which should 2892 * be the case here). But it is generally safer to never allow 2893 * small and huge TLB entries for the same virtual address to be 2894 * loaded simultaneously. So instead of doing "pmd_populate(); 2895 * flush_pmd_tlb_range();" we first mark the current pmd 2896 * notpresent (atomically because here the pmd_trans_huge must 2897 * remain set at all times on the pmd until the split is 2898 * complete for this pmd), then we flush the SMP TLB and finally 2899 * we write the non-huge version of the pmd entry with 2900 * pmd_populate. 2901 */ 2902 old_pmd = pmdp_invalidate(vma, haddr, pmd); 2903 page = pmd_page(old_pmd); 2904 folio = page_folio(page); 2905 if (pmd_dirty(old_pmd)) { 2906 dirty = true; 2907 folio_set_dirty(folio); 2908 } 2909 write = pmd_write(old_pmd); 2910 young = pmd_young(old_pmd); 2911 soft_dirty = pmd_soft_dirty(old_pmd); 2912 uffd_wp = pmd_uffd_wp(old_pmd); 2913 2914 VM_WARN_ON_FOLIO(!folio_ref_count(folio), folio); 2915 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); 2916 2917 /* 2918 * Without "freeze", we'll simply split the PMD, propagating the 2919 * PageAnonExclusive() flag for each PTE by setting it for 2920 * each subpage -- no need to (temporarily) clear. 2921 * 2922 * With "freeze" we want to replace mapped pages by 2923 * migration entries right away. This is only possible if we 2924 * managed to clear PageAnonExclusive() -- see 2925 * set_pmd_migration_entry(). 2926 * 2927 * In case we cannot clear PageAnonExclusive(), split the PMD 2928 * only and let try_to_migrate_one() fail later. 2929 * 2930 * See folio_try_share_anon_rmap_pmd(): invalidate PMD first. 2931 */ 2932 anon_exclusive = PageAnonExclusive(page); 2933 if (freeze && anon_exclusive && 2934 folio_try_share_anon_rmap_pmd(folio, page)) 2935 freeze = false; 2936 if (!freeze) { 2937 rmap_t rmap_flags = RMAP_NONE; 2938 2939 folio_ref_add(folio, HPAGE_PMD_NR - 1); 2940 if (anon_exclusive) 2941 rmap_flags |= RMAP_EXCLUSIVE; 2942 folio_add_anon_rmap_ptes(folio, page, HPAGE_PMD_NR, 2943 vma, haddr, rmap_flags); 2944 } 2945 } 2946 2947 /* 2948 * Withdraw the table only after we mark the pmd entry invalid. 2949 * This's critical for some architectures (Power). 2950 */ 2951 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2952 pmd_populate(mm, &_pmd, pgtable); 2953 2954 pte = pte_offset_map(&_pmd, haddr); 2955 VM_BUG_ON(!pte); 2956 2957 /* 2958 * Note that NUMA hinting access restrictions are not transferred to 2959 * avoid any possibility of altering permissions across VMAs. 2960 */ 2961 if (freeze || pmd_migration) { 2962 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2963 pte_t entry; 2964 swp_entry_t swp_entry; 2965 2966 if (write) 2967 swp_entry = make_writable_migration_entry( 2968 page_to_pfn(page + i)); 2969 else if (anon_exclusive) 2970 swp_entry = make_readable_exclusive_migration_entry( 2971 page_to_pfn(page + i)); 2972 else 2973 swp_entry = make_readable_migration_entry( 2974 page_to_pfn(page + i)); 2975 if (young) 2976 swp_entry = make_migration_entry_young(swp_entry); 2977 if (dirty) 2978 swp_entry = make_migration_entry_dirty(swp_entry); 2979 entry = swp_entry_to_pte(swp_entry); 2980 if (soft_dirty) 2981 entry = pte_swp_mksoft_dirty(entry); 2982 if (uffd_wp) 2983 entry = pte_swp_mkuffd_wp(entry); 2984 2985 VM_WARN_ON(!pte_none(ptep_get(pte + i))); 2986 set_pte_at(mm, addr, pte + i, entry); 2987 } 2988 } else { 2989 pte_t entry; 2990 2991 entry = mk_pte(page, READ_ONCE(vma->vm_page_prot)); 2992 if (write) 2993 entry = pte_mkwrite(entry, vma); 2994 if (!young) 2995 entry = pte_mkold(entry); 2996 /* NOTE: this may set soft-dirty too on some archs */ 2997 if (dirty) 2998 entry = pte_mkdirty(entry); 2999 if (soft_dirty) 3000 entry = pte_mksoft_dirty(entry); 3001 if (uffd_wp) 3002 entry = pte_mkuffd_wp(entry); 3003 3004 for (i = 0; i < HPAGE_PMD_NR; i++) 3005 VM_WARN_ON(!pte_none(ptep_get(pte + i))); 3006 3007 set_ptes(mm, haddr, pte, entry, HPAGE_PMD_NR); 3008 } 3009 pte_unmap(pte); 3010 3011 if (!pmd_migration) 3012 folio_remove_rmap_pmd(folio, page, vma); 3013 if (freeze) 3014 put_page(page); 3015 3016 smp_wmb(); /* make pte visible before pmd */ 3017 pmd_populate(mm, pmd, pgtable); 3018 } 3019 3020 void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, 3021 pmd_t *pmd, bool freeze) 3022 { 3023 VM_WARN_ON_ONCE(!IS_ALIGNED(address, HPAGE_PMD_SIZE)); 3024 if (pmd_trans_huge(*pmd) || is_pmd_migration_entry(*pmd)) 3025 __split_huge_pmd_locked(vma, pmd, address, freeze); 3026 } 3027 3028 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 3029 unsigned long address, bool freeze) 3030 { 3031 spinlock_t *ptl; 3032 struct mmu_notifier_range range; 3033 3034 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 3035 address & HPAGE_PMD_MASK, 3036 (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE); 3037 mmu_notifier_invalidate_range_start(&range); 3038 ptl = pmd_lock(vma->vm_mm, pmd); 3039 split_huge_pmd_locked(vma, range.start, pmd, freeze); 3040 spin_unlock(ptl); 3041 mmu_notifier_invalidate_range_end(&range); 3042 } 3043 3044 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, 3045 bool freeze) 3046 { 3047 pmd_t *pmd = mm_find_pmd(vma->vm_mm, address); 3048 3049 if (!pmd) 3050 return; 3051 3052 __split_huge_pmd(vma, pmd, address, freeze); 3053 } 3054 3055 static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address) 3056 { 3057 /* 3058 * If the new address isn't hpage aligned and it could previously 3059 * contain an hugepage: check if we need to split an huge pmd. 3060 */ 3061 if (!IS_ALIGNED(address, HPAGE_PMD_SIZE) && 3062 range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE), 3063 ALIGN(address, HPAGE_PMD_SIZE))) 3064 split_huge_pmd_address(vma, address, false); 3065 } 3066 3067 void vma_adjust_trans_huge(struct vm_area_struct *vma, 3068 unsigned long start, 3069 unsigned long end, 3070 struct vm_area_struct *next) 3071 { 3072 /* Check if we need to split start first. */ 3073 split_huge_pmd_if_needed(vma, start); 3074 3075 /* Check if we need to split end next. */ 3076 split_huge_pmd_if_needed(vma, end); 3077 3078 /* If we're incrementing next->vm_start, we might need to split it. */ 3079 if (next) 3080 split_huge_pmd_if_needed(next, end); 3081 } 3082 3083 static void unmap_folio(struct folio *folio) 3084 { 3085 enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SYNC | 3086 TTU_BATCH_FLUSH; 3087 3088 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 3089 3090 if (folio_test_pmd_mappable(folio)) 3091 ttu_flags |= TTU_SPLIT_HUGE_PMD; 3092 3093 /* 3094 * Anon pages need migration entries to preserve them, but file 3095 * pages can simply be left unmapped, then faulted back on demand. 3096 * If that is ever changed (perhaps for mlock), update remap_page(). 3097 */ 3098 if (folio_test_anon(folio)) 3099 try_to_migrate(folio, ttu_flags); 3100 else 3101 try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK); 3102 3103 try_to_unmap_flush(); 3104 } 3105 3106 static bool __discard_anon_folio_pmd_locked(struct vm_area_struct *vma, 3107 unsigned long addr, pmd_t *pmdp, 3108 struct folio *folio) 3109 { 3110 struct mm_struct *mm = vma->vm_mm; 3111 int ref_count, map_count; 3112 pmd_t orig_pmd = *pmdp; 3113 3114 if (pmd_dirty(orig_pmd)) 3115 folio_set_dirty(folio); 3116 if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { 3117 folio_set_swapbacked(folio); 3118 return false; 3119 } 3120 3121 orig_pmd = pmdp_huge_clear_flush(vma, addr, pmdp); 3122 3123 /* 3124 * Syncing against concurrent GUP-fast: 3125 * - clear PMD; barrier; read refcount 3126 * - inc refcount; barrier; read PMD 3127 */ 3128 smp_mb(); 3129 3130 ref_count = folio_ref_count(folio); 3131 map_count = folio_mapcount(folio); 3132 3133 /* 3134 * Order reads for folio refcount and dirty flag 3135 * (see comments in __remove_mapping()). 3136 */ 3137 smp_rmb(); 3138 3139 /* 3140 * If the folio or its PMD is redirtied at this point, or if there 3141 * are unexpected references, we will give up to discard this folio 3142 * and remap it. 3143 * 3144 * The only folio refs must be one from isolation plus the rmap(s). 3145 */ 3146 if (pmd_dirty(orig_pmd)) 3147 folio_set_dirty(folio); 3148 if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { 3149 folio_set_swapbacked(folio); 3150 set_pmd_at(mm, addr, pmdp, orig_pmd); 3151 return false; 3152 } 3153 3154 if (ref_count != map_count + 1) { 3155 set_pmd_at(mm, addr, pmdp, orig_pmd); 3156 return false; 3157 } 3158 3159 folio_remove_rmap_pmd(folio, pmd_page(orig_pmd), vma); 3160 zap_deposited_table(mm, pmdp); 3161 add_mm_counter(mm, MM_ANONPAGES, -HPAGE_PMD_NR); 3162 if (vma->vm_flags & VM_LOCKED) 3163 mlock_drain_local(); 3164 folio_put(folio); 3165 3166 return true; 3167 } 3168 3169 bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr, 3170 pmd_t *pmdp, struct folio *folio) 3171 { 3172 VM_WARN_ON_FOLIO(!folio_test_pmd_mappable(folio), folio); 3173 VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); 3174 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); 3175 VM_WARN_ON_FOLIO(folio_test_swapbacked(folio), folio); 3176 VM_WARN_ON_ONCE(!IS_ALIGNED(addr, HPAGE_PMD_SIZE)); 3177 3178 return __discard_anon_folio_pmd_locked(vma, addr, pmdp, folio); 3179 } 3180 3181 static void remap_page(struct folio *folio, unsigned long nr, int flags) 3182 { 3183 int i = 0; 3184 3185 /* If unmap_folio() uses try_to_migrate() on file, remove this check */ 3186 if (!folio_test_anon(folio)) 3187 return; 3188 for (;;) { 3189 remove_migration_ptes(folio, folio, RMP_LOCKED | flags); 3190 i += folio_nr_pages(folio); 3191 if (i >= nr) 3192 break; 3193 folio = folio_next(folio); 3194 } 3195 } 3196 3197 static void lru_add_split_folio(struct folio *folio, struct folio *new_folio, 3198 struct lruvec *lruvec, struct list_head *list) 3199 { 3200 VM_BUG_ON_FOLIO(folio_test_lru(new_folio), folio); 3201 lockdep_assert_held(&lruvec->lru_lock); 3202 3203 if (list) { 3204 /* page reclaim is reclaiming a huge page */ 3205 VM_WARN_ON(folio_test_lru(folio)); 3206 folio_get(new_folio); 3207 list_add_tail(&new_folio->lru, list); 3208 } else { 3209 /* head is still on lru (and we have it frozen) */ 3210 VM_WARN_ON(!folio_test_lru(folio)); 3211 if (folio_test_unevictable(folio)) 3212 new_folio->mlock_count = 0; 3213 else 3214 list_add_tail(&new_folio->lru, &folio->lru); 3215 folio_set_lru(new_folio); 3216 } 3217 } 3218 3219 /* Racy check whether the huge page can be split */ 3220 bool can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins) 3221 { 3222 int extra_pins; 3223 3224 /* Additional pins from page cache */ 3225 if (folio_test_anon(folio)) 3226 extra_pins = folio_test_swapcache(folio) ? 3227 folio_nr_pages(folio) : 0; 3228 else 3229 extra_pins = folio_nr_pages(folio); 3230 if (pextra_pins) 3231 *pextra_pins = extra_pins; 3232 return folio_mapcount(folio) == folio_ref_count(folio) - extra_pins - 3233 caller_pins; 3234 } 3235 3236 static bool page_range_has_hwpoisoned(struct page *page, long nr_pages) 3237 { 3238 for (; nr_pages; page++, nr_pages--) 3239 if (PageHWPoison(page)) 3240 return true; 3241 return false; 3242 } 3243 3244 /* 3245 * It splits @folio into @new_order folios and copies the @folio metadata to 3246 * all the resulting folios. 3247 */ 3248 static void __split_folio_to_order(struct folio *folio, int old_order, 3249 int new_order) 3250 { 3251 /* Scan poisoned pages when split a poisoned folio to large folios */ 3252 const bool handle_hwpoison = folio_test_has_hwpoisoned(folio) && new_order; 3253 long new_nr_pages = 1 << new_order; 3254 long nr_pages = 1 << old_order; 3255 long i; 3256 3257 folio_clear_has_hwpoisoned(folio); 3258 3259 /* Check first new_nr_pages since the loop below skips them */ 3260 if (handle_hwpoison && 3261 page_range_has_hwpoisoned(folio_page(folio, 0), new_nr_pages)) 3262 folio_set_has_hwpoisoned(folio); 3263 /* 3264 * Skip the first new_nr_pages, since the new folio from them have all 3265 * the flags from the original folio. 3266 */ 3267 for (i = new_nr_pages; i < nr_pages; i += new_nr_pages) { 3268 struct page *new_head = &folio->page + i; 3269 /* 3270 * Careful: new_folio is not a "real" folio before we cleared PageTail. 3271 * Don't pass it around before clear_compound_head(). 3272 */ 3273 struct folio *new_folio = (struct folio *)new_head; 3274 3275 VM_BUG_ON_PAGE(atomic_read(&new_folio->_mapcount) != -1, new_head); 3276 3277 /* 3278 * Clone page flags before unfreezing refcount. 3279 * 3280 * After successful get_page_unless_zero() might follow flags change, 3281 * for example lock_page() which set PG_waiters. 3282 * 3283 * Note that for mapped sub-pages of an anonymous THP, 3284 * PG_anon_exclusive has been cleared in unmap_folio() and is stored in 3285 * the migration entry instead from where remap_page() will restore it. 3286 * We can still have PG_anon_exclusive set on effectively unmapped and 3287 * unreferenced sub-pages of an anonymous THP: we can simply drop 3288 * PG_anon_exclusive (-> PG_mappedtodisk) for these here. 3289 */ 3290 new_folio->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP; 3291 new_folio->flags.f |= (folio->flags.f & 3292 ((1L << PG_referenced) | 3293 (1L << PG_swapbacked) | 3294 (1L << PG_swapcache) | 3295 (1L << PG_mlocked) | 3296 (1L << PG_uptodate) | 3297 (1L << PG_active) | 3298 (1L << PG_workingset) | 3299 (1L << PG_locked) | 3300 (1L << PG_unevictable) | 3301 #ifdef CONFIG_ARCH_USES_PG_ARCH_2 3302 (1L << PG_arch_2) | 3303 #endif 3304 #ifdef CONFIG_ARCH_USES_PG_ARCH_3 3305 (1L << PG_arch_3) | 3306 #endif 3307 (1L << PG_dirty) | 3308 LRU_GEN_MASK | LRU_REFS_MASK)); 3309 3310 if (handle_hwpoison && 3311 page_range_has_hwpoisoned(new_head, new_nr_pages)) 3312 folio_set_has_hwpoisoned(new_folio); 3313 3314 new_folio->mapping = folio->mapping; 3315 new_folio->index = folio->index + i; 3316 3317 /* 3318 * page->private should not be set in tail pages. Fix up and warn once 3319 * if private is unexpectedly set. 3320 */ 3321 if (unlikely(new_folio->private)) { 3322 VM_WARN_ON_ONCE_PAGE(true, new_head); 3323 new_folio->private = NULL; 3324 } 3325 3326 if (folio_test_swapcache(folio)) 3327 new_folio->swap.val = folio->swap.val + i; 3328 3329 /* Page flags must be visible before we make the page non-compound. */ 3330 smp_wmb(); 3331 3332 /* 3333 * Clear PageTail before unfreezing page refcount. 3334 * 3335 * After successful get_page_unless_zero() might follow put_page() 3336 * which needs correct compound_head(). 3337 */ 3338 clear_compound_head(new_head); 3339 if (new_order) { 3340 prep_compound_page(new_head, new_order); 3341 folio_set_large_rmappable(new_folio); 3342 } 3343 3344 if (folio_test_young(folio)) 3345 folio_set_young(new_folio); 3346 if (folio_test_idle(folio)) 3347 folio_set_idle(new_folio); 3348 #ifdef CONFIG_MEMCG 3349 new_folio->memcg_data = folio->memcg_data; 3350 #endif 3351 3352 folio_xchg_last_cpupid(new_folio, folio_last_cpupid(folio)); 3353 } 3354 3355 if (new_order) 3356 folio_set_order(folio, new_order); 3357 else 3358 ClearPageCompound(&folio->page); 3359 } 3360 3361 /* 3362 * It splits an unmapped @folio to lower order smaller folios in two ways. 3363 * @folio: the to-be-split folio 3364 * @new_order: the smallest order of the after split folios (since buddy 3365 * allocator like split generates folios with orders from @folio's 3366 * order - 1 to new_order). 3367 * @split_at: in buddy allocator like split, the folio containing @split_at 3368 * will be split until its order becomes @new_order. 3369 * @xas: xa_state pointing to folio->mapping->i_pages and locked by caller 3370 * @mapping: @folio->mapping 3371 * @uniform_split: if the split is uniform or not (buddy allocator like split) 3372 * 3373 * 3374 * 1. uniform split: the given @folio into multiple @new_order small folios, 3375 * where all small folios have the same order. This is done when 3376 * uniform_split is true. 3377 * 2. buddy allocator like (non-uniform) split: the given @folio is split into 3378 * half and one of the half (containing the given page) is split into half 3379 * until the given @page's order becomes @new_order. This is done when 3380 * uniform_split is false. 3381 * 3382 * The high level flow for these two methods are: 3383 * 1. uniform split: a single __split_folio_to_order() is called to split the 3384 * @folio into @new_order, then we traverse all the resulting folios one by 3385 * one in PFN ascending order and perform stats, unfreeze, adding to list, 3386 * and file mapping index operations. 3387 * 2. non-uniform split: in general, folio_order - @new_order calls to 3388 * __split_folio_to_order() are made in a for loop to split the @folio 3389 * to one lower order at a time. The resulting small folios are processed 3390 * like what is done during the traversal in 1, except the one containing 3391 * @page, which is split in next for loop. 3392 * 3393 * After splitting, the caller's folio reference will be transferred to the 3394 * folio containing @page. The caller needs to unlock and/or free after-split 3395 * folios if necessary. 3396 * 3397 * For !uniform_split, when -ENOMEM is returned, the original folio might be 3398 * split. The caller needs to check the input folio. 3399 */ 3400 static int __split_unmapped_folio(struct folio *folio, int new_order, 3401 struct page *split_at, struct xa_state *xas, 3402 struct address_space *mapping, bool uniform_split) 3403 { 3404 const bool is_anon = folio_test_anon(folio); 3405 int old_order = folio_order(folio); 3406 int start_order = uniform_split ? new_order : old_order - 1; 3407 int split_order; 3408 3409 /* 3410 * split to new_order one order at a time. For uniform split, 3411 * folio is split to new_order directly. 3412 */ 3413 for (split_order = start_order; 3414 split_order >= new_order; 3415 split_order--) { 3416 int nr_new_folios = 1UL << (old_order - split_order); 3417 3418 /* order-1 anonymous folio is not supported */ 3419 if (is_anon && split_order == 1) 3420 continue; 3421 3422 if (mapping) { 3423 /* 3424 * uniform split has xas_split_alloc() called before 3425 * irq is disabled to allocate enough memory, whereas 3426 * non-uniform split can handle ENOMEM. 3427 */ 3428 if (uniform_split) 3429 xas_split(xas, folio, old_order); 3430 else { 3431 xas_set_order(xas, folio->index, split_order); 3432 xas_try_split(xas, folio, old_order); 3433 if (xas_error(xas)) 3434 return xas_error(xas); 3435 } 3436 } 3437 3438 folio_split_memcg_refs(folio, old_order, split_order); 3439 split_page_owner(&folio->page, old_order, split_order); 3440 pgalloc_tag_split(folio, old_order, split_order); 3441 __split_folio_to_order(folio, old_order, split_order); 3442 3443 if (is_anon) { 3444 mod_mthp_stat(old_order, MTHP_STAT_NR_ANON, -1); 3445 mod_mthp_stat(split_order, MTHP_STAT_NR_ANON, nr_new_folios); 3446 } 3447 /* 3448 * If uniform split, the process is complete. 3449 * If non-uniform, continue splitting the folio at @split_at 3450 * as long as the next @split_order is >= @new_order. 3451 */ 3452 folio = page_folio(split_at); 3453 old_order = split_order; 3454 } 3455 3456 return 0; 3457 } 3458 3459 bool non_uniform_split_supported(struct folio *folio, unsigned int new_order, 3460 bool warns) 3461 { 3462 if (folio_test_anon(folio)) { 3463 /* order-1 is not supported for anonymous THP. */ 3464 VM_WARN_ONCE(warns && new_order == 1, 3465 "Cannot split to order-1 folio"); 3466 if (new_order == 1) 3467 return false; 3468 } else if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && 3469 !mapping_large_folio_support(folio->mapping)) { 3470 /* 3471 * No split if the file system does not support large folio. 3472 * Note that we might still have THPs in such mappings due to 3473 * CONFIG_READ_ONLY_THP_FOR_FS. But in that case, the mapping 3474 * does not actually support large folios properly. 3475 */ 3476 VM_WARN_ONCE(warns, 3477 "Cannot split file folio to non-0 order"); 3478 return false; 3479 } 3480 3481 /* Only swapping a whole PMD-mapped folio is supported */ 3482 if (folio_test_swapcache(folio)) { 3483 VM_WARN_ONCE(warns, 3484 "Cannot split swapcache folio to non-0 order"); 3485 return false; 3486 } 3487 3488 return true; 3489 } 3490 3491 /* See comments in non_uniform_split_supported() */ 3492 bool uniform_split_supported(struct folio *folio, unsigned int new_order, 3493 bool warns) 3494 { 3495 if (folio_test_anon(folio)) { 3496 VM_WARN_ONCE(warns && new_order == 1, 3497 "Cannot split to order-1 folio"); 3498 if (new_order == 1) 3499 return false; 3500 } else if (new_order) { 3501 if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && 3502 !mapping_large_folio_support(folio->mapping)) { 3503 VM_WARN_ONCE(warns, 3504 "Cannot split file folio to non-0 order"); 3505 return false; 3506 } 3507 } 3508 3509 if (new_order && folio_test_swapcache(folio)) { 3510 VM_WARN_ONCE(warns, 3511 "Cannot split swapcache folio to non-0 order"); 3512 return false; 3513 } 3514 3515 return true; 3516 } 3517 3518 /* 3519 * __folio_split: split a folio at @split_at to a @new_order folio 3520 * @folio: folio to split 3521 * @new_order: the order of the new folio 3522 * @split_at: a page within the new folio 3523 * @lock_at: a page within @folio to be left locked to caller 3524 * @list: after-split folios will be put on it if non NULL 3525 * @uniform_split: perform uniform split or not (non-uniform split) 3526 * 3527 * It calls __split_unmapped_folio() to perform uniform and non-uniform split. 3528 * It is in charge of checking whether the split is supported or not and 3529 * preparing @folio for __split_unmapped_folio(). 3530 * 3531 * After splitting, the after-split folio containing @lock_at remains locked 3532 * and others are unlocked: 3533 * 1. for uniform split, @lock_at points to one of @folio's subpages; 3534 * 2. for buddy allocator like (non-uniform) split, @lock_at points to @folio. 3535 * 3536 * return: 0: successful, <0 failed (if -ENOMEM is returned, @folio might be 3537 * split but not to @new_order, the caller needs to check) 3538 */ 3539 static int __folio_split(struct folio *folio, unsigned int new_order, 3540 struct page *split_at, struct page *lock_at, 3541 struct list_head *list, bool uniform_split) 3542 { 3543 struct deferred_split *ds_queue = get_deferred_split_queue(folio); 3544 XA_STATE(xas, &folio->mapping->i_pages, folio->index); 3545 struct folio *end_folio = folio_next(folio); 3546 bool is_anon = folio_test_anon(folio); 3547 struct address_space *mapping = NULL; 3548 struct anon_vma *anon_vma = NULL; 3549 int order = folio_order(folio); 3550 struct folio *new_folio, *next; 3551 int nr_shmem_dropped = 0; 3552 int remap_flags = 0; 3553 int extra_pins, ret; 3554 pgoff_t end; 3555 bool is_hzp; 3556 3557 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); 3558 VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio); 3559 3560 if (folio != page_folio(split_at) || folio != page_folio(lock_at)) 3561 return -EINVAL; 3562 3563 if (new_order >= folio_order(folio)) 3564 return -EINVAL; 3565 3566 if (uniform_split && !uniform_split_supported(folio, new_order, true)) 3567 return -EINVAL; 3568 3569 if (!uniform_split && 3570 !non_uniform_split_supported(folio, new_order, true)) 3571 return -EINVAL; 3572 3573 is_hzp = is_huge_zero_folio(folio); 3574 if (is_hzp) { 3575 pr_warn_ratelimited("Called split_huge_page for huge zero page\n"); 3576 return -EBUSY; 3577 } 3578 3579 if (folio_test_writeback(folio)) 3580 return -EBUSY; 3581 3582 if (is_anon) { 3583 /* 3584 * The caller does not necessarily hold an mmap_lock that would 3585 * prevent the anon_vma disappearing so we first we take a 3586 * reference to it and then lock the anon_vma for write. This 3587 * is similar to folio_lock_anon_vma_read except the write lock 3588 * is taken to serialise against parallel split or collapse 3589 * operations. 3590 */ 3591 anon_vma = folio_get_anon_vma(folio); 3592 if (!anon_vma) { 3593 ret = -EBUSY; 3594 goto out; 3595 } 3596 mapping = NULL; 3597 anon_vma_lock_write(anon_vma); 3598 } else { 3599 unsigned int min_order; 3600 gfp_t gfp; 3601 3602 mapping = folio->mapping; 3603 3604 /* Truncated ? */ 3605 /* 3606 * TODO: add support for large shmem folio in swap cache. 3607 * When shmem is in swap cache, mapping is NULL and 3608 * folio_test_swapcache() is true. 3609 */ 3610 if (!mapping) { 3611 ret = -EBUSY; 3612 goto out; 3613 } 3614 3615 min_order = mapping_min_folio_order(folio->mapping); 3616 if (new_order < min_order) { 3617 ret = -EINVAL; 3618 goto out; 3619 } 3620 3621 gfp = current_gfp_context(mapping_gfp_mask(mapping) & 3622 GFP_RECLAIM_MASK); 3623 3624 if (!filemap_release_folio(folio, gfp)) { 3625 ret = -EBUSY; 3626 goto out; 3627 } 3628 3629 if (uniform_split) { 3630 xas_set_order(&xas, folio->index, new_order); 3631 xas_split_alloc(&xas, folio, folio_order(folio), gfp); 3632 if (xas_error(&xas)) { 3633 ret = xas_error(&xas); 3634 goto out; 3635 } 3636 } 3637 3638 anon_vma = NULL; 3639 i_mmap_lock_read(mapping); 3640 3641 /* 3642 *__split_unmapped_folio() may need to trim off pages beyond 3643 * EOF: but on 32-bit, i_size_read() takes an irq-unsafe 3644 * seqlock, which cannot be nested inside the page tree lock. 3645 * So note end now: i_size itself may be changed at any moment, 3646 * but folio lock is good enough to serialize the trimming. 3647 */ 3648 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); 3649 if (shmem_mapping(mapping)) 3650 end = shmem_fallocend(mapping->host, end); 3651 } 3652 3653 /* 3654 * Racy check if we can split the page, before unmap_folio() will 3655 * split PMDs 3656 */ 3657 if (!can_split_folio(folio, 1, &extra_pins)) { 3658 ret = -EAGAIN; 3659 goto out_unlock; 3660 } 3661 3662 unmap_folio(folio); 3663 3664 /* block interrupt reentry in xa_lock and spinlock */ 3665 local_irq_disable(); 3666 if (mapping) { 3667 /* 3668 * Check if the folio is present in page cache. 3669 * We assume all tail are present too, if folio is there. 3670 */ 3671 xas_lock(&xas); 3672 xas_reset(&xas); 3673 if (xas_load(&xas) != folio) { 3674 ret = -EAGAIN; 3675 goto fail; 3676 } 3677 } 3678 3679 /* Prevent deferred_split_scan() touching ->_refcount */ 3680 spin_lock(&ds_queue->split_queue_lock); 3681 if (folio_ref_freeze(folio, 1 + extra_pins)) { 3682 struct swap_cluster_info *ci = NULL; 3683 struct lruvec *lruvec; 3684 int expected_refs; 3685 3686 if (folio_order(folio) > 1 && 3687 !list_empty(&folio->_deferred_list)) { 3688 ds_queue->split_queue_len--; 3689 if (folio_test_partially_mapped(folio)) { 3690 folio_clear_partially_mapped(folio); 3691 mod_mthp_stat(folio_order(folio), 3692 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 3693 } 3694 /* 3695 * Reinitialize page_deferred_list after removing the 3696 * page from the split_queue, otherwise a subsequent 3697 * split will see list corruption when checking the 3698 * page_deferred_list. 3699 */ 3700 list_del_init(&folio->_deferred_list); 3701 } 3702 spin_unlock(&ds_queue->split_queue_lock); 3703 if (mapping) { 3704 int nr = folio_nr_pages(folio); 3705 3706 if (folio_test_pmd_mappable(folio) && 3707 new_order < HPAGE_PMD_ORDER) { 3708 if (folio_test_swapbacked(folio)) { 3709 __lruvec_stat_mod_folio(folio, 3710 NR_SHMEM_THPS, -nr); 3711 } else { 3712 __lruvec_stat_mod_folio(folio, 3713 NR_FILE_THPS, -nr); 3714 filemap_nr_thps_dec(mapping); 3715 } 3716 } 3717 } 3718 3719 if (folio_test_swapcache(folio)) { 3720 if (mapping) { 3721 VM_WARN_ON_ONCE_FOLIO(mapping, folio); 3722 ret = -EINVAL; 3723 goto fail; 3724 } 3725 3726 ci = swap_cluster_get_and_lock(folio); 3727 } 3728 3729 /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ 3730 lruvec = folio_lruvec_lock(folio); 3731 3732 ret = __split_unmapped_folio(folio, new_order, split_at, &xas, 3733 mapping, uniform_split); 3734 3735 /* 3736 * Unfreeze after-split folios and put them back to the right 3737 * list. @folio should be kept frozon until page cache 3738 * entries are updated with all the other after-split folios 3739 * to prevent others seeing stale page cache entries. 3740 * As a result, new_folio starts from the next folio of 3741 * @folio. 3742 */ 3743 for (new_folio = folio_next(folio); new_folio != end_folio; 3744 new_folio = next) { 3745 unsigned long nr_pages = folio_nr_pages(new_folio); 3746 3747 next = folio_next(new_folio); 3748 3749 expected_refs = folio_expected_ref_count(new_folio) + 1; 3750 folio_ref_unfreeze(new_folio, expected_refs); 3751 3752 lru_add_split_folio(folio, new_folio, lruvec, list); 3753 3754 /* 3755 * Anonymous folio with swap cache. 3756 * NOTE: shmem in swap cache is not supported yet. 3757 */ 3758 if (ci) { 3759 __swap_cache_replace_folio(ci, folio, new_folio); 3760 continue; 3761 } 3762 3763 /* Anonymous folio without swap cache */ 3764 if (!mapping) 3765 continue; 3766 3767 /* Add the new folio to the page cache. */ 3768 if (new_folio->index < end) { 3769 __xa_store(&mapping->i_pages, new_folio->index, 3770 new_folio, 0); 3771 continue; 3772 } 3773 3774 /* Drop folio beyond EOF: ->index >= end */ 3775 if (shmem_mapping(mapping)) 3776 nr_shmem_dropped += nr_pages; 3777 else if (folio_test_clear_dirty(new_folio)) 3778 folio_account_cleaned( 3779 new_folio, inode_to_wb(mapping->host)); 3780 __filemap_remove_folio(new_folio, NULL); 3781 folio_put_refs(new_folio, nr_pages); 3782 } 3783 /* 3784 * Unfreeze @folio only after all page cache entries, which 3785 * used to point to it, have been updated with new folios. 3786 * Otherwise, a parallel folio_try_get() can grab @folio 3787 * and its caller can see stale page cache entries. 3788 */ 3789 expected_refs = folio_expected_ref_count(folio) + 1; 3790 folio_ref_unfreeze(folio, expected_refs); 3791 3792 unlock_page_lruvec(lruvec); 3793 3794 if (ci) 3795 swap_cluster_unlock(ci); 3796 } else { 3797 spin_unlock(&ds_queue->split_queue_lock); 3798 ret = -EAGAIN; 3799 } 3800 fail: 3801 if (mapping) 3802 xas_unlock(&xas); 3803 3804 local_irq_enable(); 3805 3806 if (nr_shmem_dropped) 3807 shmem_uncharge(mapping->host, nr_shmem_dropped); 3808 3809 if (!ret && is_anon) 3810 remap_flags = RMP_USE_SHARED_ZEROPAGE; 3811 remap_page(folio, 1 << order, remap_flags); 3812 3813 /* 3814 * Unlock all after-split folios except the one containing 3815 * @lock_at page. If @folio is not split, it will be kept locked. 3816 */ 3817 for (new_folio = folio; new_folio != end_folio; new_folio = next) { 3818 next = folio_next(new_folio); 3819 if (new_folio == page_folio(lock_at)) 3820 continue; 3821 3822 folio_unlock(new_folio); 3823 /* 3824 * Subpages may be freed if there wasn't any mapping 3825 * like if add_to_swap() is running on a lru page that 3826 * had its mapping zapped. And freeing these pages 3827 * requires taking the lru_lock so we do the put_page 3828 * of the tail pages after the split is complete. 3829 */ 3830 free_folio_and_swap_cache(new_folio); 3831 } 3832 3833 out_unlock: 3834 if (anon_vma) { 3835 anon_vma_unlock_write(anon_vma); 3836 put_anon_vma(anon_vma); 3837 } 3838 if (mapping) 3839 i_mmap_unlock_read(mapping); 3840 out: 3841 xas_destroy(&xas); 3842 if (order == HPAGE_PMD_ORDER) 3843 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); 3844 count_mthp_stat(order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED); 3845 return ret; 3846 } 3847 3848 /* 3849 * This function splits a large folio into smaller folios of order @new_order. 3850 * @page can point to any page of the large folio to split. The split operation 3851 * does not change the position of @page. 3852 * 3853 * Prerequisites: 3854 * 3855 * 1) The caller must hold a reference on the @page's owning folio, also known 3856 * as the large folio. 3857 * 3858 * 2) The large folio must be locked. 3859 * 3860 * 3) The folio must not be pinned. Any unexpected folio references, including 3861 * GUP pins, will result in the folio not getting split; instead, the caller 3862 * will receive an -EAGAIN. 3863 * 3864 * 4) @new_order > 1, usually. Splitting to order-1 anonymous folios is not 3865 * supported for non-file-backed folios, because folio->_deferred_list, which 3866 * is used by partially mapped folios, is stored in subpage 2, but an order-1 3867 * folio only has subpages 0 and 1. File-backed order-1 folios are supported, 3868 * since they do not use _deferred_list. 3869 * 3870 * After splitting, the caller's folio reference will be transferred to @page, 3871 * resulting in a raised refcount of @page after this call. The other pages may 3872 * be freed if they are not mapped. 3873 * 3874 * If @list is null, tail pages will be added to LRU list, otherwise, to @list. 3875 * 3876 * Pages in @new_order will inherit the mapping, flags, and so on from the 3877 * huge page. 3878 * 3879 * Returns 0 if the huge page was split successfully. 3880 * 3881 * Returns -EAGAIN if the folio has unexpected reference (e.g., GUP) or if 3882 * the folio was concurrently removed from the page cache. 3883 * 3884 * Returns -EBUSY when trying to split the huge zeropage, if the folio is 3885 * under writeback, if fs-specific folio metadata cannot currently be 3886 * released, or if some unexpected race happened (e.g., anon VMA disappeared, 3887 * truncation). 3888 * 3889 * Callers should ensure that the order respects the address space mapping 3890 * min-order if one is set for non-anonymous folios. 3891 * 3892 * Returns -EINVAL when trying to split to an order that is incompatible 3893 * with the folio. Splitting to order 0 is compatible with all folios. 3894 */ 3895 int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, 3896 unsigned int new_order) 3897 { 3898 struct folio *folio = page_folio(page); 3899 3900 return __folio_split(folio, new_order, &folio->page, page, list, true); 3901 } 3902 3903 /* 3904 * folio_split: split a folio at @split_at to a @new_order folio 3905 * @folio: folio to split 3906 * @new_order: the order of the new folio 3907 * @split_at: a page within the new folio 3908 * 3909 * return: 0: successful, <0 failed (if -ENOMEM is returned, @folio might be 3910 * split but not to @new_order, the caller needs to check) 3911 * 3912 * It has the same prerequisites and returns as 3913 * split_huge_page_to_list_to_order(). 3914 * 3915 * Split a folio at @split_at to a new_order folio, leave the 3916 * remaining subpages of the original folio as large as possible. For example, 3917 * in the case of splitting an order-9 folio at its third order-3 subpages to 3918 * an order-3 folio, there are 2^(9-3)=64 order-3 subpages in the order-9 folio. 3919 * After the split, there will be a group of folios with different orders and 3920 * the new folio containing @split_at is marked in bracket: 3921 * [order-4, {order-3}, order-3, order-5, order-6, order-7, order-8]. 3922 * 3923 * After split, folio is left locked for caller. 3924 */ 3925 int folio_split(struct folio *folio, unsigned int new_order, 3926 struct page *split_at, struct list_head *list) 3927 { 3928 return __folio_split(folio, new_order, split_at, &folio->page, list, 3929 false); 3930 } 3931 3932 int min_order_for_split(struct folio *folio) 3933 { 3934 if (folio_test_anon(folio)) 3935 return 0; 3936 3937 if (!folio->mapping) { 3938 if (folio_test_pmd_mappable(folio)) 3939 count_vm_event(THP_SPLIT_PAGE_FAILED); 3940 return -EBUSY; 3941 } 3942 3943 return mapping_min_folio_order(folio->mapping); 3944 } 3945 3946 int split_folio_to_list(struct folio *folio, struct list_head *list) 3947 { 3948 return split_huge_page_to_list_to_order(&folio->page, list, 0); 3949 } 3950 3951 /* 3952 * __folio_unqueue_deferred_split() is not to be called directly: 3953 * the folio_unqueue_deferred_split() inline wrapper in mm/internal.h 3954 * limits its calls to those folios which may have a _deferred_list for 3955 * queueing THP splits, and that list is (racily observed to be) non-empty. 3956 * 3957 * It is unsafe to call folio_unqueue_deferred_split() until folio refcount is 3958 * zero: because even when split_queue_lock is held, a non-empty _deferred_list 3959 * might be in use on deferred_split_scan()'s unlocked on-stack list. 3960 * 3961 * If memory cgroups are enabled, split_queue_lock is in the mem_cgroup: it is 3962 * therefore important to unqueue deferred split before changing folio memcg. 3963 */ 3964 bool __folio_unqueue_deferred_split(struct folio *folio) 3965 { 3966 struct deferred_split *ds_queue; 3967 unsigned long flags; 3968 bool unqueued = false; 3969 3970 WARN_ON_ONCE(folio_ref_count(folio)); 3971 WARN_ON_ONCE(!mem_cgroup_disabled() && !folio_memcg(folio)); 3972 3973 ds_queue = get_deferred_split_queue(folio); 3974 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 3975 if (!list_empty(&folio->_deferred_list)) { 3976 ds_queue->split_queue_len--; 3977 if (folio_test_partially_mapped(folio)) { 3978 folio_clear_partially_mapped(folio); 3979 mod_mthp_stat(folio_order(folio), 3980 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 3981 } 3982 list_del_init(&folio->_deferred_list); 3983 unqueued = true; 3984 } 3985 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 3986 3987 return unqueued; /* useful for debug warnings */ 3988 } 3989 3990 /* partially_mapped=false won't clear PG_partially_mapped folio flag */ 3991 void deferred_split_folio(struct folio *folio, bool partially_mapped) 3992 { 3993 struct deferred_split *ds_queue = get_deferred_split_queue(folio); 3994 #ifdef CONFIG_MEMCG 3995 struct mem_cgroup *memcg = folio_memcg(folio); 3996 #endif 3997 unsigned long flags; 3998 3999 /* 4000 * Order 1 folios have no space for a deferred list, but we also 4001 * won't waste much memory by not adding them to the deferred list. 4002 */ 4003 if (folio_order(folio) <= 1) 4004 return; 4005 4006 if (!partially_mapped && !split_underused_thp) 4007 return; 4008 4009 /* 4010 * Exclude swapcache: originally to avoid a corrupt deferred split 4011 * queue. Nowadays that is fully prevented by memcg1_swapout(); 4012 * but if page reclaim is already handling the same folio, it is 4013 * unnecessary to handle it again in the shrinker, so excluding 4014 * swapcache here may still be a useful optimization. 4015 */ 4016 if (folio_test_swapcache(folio)) 4017 return; 4018 4019 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 4020 if (partially_mapped) { 4021 if (!folio_test_partially_mapped(folio)) { 4022 folio_set_partially_mapped(folio); 4023 if (folio_test_pmd_mappable(folio)) 4024 count_vm_event(THP_DEFERRED_SPLIT_PAGE); 4025 count_mthp_stat(folio_order(folio), MTHP_STAT_SPLIT_DEFERRED); 4026 mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, 1); 4027 4028 } 4029 } else { 4030 /* partially mapped folios cannot become non-partially mapped */ 4031 VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio); 4032 } 4033 if (list_empty(&folio->_deferred_list)) { 4034 list_add_tail(&folio->_deferred_list, &ds_queue->split_queue); 4035 ds_queue->split_queue_len++; 4036 #ifdef CONFIG_MEMCG 4037 if (memcg) 4038 set_shrinker_bit(memcg, folio_nid(folio), 4039 deferred_split_shrinker->id); 4040 #endif 4041 } 4042 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 4043 } 4044 4045 static unsigned long deferred_split_count(struct shrinker *shrink, 4046 struct shrink_control *sc) 4047 { 4048 struct pglist_data *pgdata = NODE_DATA(sc->nid); 4049 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 4050 4051 #ifdef CONFIG_MEMCG 4052 if (sc->memcg) 4053 ds_queue = &sc->memcg->deferred_split_queue; 4054 #endif 4055 return READ_ONCE(ds_queue->split_queue_len); 4056 } 4057 4058 static bool thp_underused(struct folio *folio) 4059 { 4060 int num_zero_pages = 0, num_filled_pages = 0; 4061 int i; 4062 4063 if (khugepaged_max_ptes_none == HPAGE_PMD_NR - 1) 4064 return false; 4065 4066 if (folio_contain_hwpoisoned_page(folio)) 4067 return false; 4068 4069 for (i = 0; i < folio_nr_pages(folio); i++) { 4070 if (pages_identical(folio_page(folio, i), ZERO_PAGE(0))) { 4071 if (++num_zero_pages > khugepaged_max_ptes_none) 4072 return true; 4073 } else { 4074 /* 4075 * Another path for early exit once the number 4076 * of non-zero filled pages exceeds threshold. 4077 */ 4078 if (++num_filled_pages >= HPAGE_PMD_NR - khugepaged_max_ptes_none) 4079 return false; 4080 } 4081 } 4082 return false; 4083 } 4084 4085 static unsigned long deferred_split_scan(struct shrinker *shrink, 4086 struct shrink_control *sc) 4087 { 4088 struct pglist_data *pgdata = NODE_DATA(sc->nid); 4089 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 4090 unsigned long flags; 4091 LIST_HEAD(list); 4092 struct folio *folio, *next, *prev = NULL; 4093 int split = 0, removed = 0; 4094 4095 #ifdef CONFIG_MEMCG 4096 if (sc->memcg) 4097 ds_queue = &sc->memcg->deferred_split_queue; 4098 #endif 4099 4100 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 4101 /* Take pin on all head pages to avoid freeing them under us */ 4102 list_for_each_entry_safe(folio, next, &ds_queue->split_queue, 4103 _deferred_list) { 4104 if (folio_try_get(folio)) { 4105 list_move(&folio->_deferred_list, &list); 4106 } else { 4107 /* We lost race with folio_put() */ 4108 if (folio_test_partially_mapped(folio)) { 4109 folio_clear_partially_mapped(folio); 4110 mod_mthp_stat(folio_order(folio), 4111 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 4112 } 4113 list_del_init(&folio->_deferred_list); 4114 ds_queue->split_queue_len--; 4115 } 4116 if (!--sc->nr_to_scan) 4117 break; 4118 } 4119 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 4120 4121 list_for_each_entry_safe(folio, next, &list, _deferred_list) { 4122 bool did_split = false; 4123 bool underused = false; 4124 4125 if (!folio_test_partially_mapped(folio)) { 4126 /* 4127 * See try_to_map_unused_to_zeropage(): we cannot 4128 * optimize zero-filled pages after splitting an 4129 * mlocked folio. 4130 */ 4131 if (folio_test_mlocked(folio)) 4132 goto next; 4133 underused = thp_underused(folio); 4134 if (!underused) 4135 goto next; 4136 } 4137 if (!folio_trylock(folio)) 4138 goto next; 4139 if (!split_folio(folio)) { 4140 did_split = true; 4141 if (underused) 4142 count_vm_event(THP_UNDERUSED_SPLIT_PAGE); 4143 split++; 4144 } 4145 folio_unlock(folio); 4146 next: 4147 /* 4148 * split_folio() removes folio from list on success. 4149 * Only add back to the queue if folio is partially mapped. 4150 * If thp_underused returns false, or if split_folio fails 4151 * in the case it was underused, then consider it used and 4152 * don't add it back to split_queue. 4153 */ 4154 if (did_split) { 4155 ; /* folio already removed from list */ 4156 } else if (!folio_test_partially_mapped(folio)) { 4157 list_del_init(&folio->_deferred_list); 4158 removed++; 4159 } else { 4160 /* 4161 * That unlocked list_del_init() above would be unsafe, 4162 * unless its folio is separated from any earlier folios 4163 * left on the list (which may be concurrently unqueued) 4164 * by one safe folio with refcount still raised. 4165 */ 4166 swap(folio, prev); 4167 } 4168 if (folio) 4169 folio_put(folio); 4170 } 4171 4172 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 4173 list_splice_tail(&list, &ds_queue->split_queue); 4174 ds_queue->split_queue_len -= removed; 4175 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 4176 4177 if (prev) 4178 folio_put(prev); 4179 4180 /* 4181 * Stop shrinker if we didn't split any page, but the queue is empty. 4182 * This can happen if pages were freed under us. 4183 */ 4184 if (!split && list_empty(&ds_queue->split_queue)) 4185 return SHRINK_STOP; 4186 return split; 4187 } 4188 4189 #ifdef CONFIG_DEBUG_FS 4190 static void split_huge_pages_all(void) 4191 { 4192 struct zone *zone; 4193 struct page *page; 4194 struct folio *folio; 4195 unsigned long pfn, max_zone_pfn; 4196 unsigned long total = 0, split = 0; 4197 4198 pr_debug("Split all THPs\n"); 4199 for_each_zone(zone) { 4200 if (!managed_zone(zone)) 4201 continue; 4202 max_zone_pfn = zone_end_pfn(zone); 4203 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 4204 int nr_pages; 4205 4206 page = pfn_to_online_page(pfn); 4207 if (!page || PageTail(page)) 4208 continue; 4209 folio = page_folio(page); 4210 if (!folio_try_get(folio)) 4211 continue; 4212 4213 if (unlikely(page_folio(page) != folio)) 4214 goto next; 4215 4216 if (zone != folio_zone(folio)) 4217 goto next; 4218 4219 if (!folio_test_large(folio) 4220 || folio_test_hugetlb(folio) 4221 || !folio_test_lru(folio)) 4222 goto next; 4223 4224 total++; 4225 folio_lock(folio); 4226 nr_pages = folio_nr_pages(folio); 4227 if (!split_folio(folio)) 4228 split++; 4229 pfn += nr_pages - 1; 4230 folio_unlock(folio); 4231 next: 4232 folio_put(folio); 4233 cond_resched(); 4234 } 4235 } 4236 4237 pr_debug("%lu of %lu THP split\n", split, total); 4238 } 4239 4240 static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma) 4241 { 4242 return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) || 4243 is_vm_hugetlb_page(vma); 4244 } 4245 4246 static int split_huge_pages_pid(int pid, unsigned long vaddr_start, 4247 unsigned long vaddr_end, unsigned int new_order, 4248 long in_folio_offset) 4249 { 4250 int ret = 0; 4251 struct task_struct *task; 4252 struct mm_struct *mm; 4253 unsigned long total = 0, split = 0; 4254 unsigned long addr; 4255 4256 vaddr_start &= PAGE_MASK; 4257 vaddr_end &= PAGE_MASK; 4258 4259 task = find_get_task_by_vpid(pid); 4260 if (!task) { 4261 ret = -ESRCH; 4262 goto out; 4263 } 4264 4265 /* Find the mm_struct */ 4266 mm = get_task_mm(task); 4267 put_task_struct(task); 4268 4269 if (!mm) { 4270 ret = -EINVAL; 4271 goto out; 4272 } 4273 4274 pr_debug("Split huge pages in pid: %d, vaddr: [0x%lx - 0x%lx], new_order: %u, in_folio_offset: %ld\n", 4275 pid, vaddr_start, vaddr_end, new_order, in_folio_offset); 4276 4277 mmap_read_lock(mm); 4278 /* 4279 * always increase addr by PAGE_SIZE, since we could have a PTE page 4280 * table filled with PTE-mapped THPs, each of which is distinct. 4281 */ 4282 for (addr = vaddr_start; addr < vaddr_end; addr += PAGE_SIZE) { 4283 struct vm_area_struct *vma = vma_lookup(mm, addr); 4284 struct folio_walk fw; 4285 struct folio *folio; 4286 struct address_space *mapping; 4287 unsigned int target_order = new_order; 4288 4289 if (!vma) 4290 break; 4291 4292 /* skip special VMA and hugetlb VMA */ 4293 if (vma_not_suitable_for_thp_split(vma)) { 4294 addr = vma->vm_end; 4295 continue; 4296 } 4297 4298 folio = folio_walk_start(&fw, vma, addr, 0); 4299 if (!folio) 4300 continue; 4301 4302 if (!is_transparent_hugepage(folio)) 4303 goto next; 4304 4305 if (!folio_test_anon(folio)) { 4306 mapping = folio->mapping; 4307 target_order = max(new_order, 4308 mapping_min_folio_order(mapping)); 4309 } 4310 4311 if (target_order >= folio_order(folio)) 4312 goto next; 4313 4314 total++; 4315 /* 4316 * For folios with private, split_huge_page_to_list_to_order() 4317 * will try to drop it before split and then check if the folio 4318 * can be split or not. So skip the check here. 4319 */ 4320 if (!folio_test_private(folio) && 4321 !can_split_folio(folio, 0, NULL)) 4322 goto next; 4323 4324 if (!folio_trylock(folio)) 4325 goto next; 4326 folio_get(folio); 4327 folio_walk_end(&fw, vma); 4328 4329 if (!folio_test_anon(folio) && folio->mapping != mapping) 4330 goto unlock; 4331 4332 if (in_folio_offset < 0 || 4333 in_folio_offset >= folio_nr_pages(folio)) { 4334 if (!split_folio_to_order(folio, target_order)) 4335 split++; 4336 } else { 4337 struct page *split_at = folio_page(folio, 4338 in_folio_offset); 4339 if (!folio_split(folio, target_order, split_at, NULL)) 4340 split++; 4341 } 4342 4343 unlock: 4344 4345 folio_unlock(folio); 4346 folio_put(folio); 4347 4348 cond_resched(); 4349 continue; 4350 next: 4351 folio_walk_end(&fw, vma); 4352 cond_resched(); 4353 } 4354 mmap_read_unlock(mm); 4355 mmput(mm); 4356 4357 pr_debug("%lu of %lu THP split\n", split, total); 4358 4359 out: 4360 return ret; 4361 } 4362 4363 static int split_huge_pages_in_file(const char *file_path, pgoff_t off_start, 4364 pgoff_t off_end, unsigned int new_order, 4365 long in_folio_offset) 4366 { 4367 struct filename *file; 4368 struct file *candidate; 4369 struct address_space *mapping; 4370 int ret = -EINVAL; 4371 pgoff_t index; 4372 int nr_pages = 1; 4373 unsigned long total = 0, split = 0; 4374 unsigned int min_order; 4375 unsigned int target_order; 4376 4377 file = getname_kernel(file_path); 4378 if (IS_ERR(file)) 4379 return ret; 4380 4381 candidate = file_open_name(file, O_RDONLY, 0); 4382 if (IS_ERR(candidate)) 4383 goto out; 4384 4385 pr_debug("split file-backed THPs in file: %s, page offset: [0x%lx - 0x%lx], new_order: %u, in_folio_offset: %ld\n", 4386 file_path, off_start, off_end, new_order, in_folio_offset); 4387 4388 mapping = candidate->f_mapping; 4389 min_order = mapping_min_folio_order(mapping); 4390 target_order = max(new_order, min_order); 4391 4392 for (index = off_start; index < off_end; index += nr_pages) { 4393 struct folio *folio = filemap_get_folio(mapping, index); 4394 4395 nr_pages = 1; 4396 if (IS_ERR(folio)) 4397 continue; 4398 4399 if (!folio_test_large(folio)) 4400 goto next; 4401 4402 total++; 4403 nr_pages = folio_nr_pages(folio); 4404 4405 if (target_order >= folio_order(folio)) 4406 goto next; 4407 4408 if (!folio_trylock(folio)) 4409 goto next; 4410 4411 if (folio->mapping != mapping) 4412 goto unlock; 4413 4414 if (in_folio_offset < 0 || in_folio_offset >= nr_pages) { 4415 if (!split_folio_to_order(folio, target_order)) 4416 split++; 4417 } else { 4418 struct page *split_at = folio_page(folio, 4419 in_folio_offset); 4420 if (!folio_split(folio, target_order, split_at, NULL)) 4421 split++; 4422 } 4423 4424 unlock: 4425 folio_unlock(folio); 4426 next: 4427 folio_put(folio); 4428 cond_resched(); 4429 } 4430 4431 filp_close(candidate, NULL); 4432 ret = 0; 4433 4434 pr_debug("%lu of %lu file-backed THP split\n", split, total); 4435 out: 4436 putname(file); 4437 return ret; 4438 } 4439 4440 #define MAX_INPUT_BUF_SZ 255 4441 4442 static ssize_t split_huge_pages_write(struct file *file, const char __user *buf, 4443 size_t count, loff_t *ppops) 4444 { 4445 static DEFINE_MUTEX(split_debug_mutex); 4446 ssize_t ret; 4447 /* 4448 * hold pid, start_vaddr, end_vaddr, new_order or 4449 * file_path, off_start, off_end, new_order 4450 */ 4451 char input_buf[MAX_INPUT_BUF_SZ]; 4452 int pid; 4453 unsigned long vaddr_start, vaddr_end; 4454 unsigned int new_order = 0; 4455 long in_folio_offset = -1; 4456 4457 ret = mutex_lock_interruptible(&split_debug_mutex); 4458 if (ret) 4459 return ret; 4460 4461 ret = -EFAULT; 4462 4463 memset(input_buf, 0, MAX_INPUT_BUF_SZ); 4464 if (copy_from_user(input_buf, buf, min_t(size_t, count, MAX_INPUT_BUF_SZ))) 4465 goto out; 4466 4467 input_buf[MAX_INPUT_BUF_SZ - 1] = '\0'; 4468 4469 if (input_buf[0] == '/') { 4470 char *tok; 4471 char *tok_buf = input_buf; 4472 char file_path[MAX_INPUT_BUF_SZ]; 4473 pgoff_t off_start = 0, off_end = 0; 4474 size_t input_len = strlen(input_buf); 4475 4476 tok = strsep(&tok_buf, ","); 4477 if (tok && tok_buf) { 4478 strscpy(file_path, tok); 4479 } else { 4480 ret = -EINVAL; 4481 goto out; 4482 } 4483 4484 ret = sscanf(tok_buf, "0x%lx,0x%lx,%d,%ld", &off_start, &off_end, 4485 &new_order, &in_folio_offset); 4486 if (ret != 2 && ret != 3 && ret != 4) { 4487 ret = -EINVAL; 4488 goto out; 4489 } 4490 ret = split_huge_pages_in_file(file_path, off_start, off_end, 4491 new_order, in_folio_offset); 4492 if (!ret) 4493 ret = input_len; 4494 4495 goto out; 4496 } 4497 4498 ret = sscanf(input_buf, "%d,0x%lx,0x%lx,%d,%ld", &pid, &vaddr_start, 4499 &vaddr_end, &new_order, &in_folio_offset); 4500 if (ret == 1 && pid == 1) { 4501 split_huge_pages_all(); 4502 ret = strlen(input_buf); 4503 goto out; 4504 } else if (ret != 3 && ret != 4 && ret != 5) { 4505 ret = -EINVAL; 4506 goto out; 4507 } 4508 4509 ret = split_huge_pages_pid(pid, vaddr_start, vaddr_end, new_order, 4510 in_folio_offset); 4511 if (!ret) 4512 ret = strlen(input_buf); 4513 out: 4514 mutex_unlock(&split_debug_mutex); 4515 return ret; 4516 4517 } 4518 4519 static const struct file_operations split_huge_pages_fops = { 4520 .owner = THIS_MODULE, 4521 .write = split_huge_pages_write, 4522 }; 4523 4524 static int __init split_huge_pages_debugfs(void) 4525 { 4526 debugfs_create_file("split_huge_pages", 0200, NULL, NULL, 4527 &split_huge_pages_fops); 4528 return 0; 4529 } 4530 late_initcall(split_huge_pages_debugfs); 4531 #endif 4532 4533 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 4534 int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 4535 struct page *page) 4536 { 4537 struct folio *folio = page_folio(page); 4538 struct vm_area_struct *vma = pvmw->vma; 4539 struct mm_struct *mm = vma->vm_mm; 4540 unsigned long address = pvmw->address; 4541 bool anon_exclusive; 4542 pmd_t pmdval; 4543 swp_entry_t entry; 4544 pmd_t pmdswp; 4545 4546 if (!(pvmw->pmd && !pvmw->pte)) 4547 return 0; 4548 4549 flush_cache_range(vma, address, address + HPAGE_PMD_SIZE); 4550 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); 4551 4552 /* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. */ 4553 anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(page); 4554 if (anon_exclusive && folio_try_share_anon_rmap_pmd(folio, page)) { 4555 set_pmd_at(mm, address, pvmw->pmd, pmdval); 4556 return -EBUSY; 4557 } 4558 4559 if (pmd_dirty(pmdval)) 4560 folio_mark_dirty(folio); 4561 if (pmd_write(pmdval)) 4562 entry = make_writable_migration_entry(page_to_pfn(page)); 4563 else if (anon_exclusive) 4564 entry = make_readable_exclusive_migration_entry(page_to_pfn(page)); 4565 else 4566 entry = make_readable_migration_entry(page_to_pfn(page)); 4567 if (pmd_young(pmdval)) 4568 entry = make_migration_entry_young(entry); 4569 if (pmd_dirty(pmdval)) 4570 entry = make_migration_entry_dirty(entry); 4571 pmdswp = swp_entry_to_pmd(entry); 4572 if (pmd_soft_dirty(pmdval)) 4573 pmdswp = pmd_swp_mksoft_dirty(pmdswp); 4574 if (pmd_uffd_wp(pmdval)) 4575 pmdswp = pmd_swp_mkuffd_wp(pmdswp); 4576 set_pmd_at(mm, address, pvmw->pmd, pmdswp); 4577 folio_remove_rmap_pmd(folio, page, vma); 4578 folio_put(folio); 4579 trace_set_migration_pmd(address, pmd_val(pmdswp)); 4580 4581 return 0; 4582 } 4583 4584 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) 4585 { 4586 struct folio *folio = page_folio(new); 4587 struct vm_area_struct *vma = pvmw->vma; 4588 struct mm_struct *mm = vma->vm_mm; 4589 unsigned long address = pvmw->address; 4590 unsigned long haddr = address & HPAGE_PMD_MASK; 4591 pmd_t pmde; 4592 swp_entry_t entry; 4593 4594 if (!(pvmw->pmd && !pvmw->pte)) 4595 return; 4596 4597 entry = pmd_to_swp_entry(*pvmw->pmd); 4598 folio_get(folio); 4599 pmde = folio_mk_pmd(folio, READ_ONCE(vma->vm_page_prot)); 4600 if (pmd_swp_soft_dirty(*pvmw->pmd)) 4601 pmde = pmd_mksoft_dirty(pmde); 4602 if (is_writable_migration_entry(entry)) 4603 pmde = pmd_mkwrite(pmde, vma); 4604 if (pmd_swp_uffd_wp(*pvmw->pmd)) 4605 pmde = pmd_mkuffd_wp(pmde); 4606 if (!is_migration_entry_young(entry)) 4607 pmde = pmd_mkold(pmde); 4608 /* NOTE: this may contain setting soft-dirty on some archs */ 4609 if (folio_test_dirty(folio) && is_migration_entry_dirty(entry)) 4610 pmde = pmd_mkdirty(pmde); 4611 4612 if (folio_test_anon(folio)) { 4613 rmap_t rmap_flags = RMAP_NONE; 4614 4615 if (!is_readable_migration_entry(entry)) 4616 rmap_flags |= RMAP_EXCLUSIVE; 4617 4618 folio_add_anon_rmap_pmd(folio, new, vma, haddr, rmap_flags); 4619 } else { 4620 folio_add_file_rmap_pmd(folio, new, vma); 4621 } 4622 VM_BUG_ON(pmd_write(pmde) && folio_test_anon(folio) && !PageAnonExclusive(new)); 4623 set_pmd_at(mm, haddr, pvmw->pmd, pmde); 4624 4625 /* No need to invalidate - it was non-present before */ 4626 update_mmu_cache_pmd(vma, address, pvmw->pmd); 4627 trace_remove_migration_pmd(address, pmd_val(pmde)); 4628 } 4629 #endif 4630