1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2009 Red Hat, Inc. 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/mm.h> 9 #include <linux/sched.h> 10 #include <linux/sched/mm.h> 11 #include <linux/sched/numa_balancing.h> 12 #include <linux/highmem.h> 13 #include <linux/hugetlb.h> 14 #include <linux/mmu_notifier.h> 15 #include <linux/rmap.h> 16 #include <linux/swap.h> 17 #include <linux/shrinker.h> 18 #include <linux/mm_inline.h> 19 #include <linux/swapops.h> 20 #include <linux/backing-dev.h> 21 #include <linux/dax.h> 22 #include <linux/mm_types.h> 23 #include <linux/khugepaged.h> 24 #include <linux/freezer.h> 25 #include <linux/mman.h> 26 #include <linux/memremap.h> 27 #include <linux/pagemap.h> 28 #include <linux/debugfs.h> 29 #include <linux/migrate.h> 30 #include <linux/hashtable.h> 31 #include <linux/userfaultfd_k.h> 32 #include <linux/page_idle.h> 33 #include <linux/shmem_fs.h> 34 #include <linux/oom.h> 35 #include <linux/numa.h> 36 #include <linux/page_owner.h> 37 #include <linux/sched/sysctl.h> 38 #include <linux/memory-tiers.h> 39 #include <linux/compat.h> 40 #include <linux/pgalloc.h> 41 #include <linux/pgalloc_tag.h> 42 #include <linux/pagewalk.h> 43 44 #include <asm/tlb.h> 45 #include "internal.h" 46 #include "swap.h" 47 48 #define CREATE_TRACE_POINTS 49 #include <trace/events/thp.h> 50 51 /* 52 * By default, transparent hugepage support is disabled in order to avoid 53 * risking an increased memory footprint for applications that are not 54 * guaranteed to benefit from it. When transparent hugepage support is 55 * enabled, it is for all mappings, and khugepaged scans all mappings. 56 * Defrag is invoked by khugepaged hugepage allocations and by page faults 57 * for all hugepage allocations. 58 */ 59 unsigned long transparent_hugepage_flags __read_mostly = 60 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS 61 (1<<TRANSPARENT_HUGEPAGE_FLAG)| 62 #endif 63 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE 64 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| 65 #endif 66 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)| 67 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| 68 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 69 70 static struct shrinker *deferred_split_shrinker; 71 static unsigned long deferred_split_count(struct shrinker *shrink, 72 struct shrink_control *sc); 73 static unsigned long deferred_split_scan(struct shrinker *shrink, 74 struct shrink_control *sc); 75 static bool split_underused_thp = true; 76 77 static atomic_t huge_zero_refcount; 78 struct folio *huge_zero_folio __read_mostly; 79 unsigned long huge_zero_pfn __read_mostly = ~0UL; 80 unsigned long huge_anon_orders_always __read_mostly; 81 unsigned long huge_anon_orders_madvise __read_mostly; 82 unsigned long huge_anon_orders_inherit __read_mostly; 83 static bool anon_orders_configured __initdata; 84 85 static inline bool file_thp_enabled(struct vm_area_struct *vma) 86 { 87 struct inode *inode; 88 89 if (!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS)) 90 return false; 91 92 if (!vma->vm_file) 93 return false; 94 95 inode = file_inode(vma->vm_file); 96 97 return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); 98 } 99 100 unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, 101 vm_flags_t vm_flags, 102 enum tva_type type, 103 unsigned long orders) 104 { 105 const bool smaps = type == TVA_SMAPS; 106 const bool in_pf = type == TVA_PAGEFAULT; 107 const bool forced_collapse = type == TVA_FORCED_COLLAPSE; 108 unsigned long supported_orders; 109 110 /* Check the intersection of requested and supported orders. */ 111 if (vma_is_anonymous(vma)) 112 supported_orders = THP_ORDERS_ALL_ANON; 113 else if (vma_is_special_huge(vma)) 114 supported_orders = THP_ORDERS_ALL_SPECIAL; 115 else 116 supported_orders = THP_ORDERS_ALL_FILE_DEFAULT; 117 118 orders &= supported_orders; 119 if (!orders) 120 return 0; 121 122 if (!vma->vm_mm) /* vdso */ 123 return 0; 124 125 if (thp_disabled_by_hw() || vma_thp_disabled(vma, vm_flags, forced_collapse)) 126 return 0; 127 128 /* khugepaged doesn't collapse DAX vma, but page fault is fine. */ 129 if (vma_is_dax(vma)) 130 return in_pf ? orders : 0; 131 132 /* 133 * khugepaged special VMA and hugetlb VMA. 134 * Must be checked after dax since some dax mappings may have 135 * VM_MIXEDMAP set. 136 */ 137 if (!in_pf && !smaps && (vm_flags & VM_NO_KHUGEPAGED)) 138 return 0; 139 140 /* 141 * Check alignment for file vma and size for both file and anon vma by 142 * filtering out the unsuitable orders. 143 * 144 * Skip the check for page fault. Huge fault does the check in fault 145 * handlers. 146 */ 147 if (!in_pf) { 148 int order = highest_order(orders); 149 unsigned long addr; 150 151 while (orders) { 152 addr = vma->vm_end - (PAGE_SIZE << order); 153 if (thp_vma_suitable_order(vma, addr, order)) 154 break; 155 order = next_order(&orders, order); 156 } 157 158 if (!orders) 159 return 0; 160 } 161 162 /* 163 * Enabled via shmem mount options or sysfs settings. 164 * Must be done before hugepage flags check since shmem has its 165 * own flags. 166 */ 167 if (!in_pf && shmem_file(vma->vm_file)) 168 return orders & shmem_allowable_huge_orders(file_inode(vma->vm_file), 169 vma, vma->vm_pgoff, 0, 170 forced_collapse); 171 172 if (!vma_is_anonymous(vma)) { 173 /* 174 * Enforce THP collapse requirements as necessary. Anonymous vmas 175 * were already handled in thp_vma_allowable_orders(). 176 */ 177 if (!forced_collapse && 178 (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) && 179 !hugepage_global_always()))) 180 return 0; 181 182 /* 183 * Trust that ->huge_fault() handlers know what they are doing 184 * in fault path. 185 */ 186 if (((in_pf || smaps)) && vma->vm_ops->huge_fault) 187 return orders; 188 /* Only regular file is valid in collapse path */ 189 if (((!in_pf || smaps)) && file_thp_enabled(vma)) 190 return orders; 191 return 0; 192 } 193 194 if (vma_is_temporary_stack(vma)) 195 return 0; 196 197 /* 198 * THPeligible bit of smaps should show 1 for proper VMAs even 199 * though anon_vma is not initialized yet. 200 * 201 * Allow page fault since anon_vma may be not initialized until 202 * the first page fault. 203 */ 204 if (!vma->anon_vma) 205 return (smaps || in_pf) ? orders : 0; 206 207 return orders; 208 } 209 210 static bool get_huge_zero_folio(void) 211 { 212 struct folio *zero_folio; 213 retry: 214 if (likely(atomic_inc_not_zero(&huge_zero_refcount))) 215 return true; 216 217 zero_folio = folio_alloc((GFP_TRANSHUGE | __GFP_ZERO | __GFP_ZEROTAGS) & 218 ~__GFP_MOVABLE, 219 HPAGE_PMD_ORDER); 220 if (!zero_folio) { 221 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); 222 return false; 223 } 224 /* Ensure zero folio won't have large_rmappable flag set. */ 225 folio_clear_large_rmappable(zero_folio); 226 preempt_disable(); 227 if (cmpxchg(&huge_zero_folio, NULL, zero_folio)) { 228 preempt_enable(); 229 folio_put(zero_folio); 230 goto retry; 231 } 232 WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio)); 233 234 /* We take additional reference here. It will be put back by shrinker */ 235 atomic_set(&huge_zero_refcount, 2); 236 preempt_enable(); 237 count_vm_event(THP_ZERO_PAGE_ALLOC); 238 return true; 239 } 240 241 static void put_huge_zero_folio(void) 242 { 243 /* 244 * Counter should never go to zero here. Only shrinker can put 245 * last reference. 246 */ 247 BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); 248 } 249 250 struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) 251 { 252 if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) 253 return huge_zero_folio; 254 255 if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm)) 256 return READ_ONCE(huge_zero_folio); 257 258 if (!get_huge_zero_folio()) 259 return NULL; 260 261 if (mm_flags_test_and_set(MMF_HUGE_ZERO_FOLIO, mm)) 262 put_huge_zero_folio(); 263 264 return READ_ONCE(huge_zero_folio); 265 } 266 267 void mm_put_huge_zero_folio(struct mm_struct *mm) 268 { 269 if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) 270 return; 271 272 if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm)) 273 put_huge_zero_folio(); 274 } 275 276 static unsigned long shrink_huge_zero_folio_count(struct shrinker *shrink, 277 struct shrink_control *sc) 278 { 279 /* we can free zero page only if last reference remains */ 280 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; 281 } 282 283 static unsigned long shrink_huge_zero_folio_scan(struct shrinker *shrink, 284 struct shrink_control *sc) 285 { 286 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { 287 struct folio *zero_folio = xchg(&huge_zero_folio, NULL); 288 BUG_ON(zero_folio == NULL); 289 WRITE_ONCE(huge_zero_pfn, ~0UL); 290 folio_put(zero_folio); 291 return HPAGE_PMD_NR; 292 } 293 294 return 0; 295 } 296 297 static struct shrinker *huge_zero_folio_shrinker; 298 299 #ifdef CONFIG_SYSFS 300 static ssize_t enabled_show(struct kobject *kobj, 301 struct kobj_attribute *attr, char *buf) 302 { 303 const char *output; 304 305 if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags)) 306 output = "[always] madvise never"; 307 else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 308 &transparent_hugepage_flags)) 309 output = "always [madvise] never"; 310 else 311 output = "always madvise [never]"; 312 313 return sysfs_emit(buf, "%s\n", output); 314 } 315 316 static ssize_t enabled_store(struct kobject *kobj, 317 struct kobj_attribute *attr, 318 const char *buf, size_t count) 319 { 320 ssize_t ret = count; 321 322 if (sysfs_streq(buf, "always")) { 323 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 324 set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 325 } else if (sysfs_streq(buf, "madvise")) { 326 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 327 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 328 } else if (sysfs_streq(buf, "never")) { 329 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 330 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 331 } else 332 ret = -EINVAL; 333 334 if (ret > 0) { 335 int err = start_stop_khugepaged(); 336 if (err) 337 ret = err; 338 } 339 return ret; 340 } 341 342 static struct kobj_attribute enabled_attr = __ATTR_RW(enabled); 343 344 ssize_t single_hugepage_flag_show(struct kobject *kobj, 345 struct kobj_attribute *attr, char *buf, 346 enum transparent_hugepage_flag flag) 347 { 348 return sysfs_emit(buf, "%d\n", 349 !!test_bit(flag, &transparent_hugepage_flags)); 350 } 351 352 ssize_t single_hugepage_flag_store(struct kobject *kobj, 353 struct kobj_attribute *attr, 354 const char *buf, size_t count, 355 enum transparent_hugepage_flag flag) 356 { 357 unsigned long value; 358 int ret; 359 360 ret = kstrtoul(buf, 10, &value); 361 if (ret < 0) 362 return ret; 363 if (value > 1) 364 return -EINVAL; 365 366 if (value) 367 set_bit(flag, &transparent_hugepage_flags); 368 else 369 clear_bit(flag, &transparent_hugepage_flags); 370 371 return count; 372 } 373 374 static ssize_t defrag_show(struct kobject *kobj, 375 struct kobj_attribute *attr, char *buf) 376 { 377 const char *output; 378 379 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, 380 &transparent_hugepage_flags)) 381 output = "[always] defer defer+madvise madvise never"; 382 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, 383 &transparent_hugepage_flags)) 384 output = "always [defer] defer+madvise madvise never"; 385 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, 386 &transparent_hugepage_flags)) 387 output = "always defer [defer+madvise] madvise never"; 388 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, 389 &transparent_hugepage_flags)) 390 output = "always defer defer+madvise [madvise] never"; 391 else 392 output = "always defer defer+madvise madvise [never]"; 393 394 return sysfs_emit(buf, "%s\n", output); 395 } 396 397 static ssize_t defrag_store(struct kobject *kobj, 398 struct kobj_attribute *attr, 399 const char *buf, size_t count) 400 { 401 if (sysfs_streq(buf, "always")) { 402 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 403 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 404 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 405 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 406 } else if (sysfs_streq(buf, "defer+madvise")) { 407 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 408 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 409 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 410 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 411 } else if (sysfs_streq(buf, "defer")) { 412 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 413 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 414 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 415 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 416 } else if (sysfs_streq(buf, "madvise")) { 417 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 418 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 419 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 420 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 421 } else if (sysfs_streq(buf, "never")) { 422 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 423 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 424 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 425 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 426 } else 427 return -EINVAL; 428 429 return count; 430 } 431 static struct kobj_attribute defrag_attr = __ATTR_RW(defrag); 432 433 static ssize_t use_zero_page_show(struct kobject *kobj, 434 struct kobj_attribute *attr, char *buf) 435 { 436 return single_hugepage_flag_show(kobj, attr, buf, 437 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 438 } 439 static ssize_t use_zero_page_store(struct kobject *kobj, 440 struct kobj_attribute *attr, const char *buf, size_t count) 441 { 442 return single_hugepage_flag_store(kobj, attr, buf, count, 443 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 444 } 445 static struct kobj_attribute use_zero_page_attr = __ATTR_RW(use_zero_page); 446 447 static ssize_t hpage_pmd_size_show(struct kobject *kobj, 448 struct kobj_attribute *attr, char *buf) 449 { 450 return sysfs_emit(buf, "%lu\n", HPAGE_PMD_SIZE); 451 } 452 static struct kobj_attribute hpage_pmd_size_attr = 453 __ATTR_RO(hpage_pmd_size); 454 455 static ssize_t split_underused_thp_show(struct kobject *kobj, 456 struct kobj_attribute *attr, char *buf) 457 { 458 return sysfs_emit(buf, "%d\n", split_underused_thp); 459 } 460 461 static ssize_t split_underused_thp_store(struct kobject *kobj, 462 struct kobj_attribute *attr, 463 const char *buf, size_t count) 464 { 465 int err = kstrtobool(buf, &split_underused_thp); 466 467 if (err < 0) 468 return err; 469 470 return count; 471 } 472 473 static struct kobj_attribute split_underused_thp_attr = __ATTR( 474 shrink_underused, 0644, split_underused_thp_show, split_underused_thp_store); 475 476 static struct attribute *hugepage_attr[] = { 477 &enabled_attr.attr, 478 &defrag_attr.attr, 479 &use_zero_page_attr.attr, 480 &hpage_pmd_size_attr.attr, 481 #ifdef CONFIG_SHMEM 482 &shmem_enabled_attr.attr, 483 #endif 484 &split_underused_thp_attr.attr, 485 NULL, 486 }; 487 488 static const struct attribute_group hugepage_attr_group = { 489 .attrs = hugepage_attr, 490 }; 491 492 static void hugepage_exit_sysfs(struct kobject *hugepage_kobj); 493 static void thpsize_release(struct kobject *kobj); 494 static DEFINE_SPINLOCK(huge_anon_orders_lock); 495 static LIST_HEAD(thpsize_list); 496 497 static ssize_t anon_enabled_show(struct kobject *kobj, 498 struct kobj_attribute *attr, char *buf) 499 { 500 int order = to_thpsize(kobj)->order; 501 const char *output; 502 503 if (test_bit(order, &huge_anon_orders_always)) 504 output = "[always] inherit madvise never"; 505 else if (test_bit(order, &huge_anon_orders_inherit)) 506 output = "always [inherit] madvise never"; 507 else if (test_bit(order, &huge_anon_orders_madvise)) 508 output = "always inherit [madvise] never"; 509 else 510 output = "always inherit madvise [never]"; 511 512 return sysfs_emit(buf, "%s\n", output); 513 } 514 515 static ssize_t anon_enabled_store(struct kobject *kobj, 516 struct kobj_attribute *attr, 517 const char *buf, size_t count) 518 { 519 int order = to_thpsize(kobj)->order; 520 ssize_t ret = count; 521 522 if (sysfs_streq(buf, "always")) { 523 spin_lock(&huge_anon_orders_lock); 524 clear_bit(order, &huge_anon_orders_inherit); 525 clear_bit(order, &huge_anon_orders_madvise); 526 set_bit(order, &huge_anon_orders_always); 527 spin_unlock(&huge_anon_orders_lock); 528 } else if (sysfs_streq(buf, "inherit")) { 529 spin_lock(&huge_anon_orders_lock); 530 clear_bit(order, &huge_anon_orders_always); 531 clear_bit(order, &huge_anon_orders_madvise); 532 set_bit(order, &huge_anon_orders_inherit); 533 spin_unlock(&huge_anon_orders_lock); 534 } else if (sysfs_streq(buf, "madvise")) { 535 spin_lock(&huge_anon_orders_lock); 536 clear_bit(order, &huge_anon_orders_always); 537 clear_bit(order, &huge_anon_orders_inherit); 538 set_bit(order, &huge_anon_orders_madvise); 539 spin_unlock(&huge_anon_orders_lock); 540 } else if (sysfs_streq(buf, "never")) { 541 spin_lock(&huge_anon_orders_lock); 542 clear_bit(order, &huge_anon_orders_always); 543 clear_bit(order, &huge_anon_orders_inherit); 544 clear_bit(order, &huge_anon_orders_madvise); 545 spin_unlock(&huge_anon_orders_lock); 546 } else 547 ret = -EINVAL; 548 549 if (ret > 0) { 550 int err; 551 552 err = start_stop_khugepaged(); 553 if (err) 554 ret = err; 555 } 556 return ret; 557 } 558 559 static struct kobj_attribute anon_enabled_attr = 560 __ATTR(enabled, 0644, anon_enabled_show, anon_enabled_store); 561 562 static struct attribute *anon_ctrl_attrs[] = { 563 &anon_enabled_attr.attr, 564 NULL, 565 }; 566 567 static const struct attribute_group anon_ctrl_attr_grp = { 568 .attrs = anon_ctrl_attrs, 569 }; 570 571 static struct attribute *file_ctrl_attrs[] = { 572 #ifdef CONFIG_SHMEM 573 &thpsize_shmem_enabled_attr.attr, 574 #endif 575 NULL, 576 }; 577 578 static const struct attribute_group file_ctrl_attr_grp = { 579 .attrs = file_ctrl_attrs, 580 }; 581 582 static struct attribute *any_ctrl_attrs[] = { 583 NULL, 584 }; 585 586 static const struct attribute_group any_ctrl_attr_grp = { 587 .attrs = any_ctrl_attrs, 588 }; 589 590 static const struct kobj_type thpsize_ktype = { 591 .release = &thpsize_release, 592 .sysfs_ops = &kobj_sysfs_ops, 593 }; 594 595 DEFINE_PER_CPU(struct mthp_stat, mthp_stats) = {{{0}}}; 596 597 static unsigned long sum_mthp_stat(int order, enum mthp_stat_item item) 598 { 599 unsigned long sum = 0; 600 int cpu; 601 602 for_each_possible_cpu(cpu) { 603 struct mthp_stat *this = &per_cpu(mthp_stats, cpu); 604 605 sum += this->stats[order][item]; 606 } 607 608 return sum; 609 } 610 611 #define DEFINE_MTHP_STAT_ATTR(_name, _index) \ 612 static ssize_t _name##_show(struct kobject *kobj, \ 613 struct kobj_attribute *attr, char *buf) \ 614 { \ 615 int order = to_thpsize(kobj)->order; \ 616 \ 617 return sysfs_emit(buf, "%lu\n", sum_mthp_stat(order, _index)); \ 618 } \ 619 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 620 621 DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC); 622 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK); 623 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); 624 DEFINE_MTHP_STAT_ATTR(zswpout, MTHP_STAT_ZSWPOUT); 625 DEFINE_MTHP_STAT_ATTR(swpin, MTHP_STAT_SWPIN); 626 DEFINE_MTHP_STAT_ATTR(swpin_fallback, MTHP_STAT_SWPIN_FALLBACK); 627 DEFINE_MTHP_STAT_ATTR(swpin_fallback_charge, MTHP_STAT_SWPIN_FALLBACK_CHARGE); 628 DEFINE_MTHP_STAT_ATTR(swpout, MTHP_STAT_SWPOUT); 629 DEFINE_MTHP_STAT_ATTR(swpout_fallback, MTHP_STAT_SWPOUT_FALLBACK); 630 #ifdef CONFIG_SHMEM 631 DEFINE_MTHP_STAT_ATTR(shmem_alloc, MTHP_STAT_SHMEM_ALLOC); 632 DEFINE_MTHP_STAT_ATTR(shmem_fallback, MTHP_STAT_SHMEM_FALLBACK); 633 DEFINE_MTHP_STAT_ATTR(shmem_fallback_charge, MTHP_STAT_SHMEM_FALLBACK_CHARGE); 634 #endif 635 DEFINE_MTHP_STAT_ATTR(split, MTHP_STAT_SPLIT); 636 DEFINE_MTHP_STAT_ATTR(split_failed, MTHP_STAT_SPLIT_FAILED); 637 DEFINE_MTHP_STAT_ATTR(split_deferred, MTHP_STAT_SPLIT_DEFERRED); 638 DEFINE_MTHP_STAT_ATTR(nr_anon, MTHP_STAT_NR_ANON); 639 DEFINE_MTHP_STAT_ATTR(nr_anon_partially_mapped, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED); 640 641 static struct attribute *anon_stats_attrs[] = { 642 &anon_fault_alloc_attr.attr, 643 &anon_fault_fallback_attr.attr, 644 &anon_fault_fallback_charge_attr.attr, 645 #ifndef CONFIG_SHMEM 646 &zswpout_attr.attr, 647 &swpin_attr.attr, 648 &swpin_fallback_attr.attr, 649 &swpin_fallback_charge_attr.attr, 650 &swpout_attr.attr, 651 &swpout_fallback_attr.attr, 652 #endif 653 &split_deferred_attr.attr, 654 &nr_anon_attr.attr, 655 &nr_anon_partially_mapped_attr.attr, 656 NULL, 657 }; 658 659 static struct attribute_group anon_stats_attr_grp = { 660 .name = "stats", 661 .attrs = anon_stats_attrs, 662 }; 663 664 static struct attribute *file_stats_attrs[] = { 665 #ifdef CONFIG_SHMEM 666 &shmem_alloc_attr.attr, 667 &shmem_fallback_attr.attr, 668 &shmem_fallback_charge_attr.attr, 669 #endif 670 NULL, 671 }; 672 673 static struct attribute_group file_stats_attr_grp = { 674 .name = "stats", 675 .attrs = file_stats_attrs, 676 }; 677 678 static struct attribute *any_stats_attrs[] = { 679 #ifdef CONFIG_SHMEM 680 &zswpout_attr.attr, 681 &swpin_attr.attr, 682 &swpin_fallback_attr.attr, 683 &swpin_fallback_charge_attr.attr, 684 &swpout_attr.attr, 685 &swpout_fallback_attr.attr, 686 #endif 687 &split_attr.attr, 688 &split_failed_attr.attr, 689 NULL, 690 }; 691 692 static struct attribute_group any_stats_attr_grp = { 693 .name = "stats", 694 .attrs = any_stats_attrs, 695 }; 696 697 static int sysfs_add_group(struct kobject *kobj, 698 const struct attribute_group *grp) 699 { 700 int ret = -ENOENT; 701 702 /* 703 * If the group is named, try to merge first, assuming the subdirectory 704 * was already created. This avoids the warning emitted by 705 * sysfs_create_group() if the directory already exists. 706 */ 707 if (grp->name) 708 ret = sysfs_merge_group(kobj, grp); 709 if (ret) 710 ret = sysfs_create_group(kobj, grp); 711 712 return ret; 713 } 714 715 static struct thpsize *thpsize_create(int order, struct kobject *parent) 716 { 717 unsigned long size = (PAGE_SIZE << order) / SZ_1K; 718 struct thpsize *thpsize; 719 int ret = -ENOMEM; 720 721 thpsize = kzalloc(sizeof(*thpsize), GFP_KERNEL); 722 if (!thpsize) 723 goto err; 724 725 thpsize->order = order; 726 727 ret = kobject_init_and_add(&thpsize->kobj, &thpsize_ktype, parent, 728 "hugepages-%lukB", size); 729 if (ret) { 730 kfree(thpsize); 731 goto err; 732 } 733 734 735 ret = sysfs_add_group(&thpsize->kobj, &any_ctrl_attr_grp); 736 if (ret) 737 goto err_put; 738 739 ret = sysfs_add_group(&thpsize->kobj, &any_stats_attr_grp); 740 if (ret) 741 goto err_put; 742 743 if (BIT(order) & THP_ORDERS_ALL_ANON) { 744 ret = sysfs_add_group(&thpsize->kobj, &anon_ctrl_attr_grp); 745 if (ret) 746 goto err_put; 747 748 ret = sysfs_add_group(&thpsize->kobj, &anon_stats_attr_grp); 749 if (ret) 750 goto err_put; 751 } 752 753 if (BIT(order) & THP_ORDERS_ALL_FILE_DEFAULT) { 754 ret = sysfs_add_group(&thpsize->kobj, &file_ctrl_attr_grp); 755 if (ret) 756 goto err_put; 757 758 ret = sysfs_add_group(&thpsize->kobj, &file_stats_attr_grp); 759 if (ret) 760 goto err_put; 761 } 762 763 return thpsize; 764 err_put: 765 kobject_put(&thpsize->kobj); 766 err: 767 return ERR_PTR(ret); 768 } 769 770 static void thpsize_release(struct kobject *kobj) 771 { 772 kfree(to_thpsize(kobj)); 773 } 774 775 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj) 776 { 777 int err; 778 struct thpsize *thpsize; 779 unsigned long orders; 780 int order; 781 782 /* 783 * Default to setting PMD-sized THP to inherit the global setting and 784 * disable all other sizes. powerpc's PMD_ORDER isn't a compile-time 785 * constant so we have to do this here. 786 */ 787 if (!anon_orders_configured) 788 huge_anon_orders_inherit = BIT(PMD_ORDER); 789 790 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj); 791 if (unlikely(!*hugepage_kobj)) { 792 pr_err("failed to create transparent hugepage kobject\n"); 793 return -ENOMEM; 794 } 795 796 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group); 797 if (err) { 798 pr_err("failed to register transparent hugepage group\n"); 799 goto delete_obj; 800 } 801 802 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group); 803 if (err) { 804 pr_err("failed to register transparent hugepage group\n"); 805 goto remove_hp_group; 806 } 807 808 orders = THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_FILE_DEFAULT; 809 order = highest_order(orders); 810 while (orders) { 811 thpsize = thpsize_create(order, *hugepage_kobj); 812 if (IS_ERR(thpsize)) { 813 pr_err("failed to create thpsize for order %d\n", order); 814 err = PTR_ERR(thpsize); 815 goto remove_all; 816 } 817 list_add(&thpsize->node, &thpsize_list); 818 order = next_order(&orders, order); 819 } 820 821 return 0; 822 823 remove_all: 824 hugepage_exit_sysfs(*hugepage_kobj); 825 return err; 826 remove_hp_group: 827 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group); 828 delete_obj: 829 kobject_put(*hugepage_kobj); 830 return err; 831 } 832 833 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj) 834 { 835 struct thpsize *thpsize, *tmp; 836 837 list_for_each_entry_safe(thpsize, tmp, &thpsize_list, node) { 838 list_del(&thpsize->node); 839 kobject_put(&thpsize->kobj); 840 } 841 842 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group); 843 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group); 844 kobject_put(hugepage_kobj); 845 } 846 #else 847 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj) 848 { 849 return 0; 850 } 851 852 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) 853 { 854 } 855 #endif /* CONFIG_SYSFS */ 856 857 static int __init thp_shrinker_init(void) 858 { 859 deferred_split_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE | 860 SHRINKER_MEMCG_AWARE | 861 SHRINKER_NONSLAB, 862 "thp-deferred_split"); 863 if (!deferred_split_shrinker) 864 return -ENOMEM; 865 866 deferred_split_shrinker->count_objects = deferred_split_count; 867 deferred_split_shrinker->scan_objects = deferred_split_scan; 868 shrinker_register(deferred_split_shrinker); 869 870 if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) { 871 /* 872 * Bump the reference of the huge_zero_folio and do not 873 * initialize the shrinker. 874 * 875 * huge_zero_folio will always be NULL on failure. We assume 876 * that get_huge_zero_folio() will most likely not fail as 877 * thp_shrinker_init() is invoked early on during boot. 878 */ 879 if (!get_huge_zero_folio()) 880 pr_warn("Allocating persistent huge zero folio failed\n"); 881 return 0; 882 } 883 884 huge_zero_folio_shrinker = shrinker_alloc(0, "thp-zero"); 885 if (!huge_zero_folio_shrinker) { 886 shrinker_free(deferred_split_shrinker); 887 return -ENOMEM; 888 } 889 890 huge_zero_folio_shrinker->count_objects = shrink_huge_zero_folio_count; 891 huge_zero_folio_shrinker->scan_objects = shrink_huge_zero_folio_scan; 892 shrinker_register(huge_zero_folio_shrinker); 893 894 return 0; 895 } 896 897 static void __init thp_shrinker_exit(void) 898 { 899 shrinker_free(huge_zero_folio_shrinker); 900 shrinker_free(deferred_split_shrinker); 901 } 902 903 static int __init hugepage_init(void) 904 { 905 int err; 906 struct kobject *hugepage_kobj; 907 908 if (!has_transparent_hugepage()) { 909 transparent_hugepage_flags = 1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED; 910 return -EINVAL; 911 } 912 913 /* 914 * hugepages can't be allocated by the buddy allocator 915 */ 916 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER > MAX_PAGE_ORDER); 917 918 err = hugepage_init_sysfs(&hugepage_kobj); 919 if (err) 920 goto err_sysfs; 921 922 err = khugepaged_init(); 923 if (err) 924 goto err_slab; 925 926 err = thp_shrinker_init(); 927 if (err) 928 goto err_shrinker; 929 930 /* 931 * By default disable transparent hugepages on smaller systems, 932 * where the extra memory used could hurt more than TLB overhead 933 * is likely to save. The admin can still enable it through /sys. 934 */ 935 if (totalram_pages() < MB_TO_PAGES(512)) { 936 transparent_hugepage_flags = 0; 937 return 0; 938 } 939 940 err = start_stop_khugepaged(); 941 if (err) 942 goto err_khugepaged; 943 944 return 0; 945 err_khugepaged: 946 thp_shrinker_exit(); 947 err_shrinker: 948 khugepaged_destroy(); 949 err_slab: 950 hugepage_exit_sysfs(hugepage_kobj); 951 err_sysfs: 952 return err; 953 } 954 subsys_initcall(hugepage_init); 955 956 static int __init setup_transparent_hugepage(char *str) 957 { 958 int ret = 0; 959 if (!str) 960 goto out; 961 if (!strcmp(str, "always")) { 962 set_bit(TRANSPARENT_HUGEPAGE_FLAG, 963 &transparent_hugepage_flags); 964 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 965 &transparent_hugepage_flags); 966 ret = 1; 967 } else if (!strcmp(str, "madvise")) { 968 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 969 &transparent_hugepage_flags); 970 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 971 &transparent_hugepage_flags); 972 ret = 1; 973 } else if (!strcmp(str, "never")) { 974 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 975 &transparent_hugepage_flags); 976 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 977 &transparent_hugepage_flags); 978 ret = 1; 979 } 980 out: 981 if (!ret) 982 pr_warn("transparent_hugepage= cannot parse, ignored\n"); 983 return ret; 984 } 985 __setup("transparent_hugepage=", setup_transparent_hugepage); 986 987 static char str_dup[PAGE_SIZE] __initdata; 988 static int __init setup_thp_anon(char *str) 989 { 990 char *token, *range, *policy, *subtoken; 991 unsigned long always, inherit, madvise; 992 char *start_size, *end_size; 993 int start, end, nr; 994 char *p; 995 996 if (!str || strlen(str) + 1 > PAGE_SIZE) 997 goto err; 998 strscpy(str_dup, str); 999 1000 always = huge_anon_orders_always; 1001 madvise = huge_anon_orders_madvise; 1002 inherit = huge_anon_orders_inherit; 1003 p = str_dup; 1004 while ((token = strsep(&p, ";")) != NULL) { 1005 range = strsep(&token, ":"); 1006 policy = token; 1007 1008 if (!policy) 1009 goto err; 1010 1011 while ((subtoken = strsep(&range, ",")) != NULL) { 1012 if (strchr(subtoken, '-')) { 1013 start_size = strsep(&subtoken, "-"); 1014 end_size = subtoken; 1015 1016 start = get_order_from_str(start_size, THP_ORDERS_ALL_ANON); 1017 end = get_order_from_str(end_size, THP_ORDERS_ALL_ANON); 1018 } else { 1019 start_size = end_size = subtoken; 1020 start = end = get_order_from_str(subtoken, 1021 THP_ORDERS_ALL_ANON); 1022 } 1023 1024 if (start == -EINVAL) { 1025 pr_err("invalid size %s in thp_anon boot parameter\n", start_size); 1026 goto err; 1027 } 1028 1029 if (end == -EINVAL) { 1030 pr_err("invalid size %s in thp_anon boot parameter\n", end_size); 1031 goto err; 1032 } 1033 1034 if (start < 0 || end < 0 || start > end) 1035 goto err; 1036 1037 nr = end - start + 1; 1038 if (!strcmp(policy, "always")) { 1039 bitmap_set(&always, start, nr); 1040 bitmap_clear(&inherit, start, nr); 1041 bitmap_clear(&madvise, start, nr); 1042 } else if (!strcmp(policy, "madvise")) { 1043 bitmap_set(&madvise, start, nr); 1044 bitmap_clear(&inherit, start, nr); 1045 bitmap_clear(&always, start, nr); 1046 } else if (!strcmp(policy, "inherit")) { 1047 bitmap_set(&inherit, start, nr); 1048 bitmap_clear(&madvise, start, nr); 1049 bitmap_clear(&always, start, nr); 1050 } else if (!strcmp(policy, "never")) { 1051 bitmap_clear(&inherit, start, nr); 1052 bitmap_clear(&madvise, start, nr); 1053 bitmap_clear(&always, start, nr); 1054 } else { 1055 pr_err("invalid policy %s in thp_anon boot parameter\n", policy); 1056 goto err; 1057 } 1058 } 1059 } 1060 1061 huge_anon_orders_always = always; 1062 huge_anon_orders_madvise = madvise; 1063 huge_anon_orders_inherit = inherit; 1064 anon_orders_configured = true; 1065 return 1; 1066 1067 err: 1068 pr_warn("thp_anon=%s: error parsing string, ignoring setting\n", str); 1069 return 0; 1070 } 1071 __setup("thp_anon=", setup_thp_anon); 1072 1073 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 1074 { 1075 if (likely(vma->vm_flags & VM_WRITE)) 1076 pmd = pmd_mkwrite(pmd, vma); 1077 return pmd; 1078 } 1079 1080 static struct deferred_split *split_queue_node(int nid) 1081 { 1082 struct pglist_data *pgdata = NODE_DATA(nid); 1083 1084 return &pgdata->deferred_split_queue; 1085 } 1086 1087 #ifdef CONFIG_MEMCG 1088 static inline 1089 struct mem_cgroup *folio_split_queue_memcg(struct folio *folio, 1090 struct deferred_split *queue) 1091 { 1092 if (mem_cgroup_disabled()) 1093 return NULL; 1094 if (split_queue_node(folio_nid(folio)) == queue) 1095 return NULL; 1096 return container_of(queue, struct mem_cgroup, deferred_split_queue); 1097 } 1098 1099 static struct deferred_split *memcg_split_queue(int nid, struct mem_cgroup *memcg) 1100 { 1101 return memcg ? &memcg->deferred_split_queue : split_queue_node(nid); 1102 } 1103 #else 1104 static inline 1105 struct mem_cgroup *folio_split_queue_memcg(struct folio *folio, 1106 struct deferred_split *queue) 1107 { 1108 return NULL; 1109 } 1110 1111 static struct deferred_split *memcg_split_queue(int nid, struct mem_cgroup *memcg) 1112 { 1113 return split_queue_node(nid); 1114 } 1115 #endif 1116 1117 static struct deferred_split *split_queue_lock(int nid, struct mem_cgroup *memcg) 1118 { 1119 struct deferred_split *queue; 1120 1121 retry: 1122 queue = memcg_split_queue(nid, memcg); 1123 spin_lock(&queue->split_queue_lock); 1124 /* 1125 * There is a period between setting memcg to dying and reparenting 1126 * deferred split queue, and during this period the THPs in the deferred 1127 * split queue will be hidden from the shrinker side. 1128 */ 1129 if (unlikely(memcg_is_dying(memcg))) { 1130 spin_unlock(&queue->split_queue_lock); 1131 memcg = parent_mem_cgroup(memcg); 1132 goto retry; 1133 } 1134 1135 return queue; 1136 } 1137 1138 static struct deferred_split * 1139 split_queue_lock_irqsave(int nid, struct mem_cgroup *memcg, unsigned long *flags) 1140 { 1141 struct deferred_split *queue; 1142 1143 retry: 1144 queue = memcg_split_queue(nid, memcg); 1145 spin_lock_irqsave(&queue->split_queue_lock, *flags); 1146 if (unlikely(memcg_is_dying(memcg))) { 1147 spin_unlock_irqrestore(&queue->split_queue_lock, *flags); 1148 memcg = parent_mem_cgroup(memcg); 1149 goto retry; 1150 } 1151 1152 return queue; 1153 } 1154 1155 static struct deferred_split *folio_split_queue_lock(struct folio *folio) 1156 { 1157 return split_queue_lock(folio_nid(folio), folio_memcg(folio)); 1158 } 1159 1160 static struct deferred_split * 1161 folio_split_queue_lock_irqsave(struct folio *folio, unsigned long *flags) 1162 { 1163 return split_queue_lock_irqsave(folio_nid(folio), folio_memcg(folio), flags); 1164 } 1165 1166 static inline void split_queue_unlock(struct deferred_split *queue) 1167 { 1168 spin_unlock(&queue->split_queue_lock); 1169 } 1170 1171 static inline void split_queue_unlock_irqrestore(struct deferred_split *queue, 1172 unsigned long flags) 1173 { 1174 spin_unlock_irqrestore(&queue->split_queue_lock, flags); 1175 } 1176 1177 static inline bool is_transparent_hugepage(const struct folio *folio) 1178 { 1179 if (!folio_test_large(folio)) 1180 return false; 1181 1182 return is_huge_zero_folio(folio) || 1183 folio_test_large_rmappable(folio); 1184 } 1185 1186 static unsigned long __thp_get_unmapped_area(struct file *filp, 1187 unsigned long addr, unsigned long len, 1188 loff_t off, unsigned long flags, unsigned long size, 1189 vm_flags_t vm_flags) 1190 { 1191 loff_t off_end = off + len; 1192 loff_t off_align = round_up(off, size); 1193 unsigned long len_pad, ret, off_sub; 1194 1195 if (!IS_ENABLED(CONFIG_64BIT) || in_compat_syscall()) 1196 return 0; 1197 1198 if (off_end <= off_align || (off_end - off_align) < size) 1199 return 0; 1200 1201 len_pad = len + size; 1202 if (len_pad < len || (off + len_pad) < off) 1203 return 0; 1204 1205 ret = mm_get_unmapped_area_vmflags(filp, addr, len_pad, 1206 off >> PAGE_SHIFT, flags, vm_flags); 1207 1208 /* 1209 * The failure might be due to length padding. The caller will retry 1210 * without the padding. 1211 */ 1212 if (IS_ERR_VALUE(ret)) 1213 return 0; 1214 1215 /* 1216 * Do not try to align to THP boundary if allocation at the address 1217 * hint succeeds. 1218 */ 1219 if (ret == addr) 1220 return addr; 1221 1222 off_sub = (off - ret) & (size - 1); 1223 1224 if (mm_flags_test(MMF_TOPDOWN, current->mm) && !off_sub) 1225 return ret + size; 1226 1227 ret += off_sub; 1228 return ret; 1229 } 1230 1231 unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, 1232 unsigned long len, unsigned long pgoff, unsigned long flags, 1233 vm_flags_t vm_flags) 1234 { 1235 unsigned long ret; 1236 loff_t off = (loff_t)pgoff << PAGE_SHIFT; 1237 1238 ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, vm_flags); 1239 if (ret) 1240 return ret; 1241 1242 return mm_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 1243 vm_flags); 1244 } 1245 1246 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, 1247 unsigned long len, unsigned long pgoff, unsigned long flags) 1248 { 1249 return thp_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0); 1250 } 1251 EXPORT_SYMBOL_GPL(thp_get_unmapped_area); 1252 1253 static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma, 1254 unsigned long addr) 1255 { 1256 gfp_t gfp = vma_thp_gfp_mask(vma); 1257 const int order = HPAGE_PMD_ORDER; 1258 struct folio *folio; 1259 1260 folio = vma_alloc_folio(gfp, order, vma, addr & HPAGE_PMD_MASK); 1261 1262 if (unlikely(!folio)) { 1263 count_vm_event(THP_FAULT_FALLBACK); 1264 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK); 1265 return NULL; 1266 } 1267 1268 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 1269 if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) { 1270 folio_put(folio); 1271 count_vm_event(THP_FAULT_FALLBACK); 1272 count_vm_event(THP_FAULT_FALLBACK_CHARGE); 1273 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK); 1274 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); 1275 return NULL; 1276 } 1277 folio_throttle_swaprate(folio, gfp); 1278 1279 /* 1280 * When a folio is not zeroed during allocation (__GFP_ZERO not used) 1281 * or user folios require special handling, folio_zero_user() is used to 1282 * make sure that the page corresponding to the faulting address will be 1283 * hot in the cache after zeroing. 1284 */ 1285 if (user_alloc_needs_zeroing()) 1286 folio_zero_user(folio, addr); 1287 /* 1288 * The memory barrier inside __folio_mark_uptodate makes sure that 1289 * folio_zero_user writes become visible before the set_pmd_at() 1290 * write. 1291 */ 1292 __folio_mark_uptodate(folio); 1293 return folio; 1294 } 1295 1296 void map_anon_folio_pmd_nopf(struct folio *folio, pmd_t *pmd, 1297 struct vm_area_struct *vma, unsigned long haddr) 1298 { 1299 pmd_t entry; 1300 1301 entry = folio_mk_pmd(folio, vma->vm_page_prot); 1302 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1303 folio_add_new_anon_rmap(folio, vma, haddr, RMAP_EXCLUSIVE); 1304 folio_add_lru_vma(folio, vma); 1305 set_pmd_at(vma->vm_mm, haddr, pmd, entry); 1306 update_mmu_cache_pmd(vma, haddr, pmd); 1307 deferred_split_folio(folio, false); 1308 } 1309 1310 static void map_anon_folio_pmd_pf(struct folio *folio, pmd_t *pmd, 1311 struct vm_area_struct *vma, unsigned long haddr) 1312 { 1313 map_anon_folio_pmd_nopf(folio, pmd, vma, haddr); 1314 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1315 count_vm_event(THP_FAULT_ALLOC); 1316 count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_ALLOC); 1317 count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); 1318 } 1319 1320 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf) 1321 { 1322 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1323 struct vm_area_struct *vma = vmf->vma; 1324 struct folio *folio; 1325 pgtable_t pgtable; 1326 vm_fault_t ret = 0; 1327 1328 folio = vma_alloc_anon_folio_pmd(vma, vmf->address); 1329 if (unlikely(!folio)) 1330 return VM_FAULT_FALLBACK; 1331 1332 pgtable = pte_alloc_one(vma->vm_mm); 1333 if (unlikely(!pgtable)) { 1334 ret = VM_FAULT_OOM; 1335 goto release; 1336 } 1337 1338 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1339 if (unlikely(!pmd_none(*vmf->pmd))) { 1340 goto unlock_release; 1341 } else { 1342 ret = check_stable_address_space(vma->vm_mm); 1343 if (ret) 1344 goto unlock_release; 1345 1346 /* Deliver the page fault to userland */ 1347 if (userfaultfd_missing(vma)) { 1348 spin_unlock(vmf->ptl); 1349 folio_put(folio); 1350 pte_free(vma->vm_mm, pgtable); 1351 ret = handle_userfault(vmf, VM_UFFD_MISSING); 1352 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 1353 return ret; 1354 } 1355 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); 1356 map_anon_folio_pmd_pf(folio, vmf->pmd, vma, haddr); 1357 mm_inc_nr_ptes(vma->vm_mm); 1358 spin_unlock(vmf->ptl); 1359 } 1360 1361 return 0; 1362 unlock_release: 1363 spin_unlock(vmf->ptl); 1364 release: 1365 if (pgtable) 1366 pte_free(vma->vm_mm, pgtable); 1367 folio_put(folio); 1368 return ret; 1369 1370 } 1371 1372 vm_fault_t do_huge_pmd_device_private(struct vm_fault *vmf) 1373 { 1374 struct vm_area_struct *vma = vmf->vma; 1375 vm_fault_t ret = 0; 1376 spinlock_t *ptl; 1377 softleaf_t entry; 1378 struct page *page; 1379 struct folio *folio; 1380 1381 if (vmf->flags & FAULT_FLAG_VMA_LOCK) { 1382 vma_end_read(vma); 1383 return VM_FAULT_RETRY; 1384 } 1385 1386 ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1387 if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd))) { 1388 spin_unlock(ptl); 1389 return 0; 1390 } 1391 1392 entry = softleaf_from_pmd(vmf->orig_pmd); 1393 page = softleaf_to_page(entry); 1394 folio = page_folio(page); 1395 vmf->page = page; 1396 vmf->pte = NULL; 1397 if (folio_trylock(folio)) { 1398 folio_get(folio); 1399 spin_unlock(ptl); 1400 ret = page_pgmap(page)->ops->migrate_to_ram(vmf); 1401 folio_unlock(folio); 1402 folio_put(folio); 1403 } else { 1404 spin_unlock(ptl); 1405 } 1406 1407 return ret; 1408 } 1409 1410 /* 1411 * always: directly stall for all thp allocations 1412 * defer: wake kswapd and fail if not immediately available 1413 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise 1414 * fail if not immediately available 1415 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately 1416 * available 1417 * never: never stall for any thp allocation 1418 */ 1419 gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma) 1420 { 1421 const bool vma_madvised = vma && (vma->vm_flags & VM_HUGEPAGE); 1422 1423 /* Always do synchronous compaction */ 1424 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) 1425 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY); 1426 1427 /* Kick kcompactd and fail quickly */ 1428 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) 1429 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM; 1430 1431 /* Synchronous compaction if madvised, otherwise kick kcompactd */ 1432 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags)) 1433 return GFP_TRANSHUGE_LIGHT | 1434 (vma_madvised ? __GFP_DIRECT_RECLAIM : 1435 __GFP_KSWAPD_RECLAIM); 1436 1437 /* Only do synchronous compaction if madvised */ 1438 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags)) 1439 return GFP_TRANSHUGE_LIGHT | 1440 (vma_madvised ? __GFP_DIRECT_RECLAIM : 0); 1441 1442 return GFP_TRANSHUGE_LIGHT; 1443 } 1444 1445 /* Caller must hold page table lock. */ 1446 static void set_huge_zero_folio(pgtable_t pgtable, struct mm_struct *mm, 1447 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, 1448 struct folio *zero_folio) 1449 { 1450 pmd_t entry; 1451 entry = folio_mk_pmd(zero_folio, vma->vm_page_prot); 1452 entry = pmd_mkspecial(entry); 1453 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1454 set_pmd_at(mm, haddr, pmd, entry); 1455 mm_inc_nr_ptes(mm); 1456 } 1457 1458 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) 1459 { 1460 struct vm_area_struct *vma = vmf->vma; 1461 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1462 vm_fault_t ret; 1463 1464 if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER)) 1465 return VM_FAULT_FALLBACK; 1466 ret = vmf_anon_prepare(vmf); 1467 if (ret) 1468 return ret; 1469 khugepaged_enter_vma(vma, vma->vm_flags); 1470 1471 if (!(vmf->flags & FAULT_FLAG_WRITE) && 1472 !mm_forbids_zeropage(vma->vm_mm) && 1473 transparent_hugepage_use_zero_page()) { 1474 pgtable_t pgtable; 1475 struct folio *zero_folio; 1476 vm_fault_t ret; 1477 1478 pgtable = pte_alloc_one(vma->vm_mm); 1479 if (unlikely(!pgtable)) 1480 return VM_FAULT_OOM; 1481 zero_folio = mm_get_huge_zero_folio(vma->vm_mm); 1482 if (unlikely(!zero_folio)) { 1483 pte_free(vma->vm_mm, pgtable); 1484 count_vm_event(THP_FAULT_FALLBACK); 1485 return VM_FAULT_FALLBACK; 1486 } 1487 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1488 ret = 0; 1489 if (pmd_none(*vmf->pmd)) { 1490 ret = check_stable_address_space(vma->vm_mm); 1491 if (ret) { 1492 spin_unlock(vmf->ptl); 1493 pte_free(vma->vm_mm, pgtable); 1494 } else if (userfaultfd_missing(vma)) { 1495 spin_unlock(vmf->ptl); 1496 pte_free(vma->vm_mm, pgtable); 1497 ret = handle_userfault(vmf, VM_UFFD_MISSING); 1498 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 1499 } else { 1500 set_huge_zero_folio(pgtable, vma->vm_mm, vma, 1501 haddr, vmf->pmd, zero_folio); 1502 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1503 spin_unlock(vmf->ptl); 1504 } 1505 } else { 1506 spin_unlock(vmf->ptl); 1507 pte_free(vma->vm_mm, pgtable); 1508 } 1509 return ret; 1510 } 1511 1512 return __do_huge_pmd_anonymous_page(vmf); 1513 } 1514 1515 struct folio_or_pfn { 1516 union { 1517 struct folio *folio; 1518 unsigned long pfn; 1519 }; 1520 bool is_folio; 1521 }; 1522 1523 static vm_fault_t insert_pmd(struct vm_area_struct *vma, unsigned long addr, 1524 pmd_t *pmd, struct folio_or_pfn fop, pgprot_t prot, 1525 bool write) 1526 { 1527 struct mm_struct *mm = vma->vm_mm; 1528 pgtable_t pgtable = NULL; 1529 spinlock_t *ptl; 1530 pmd_t entry; 1531 1532 if (addr < vma->vm_start || addr >= vma->vm_end) 1533 return VM_FAULT_SIGBUS; 1534 1535 if (arch_needs_pgtable_deposit()) { 1536 pgtable = pte_alloc_one(vma->vm_mm); 1537 if (!pgtable) 1538 return VM_FAULT_OOM; 1539 } 1540 1541 ptl = pmd_lock(mm, pmd); 1542 if (!pmd_none(*pmd)) { 1543 const unsigned long pfn = fop.is_folio ? folio_pfn(fop.folio) : 1544 fop.pfn; 1545 1546 if (write) { 1547 if (pmd_pfn(*pmd) != pfn) { 1548 WARN_ON_ONCE(!is_huge_zero_pmd(*pmd)); 1549 goto out_unlock; 1550 } 1551 entry = pmd_mkyoung(*pmd); 1552 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1553 if (pmdp_set_access_flags(vma, addr, pmd, entry, 1)) 1554 update_mmu_cache_pmd(vma, addr, pmd); 1555 } 1556 goto out_unlock; 1557 } 1558 1559 if (fop.is_folio) { 1560 entry = folio_mk_pmd(fop.folio, vma->vm_page_prot); 1561 1562 if (is_huge_zero_folio(fop.folio)) { 1563 entry = pmd_mkspecial(entry); 1564 } else { 1565 folio_get(fop.folio); 1566 folio_add_file_rmap_pmd(fop.folio, &fop.folio->page, vma); 1567 add_mm_counter(mm, mm_counter_file(fop.folio), HPAGE_PMD_NR); 1568 } 1569 } else { 1570 entry = pmd_mkhuge(pfn_pmd(fop.pfn, prot)); 1571 entry = pmd_mkspecial(entry); 1572 } 1573 if (write) { 1574 entry = pmd_mkyoung(pmd_mkdirty(entry)); 1575 entry = maybe_pmd_mkwrite(entry, vma); 1576 } 1577 1578 if (pgtable) { 1579 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1580 mm_inc_nr_ptes(mm); 1581 pgtable = NULL; 1582 } 1583 1584 set_pmd_at(mm, addr, pmd, entry); 1585 update_mmu_cache_pmd(vma, addr, pmd); 1586 1587 out_unlock: 1588 spin_unlock(ptl); 1589 if (pgtable) 1590 pte_free(mm, pgtable); 1591 return VM_FAULT_NOPAGE; 1592 } 1593 1594 /** 1595 * vmf_insert_pfn_pmd - insert a pmd size pfn 1596 * @vmf: Structure describing the fault 1597 * @pfn: pfn to insert 1598 * @write: whether it's a write fault 1599 * 1600 * Insert a pmd size pfn. See vmf_insert_pfn() for additional info. 1601 * 1602 * Return: vm_fault_t value. 1603 */ 1604 vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, unsigned long pfn, 1605 bool write) 1606 { 1607 unsigned long addr = vmf->address & PMD_MASK; 1608 struct vm_area_struct *vma = vmf->vma; 1609 pgprot_t pgprot = vma->vm_page_prot; 1610 struct folio_or_pfn fop = { 1611 .pfn = pfn, 1612 }; 1613 1614 /* 1615 * If we had pmd_special, we could avoid all these restrictions, 1616 * but we need to be consistent with PTEs and architectures that 1617 * can't support a 'special' bit. 1618 */ 1619 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 1620 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1621 (VM_PFNMAP|VM_MIXEDMAP)); 1622 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1623 1624 pfnmap_setup_cachemode_pfn(pfn, &pgprot); 1625 1626 return insert_pmd(vma, addr, vmf->pmd, fop, pgprot, write); 1627 } 1628 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd); 1629 1630 vm_fault_t vmf_insert_folio_pmd(struct vm_fault *vmf, struct folio *folio, 1631 bool write) 1632 { 1633 struct vm_area_struct *vma = vmf->vma; 1634 unsigned long addr = vmf->address & PMD_MASK; 1635 struct folio_or_pfn fop = { 1636 .folio = folio, 1637 .is_folio = true, 1638 }; 1639 1640 if (WARN_ON_ONCE(folio_order(folio) != PMD_ORDER)) 1641 return VM_FAULT_SIGBUS; 1642 1643 return insert_pmd(vma, addr, vmf->pmd, fop, vma->vm_page_prot, write); 1644 } 1645 EXPORT_SYMBOL_GPL(vmf_insert_folio_pmd); 1646 1647 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1648 static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma) 1649 { 1650 if (likely(vma->vm_flags & VM_WRITE)) 1651 pud = pud_mkwrite(pud); 1652 return pud; 1653 } 1654 1655 static vm_fault_t insert_pud(struct vm_area_struct *vma, unsigned long addr, 1656 pud_t *pud, struct folio_or_pfn fop, pgprot_t prot, bool write) 1657 { 1658 struct mm_struct *mm = vma->vm_mm; 1659 spinlock_t *ptl; 1660 pud_t entry; 1661 1662 if (addr < vma->vm_start || addr >= vma->vm_end) 1663 return VM_FAULT_SIGBUS; 1664 1665 ptl = pud_lock(mm, pud); 1666 if (!pud_none(*pud)) { 1667 const unsigned long pfn = fop.is_folio ? folio_pfn(fop.folio) : 1668 fop.pfn; 1669 1670 if (write) { 1671 if (WARN_ON_ONCE(pud_pfn(*pud) != pfn)) 1672 goto out_unlock; 1673 entry = pud_mkyoung(*pud); 1674 entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma); 1675 if (pudp_set_access_flags(vma, addr, pud, entry, 1)) 1676 update_mmu_cache_pud(vma, addr, pud); 1677 } 1678 goto out_unlock; 1679 } 1680 1681 if (fop.is_folio) { 1682 entry = folio_mk_pud(fop.folio, vma->vm_page_prot); 1683 1684 folio_get(fop.folio); 1685 folio_add_file_rmap_pud(fop.folio, &fop.folio->page, vma); 1686 add_mm_counter(mm, mm_counter_file(fop.folio), HPAGE_PUD_NR); 1687 } else { 1688 entry = pud_mkhuge(pfn_pud(fop.pfn, prot)); 1689 entry = pud_mkspecial(entry); 1690 } 1691 if (write) { 1692 entry = pud_mkyoung(pud_mkdirty(entry)); 1693 entry = maybe_pud_mkwrite(entry, vma); 1694 } 1695 set_pud_at(mm, addr, pud, entry); 1696 update_mmu_cache_pud(vma, addr, pud); 1697 out_unlock: 1698 spin_unlock(ptl); 1699 return VM_FAULT_NOPAGE; 1700 } 1701 1702 /** 1703 * vmf_insert_pfn_pud - insert a pud size pfn 1704 * @vmf: Structure describing the fault 1705 * @pfn: pfn to insert 1706 * @write: whether it's a write fault 1707 * 1708 * Insert a pud size pfn. See vmf_insert_pfn() for additional info. 1709 * 1710 * Return: vm_fault_t value. 1711 */ 1712 vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, unsigned long pfn, 1713 bool write) 1714 { 1715 unsigned long addr = vmf->address & PUD_MASK; 1716 struct vm_area_struct *vma = vmf->vma; 1717 pgprot_t pgprot = vma->vm_page_prot; 1718 struct folio_or_pfn fop = { 1719 .pfn = pfn, 1720 }; 1721 1722 /* 1723 * If we had pud_special, we could avoid all these restrictions, 1724 * but we need to be consistent with PTEs and architectures that 1725 * can't support a 'special' bit. 1726 */ 1727 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 1728 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1729 (VM_PFNMAP|VM_MIXEDMAP)); 1730 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1731 1732 pfnmap_setup_cachemode_pfn(pfn, &pgprot); 1733 1734 return insert_pud(vma, addr, vmf->pud, fop, pgprot, write); 1735 } 1736 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud); 1737 1738 /** 1739 * vmf_insert_folio_pud - insert a pud size folio mapped by a pud entry 1740 * @vmf: Structure describing the fault 1741 * @folio: folio to insert 1742 * @write: whether it's a write fault 1743 * 1744 * Return: vm_fault_t value. 1745 */ 1746 vm_fault_t vmf_insert_folio_pud(struct vm_fault *vmf, struct folio *folio, 1747 bool write) 1748 { 1749 struct vm_area_struct *vma = vmf->vma; 1750 unsigned long addr = vmf->address & PUD_MASK; 1751 struct folio_or_pfn fop = { 1752 .folio = folio, 1753 .is_folio = true, 1754 }; 1755 1756 if (WARN_ON_ONCE(folio_order(folio) != PUD_ORDER)) 1757 return VM_FAULT_SIGBUS; 1758 1759 return insert_pud(vma, addr, vmf->pud, fop, vma->vm_page_prot, write); 1760 } 1761 EXPORT_SYMBOL_GPL(vmf_insert_folio_pud); 1762 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1763 1764 void touch_pmd(struct vm_area_struct *vma, unsigned long addr, 1765 pmd_t *pmd, bool write) 1766 { 1767 pmd_t _pmd; 1768 1769 _pmd = pmd_mkyoung(*pmd); 1770 if (write) 1771 _pmd = pmd_mkdirty(_pmd); 1772 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, 1773 pmd, _pmd, write)) 1774 update_mmu_cache_pmd(vma, addr, pmd); 1775 } 1776 1777 static void copy_huge_non_present_pmd( 1778 struct mm_struct *dst_mm, struct mm_struct *src_mm, 1779 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 1780 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, 1781 pmd_t pmd, pgtable_t pgtable) 1782 { 1783 softleaf_t entry = softleaf_from_pmd(pmd); 1784 struct folio *src_folio; 1785 1786 VM_WARN_ON_ONCE(!pmd_is_valid_softleaf(pmd)); 1787 1788 if (softleaf_is_migration_write(entry) || 1789 softleaf_is_migration_read_exclusive(entry)) { 1790 entry = make_readable_migration_entry(swp_offset(entry)); 1791 pmd = swp_entry_to_pmd(entry); 1792 if (pmd_swp_soft_dirty(*src_pmd)) 1793 pmd = pmd_swp_mksoft_dirty(pmd); 1794 if (pmd_swp_uffd_wp(*src_pmd)) 1795 pmd = pmd_swp_mkuffd_wp(pmd); 1796 set_pmd_at(src_mm, addr, src_pmd, pmd); 1797 } else if (softleaf_is_device_private(entry)) { 1798 /* 1799 * For device private entries, since there are no 1800 * read exclusive entries, writable = !readable 1801 */ 1802 if (softleaf_is_device_private_write(entry)) { 1803 entry = make_readable_device_private_entry(swp_offset(entry)); 1804 pmd = swp_entry_to_pmd(entry); 1805 1806 if (pmd_swp_soft_dirty(*src_pmd)) 1807 pmd = pmd_swp_mksoft_dirty(pmd); 1808 if (pmd_swp_uffd_wp(*src_pmd)) 1809 pmd = pmd_swp_mkuffd_wp(pmd); 1810 set_pmd_at(src_mm, addr, src_pmd, pmd); 1811 } 1812 1813 src_folio = softleaf_to_folio(entry); 1814 VM_WARN_ON(!folio_test_large(src_folio)); 1815 1816 folio_get(src_folio); 1817 /* 1818 * folio_try_dup_anon_rmap_pmd does not fail for 1819 * device private entries. 1820 */ 1821 folio_try_dup_anon_rmap_pmd(src_folio, &src_folio->page, 1822 dst_vma, src_vma); 1823 } 1824 1825 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1826 mm_inc_nr_ptes(dst_mm); 1827 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1828 if (!userfaultfd_wp(dst_vma)) 1829 pmd = pmd_swp_clear_uffd_wp(pmd); 1830 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1831 } 1832 1833 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1834 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 1835 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) 1836 { 1837 spinlock_t *dst_ptl, *src_ptl; 1838 struct page *src_page; 1839 struct folio *src_folio; 1840 pmd_t pmd; 1841 pgtable_t pgtable = NULL; 1842 int ret = -ENOMEM; 1843 1844 pmd = pmdp_get_lockless(src_pmd); 1845 if (unlikely(pmd_present(pmd) && pmd_special(pmd) && 1846 !is_huge_zero_pmd(pmd))) { 1847 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1848 src_ptl = pmd_lockptr(src_mm, src_pmd); 1849 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1850 /* 1851 * No need to recheck the pmd, it can't change with write 1852 * mmap lock held here. 1853 * 1854 * Meanwhile, making sure it's not a CoW VMA with writable 1855 * mapping, otherwise it means either the anon page wrongly 1856 * applied special bit, or we made the PRIVATE mapping be 1857 * able to wrongly write to the backend MMIO. 1858 */ 1859 VM_WARN_ON_ONCE(is_cow_mapping(src_vma->vm_flags) && pmd_write(pmd)); 1860 goto set_pmd; 1861 } 1862 1863 /* Skip if can be re-fill on fault */ 1864 if (!vma_is_anonymous(dst_vma)) 1865 return 0; 1866 1867 pgtable = pte_alloc_one(dst_mm); 1868 if (unlikely(!pgtable)) 1869 goto out; 1870 1871 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1872 src_ptl = pmd_lockptr(src_mm, src_pmd); 1873 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1874 1875 ret = -EAGAIN; 1876 pmd = *src_pmd; 1877 1878 if (unlikely(thp_migration_supported() && 1879 pmd_is_valid_softleaf(pmd))) { 1880 copy_huge_non_present_pmd(dst_mm, src_mm, dst_pmd, src_pmd, addr, 1881 dst_vma, src_vma, pmd, pgtable); 1882 ret = 0; 1883 goto out_unlock; 1884 } 1885 1886 if (unlikely(!pmd_trans_huge(pmd))) { 1887 pte_free(dst_mm, pgtable); 1888 goto out_unlock; 1889 } 1890 /* 1891 * When page table lock is held, the huge zero pmd should not be 1892 * under splitting since we don't split the page itself, only pmd to 1893 * a page table. 1894 */ 1895 if (is_huge_zero_pmd(pmd)) { 1896 /* 1897 * mm_get_huge_zero_folio() will never allocate a new 1898 * folio here, since we already have a zero page to 1899 * copy. It just takes a reference. 1900 */ 1901 mm_get_huge_zero_folio(dst_mm); 1902 goto out_zero_page; 1903 } 1904 1905 src_page = pmd_page(pmd); 1906 VM_BUG_ON_PAGE(!PageHead(src_page), src_page); 1907 src_folio = page_folio(src_page); 1908 1909 folio_get(src_folio); 1910 if (unlikely(folio_try_dup_anon_rmap_pmd(src_folio, src_page, dst_vma, src_vma))) { 1911 /* Page maybe pinned: split and retry the fault on PTEs. */ 1912 folio_put(src_folio); 1913 pte_free(dst_mm, pgtable); 1914 spin_unlock(src_ptl); 1915 spin_unlock(dst_ptl); 1916 __split_huge_pmd(src_vma, src_pmd, addr, false); 1917 return -EAGAIN; 1918 } 1919 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1920 out_zero_page: 1921 mm_inc_nr_ptes(dst_mm); 1922 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1923 pmdp_set_wrprotect(src_mm, addr, src_pmd); 1924 if (!userfaultfd_wp(dst_vma)) 1925 pmd = pmd_clear_uffd_wp(pmd); 1926 pmd = pmd_wrprotect(pmd); 1927 set_pmd: 1928 pmd = pmd_mkold(pmd); 1929 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1930 1931 ret = 0; 1932 out_unlock: 1933 spin_unlock(src_ptl); 1934 spin_unlock(dst_ptl); 1935 out: 1936 return ret; 1937 } 1938 1939 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1940 void touch_pud(struct vm_area_struct *vma, unsigned long addr, 1941 pud_t *pud, bool write) 1942 { 1943 pud_t _pud; 1944 1945 _pud = pud_mkyoung(*pud); 1946 if (write) 1947 _pud = pud_mkdirty(_pud); 1948 if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK, 1949 pud, _pud, write)) 1950 update_mmu_cache_pud(vma, addr, pud); 1951 } 1952 1953 int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1954 pud_t *dst_pud, pud_t *src_pud, unsigned long addr, 1955 struct vm_area_struct *vma) 1956 { 1957 spinlock_t *dst_ptl, *src_ptl; 1958 pud_t pud; 1959 int ret; 1960 1961 dst_ptl = pud_lock(dst_mm, dst_pud); 1962 src_ptl = pud_lockptr(src_mm, src_pud); 1963 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1964 1965 ret = -EAGAIN; 1966 pud = *src_pud; 1967 if (unlikely(!pud_trans_huge(pud))) 1968 goto out_unlock; 1969 1970 /* 1971 * TODO: once we support anonymous pages, use 1972 * folio_try_dup_anon_rmap_*() and split if duplicating fails. 1973 */ 1974 if (is_cow_mapping(vma->vm_flags) && pud_write(pud)) { 1975 pudp_set_wrprotect(src_mm, addr, src_pud); 1976 pud = pud_wrprotect(pud); 1977 } 1978 pud = pud_mkold(pud); 1979 set_pud_at(dst_mm, addr, dst_pud, pud); 1980 1981 ret = 0; 1982 out_unlock: 1983 spin_unlock(src_ptl); 1984 spin_unlock(dst_ptl); 1985 return ret; 1986 } 1987 1988 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud) 1989 { 1990 bool write = vmf->flags & FAULT_FLAG_WRITE; 1991 1992 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); 1993 if (unlikely(!pud_same(*vmf->pud, orig_pud))) 1994 goto unlock; 1995 1996 touch_pud(vmf->vma, vmf->address, vmf->pud, write); 1997 unlock: 1998 spin_unlock(vmf->ptl); 1999 } 2000 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 2001 2002 void huge_pmd_set_accessed(struct vm_fault *vmf) 2003 { 2004 bool write = vmf->flags & FAULT_FLAG_WRITE; 2005 2006 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); 2007 if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd))) 2008 goto unlock; 2009 2010 touch_pmd(vmf->vma, vmf->address, vmf->pmd, write); 2011 2012 unlock: 2013 spin_unlock(vmf->ptl); 2014 } 2015 2016 static vm_fault_t do_huge_zero_wp_pmd(struct vm_fault *vmf) 2017 { 2018 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 2019 struct vm_area_struct *vma = vmf->vma; 2020 struct mmu_notifier_range range; 2021 struct folio *folio; 2022 vm_fault_t ret = 0; 2023 2024 folio = vma_alloc_anon_folio_pmd(vma, vmf->address); 2025 if (unlikely(!folio)) 2026 return VM_FAULT_FALLBACK; 2027 2028 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, haddr, 2029 haddr + HPAGE_PMD_SIZE); 2030 mmu_notifier_invalidate_range_start(&range); 2031 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 2032 if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) 2033 goto release; 2034 ret = check_stable_address_space(vma->vm_mm); 2035 if (ret) 2036 goto release; 2037 (void)pmdp_huge_clear_flush(vma, haddr, vmf->pmd); 2038 map_anon_folio_pmd_pf(folio, vmf->pmd, vma, haddr); 2039 goto unlock; 2040 release: 2041 folio_put(folio); 2042 unlock: 2043 spin_unlock(vmf->ptl); 2044 mmu_notifier_invalidate_range_end(&range); 2045 return ret; 2046 } 2047 2048 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) 2049 { 2050 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; 2051 struct vm_area_struct *vma = vmf->vma; 2052 struct folio *folio; 2053 struct page *page; 2054 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 2055 pmd_t orig_pmd = vmf->orig_pmd; 2056 2057 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); 2058 VM_BUG_ON_VMA(!vma->anon_vma, vma); 2059 2060 if (is_huge_zero_pmd(orig_pmd)) { 2061 vm_fault_t ret = do_huge_zero_wp_pmd(vmf); 2062 2063 if (!(ret & VM_FAULT_FALLBACK)) 2064 return ret; 2065 2066 /* Fallback to splitting PMD if THP cannot be allocated */ 2067 goto fallback; 2068 } 2069 2070 spin_lock(vmf->ptl); 2071 2072 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 2073 spin_unlock(vmf->ptl); 2074 return 0; 2075 } 2076 2077 page = pmd_page(orig_pmd); 2078 folio = page_folio(page); 2079 VM_BUG_ON_PAGE(!PageHead(page), page); 2080 2081 /* Early check when only holding the PT lock. */ 2082 if (PageAnonExclusive(page)) 2083 goto reuse; 2084 2085 if (!folio_trylock(folio)) { 2086 folio_get(folio); 2087 spin_unlock(vmf->ptl); 2088 folio_lock(folio); 2089 spin_lock(vmf->ptl); 2090 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 2091 spin_unlock(vmf->ptl); 2092 folio_unlock(folio); 2093 folio_put(folio); 2094 return 0; 2095 } 2096 folio_put(folio); 2097 } 2098 2099 /* Recheck after temporarily dropping the PT lock. */ 2100 if (PageAnonExclusive(page)) { 2101 folio_unlock(folio); 2102 goto reuse; 2103 } 2104 2105 /* 2106 * See do_wp_page(): we can only reuse the folio exclusively if 2107 * there are no additional references. Note that we always drain 2108 * the LRU cache immediately after adding a THP. 2109 */ 2110 if (folio_ref_count(folio) > 2111 1 + folio_test_swapcache(folio) * folio_nr_pages(folio)) 2112 goto unlock_fallback; 2113 if (folio_test_swapcache(folio)) 2114 folio_free_swap(folio); 2115 if (folio_ref_count(folio) == 1) { 2116 pmd_t entry; 2117 2118 folio_move_anon_rmap(folio, vma); 2119 SetPageAnonExclusive(page); 2120 folio_unlock(folio); 2121 reuse: 2122 if (unlikely(unshare)) { 2123 spin_unlock(vmf->ptl); 2124 return 0; 2125 } 2126 entry = pmd_mkyoung(orig_pmd); 2127 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 2128 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) 2129 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 2130 spin_unlock(vmf->ptl); 2131 return 0; 2132 } 2133 2134 unlock_fallback: 2135 folio_unlock(folio); 2136 spin_unlock(vmf->ptl); 2137 fallback: 2138 __split_huge_pmd(vma, vmf->pmd, vmf->address, false); 2139 return VM_FAULT_FALLBACK; 2140 } 2141 2142 static inline bool can_change_pmd_writable(struct vm_area_struct *vma, 2143 unsigned long addr, pmd_t pmd) 2144 { 2145 struct page *page; 2146 2147 if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) 2148 return false; 2149 2150 /* Don't touch entries that are not even readable (NUMA hinting). */ 2151 if (pmd_protnone(pmd)) 2152 return false; 2153 2154 /* Do we need write faults for softdirty tracking? */ 2155 if (pmd_needs_soft_dirty_wp(vma, pmd)) 2156 return false; 2157 2158 /* Do we need write faults for uffd-wp tracking? */ 2159 if (userfaultfd_huge_pmd_wp(vma, pmd)) 2160 return false; 2161 2162 if (!(vma->vm_flags & VM_SHARED)) { 2163 /* See can_change_pte_writable(). */ 2164 page = vm_normal_page_pmd(vma, addr, pmd); 2165 return page && PageAnon(page) && PageAnonExclusive(page); 2166 } 2167 2168 /* See can_change_pte_writable(). */ 2169 return pmd_dirty(pmd); 2170 } 2171 2172 /* NUMA hinting page fault entry point for trans huge pmds */ 2173 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) 2174 { 2175 struct vm_area_struct *vma = vmf->vma; 2176 struct folio *folio; 2177 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 2178 int nid = NUMA_NO_NODE; 2179 int target_nid, last_cpupid; 2180 pmd_t pmd, old_pmd; 2181 bool writable = false; 2182 int flags = 0; 2183 2184 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 2185 old_pmd = pmdp_get(vmf->pmd); 2186 2187 if (unlikely(!pmd_same(old_pmd, vmf->orig_pmd))) { 2188 spin_unlock(vmf->ptl); 2189 return 0; 2190 } 2191 2192 pmd = pmd_modify(old_pmd, vma->vm_page_prot); 2193 2194 /* 2195 * Detect now whether the PMD could be writable; this information 2196 * is only valid while holding the PT lock. 2197 */ 2198 writable = pmd_write(pmd); 2199 if (!writable && vma_wants_manual_pte_write_upgrade(vma) && 2200 can_change_pmd_writable(vma, vmf->address, pmd)) 2201 writable = true; 2202 2203 folio = vm_normal_folio_pmd(vma, haddr, pmd); 2204 if (!folio) 2205 goto out_map; 2206 2207 nid = folio_nid(folio); 2208 2209 target_nid = numa_migrate_check(folio, vmf, haddr, &flags, writable, 2210 &last_cpupid); 2211 if (target_nid == NUMA_NO_NODE) 2212 goto out_map; 2213 if (migrate_misplaced_folio_prepare(folio, vma, target_nid)) { 2214 flags |= TNF_MIGRATE_FAIL; 2215 goto out_map; 2216 } 2217 /* The folio is isolated and isolation code holds a folio reference. */ 2218 spin_unlock(vmf->ptl); 2219 writable = false; 2220 2221 if (!migrate_misplaced_folio(folio, target_nid)) { 2222 flags |= TNF_MIGRATED; 2223 nid = target_nid; 2224 task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags); 2225 return 0; 2226 } 2227 2228 flags |= TNF_MIGRATE_FAIL; 2229 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 2230 if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) { 2231 spin_unlock(vmf->ptl); 2232 return 0; 2233 } 2234 out_map: 2235 /* Restore the PMD */ 2236 pmd = pmd_modify(pmdp_get(vmf->pmd), vma->vm_page_prot); 2237 pmd = pmd_mkyoung(pmd); 2238 if (writable) 2239 pmd = pmd_mkwrite(pmd, vma); 2240 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); 2241 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 2242 spin_unlock(vmf->ptl); 2243 2244 if (nid != NUMA_NO_NODE) 2245 task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags); 2246 return 0; 2247 } 2248 2249 /* 2250 * Return true if we do MADV_FREE successfully on entire pmd page. 2251 * Otherwise, return false. 2252 */ 2253 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2254 pmd_t *pmd, unsigned long addr, unsigned long next) 2255 { 2256 spinlock_t *ptl; 2257 pmd_t orig_pmd; 2258 struct folio *folio; 2259 struct mm_struct *mm = tlb->mm; 2260 bool ret = false; 2261 2262 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2263 2264 ptl = pmd_trans_huge_lock(pmd, vma); 2265 if (!ptl) 2266 goto out_unlocked; 2267 2268 orig_pmd = *pmd; 2269 if (is_huge_zero_pmd(orig_pmd)) 2270 goto out; 2271 2272 if (unlikely(!pmd_present(orig_pmd))) { 2273 VM_BUG_ON(thp_migration_supported() && 2274 !pmd_is_migration_entry(orig_pmd)); 2275 goto out; 2276 } 2277 2278 folio = pmd_folio(orig_pmd); 2279 /* 2280 * If other processes are mapping this folio, we couldn't discard 2281 * the folio unless they all do MADV_FREE so let's skip the folio. 2282 */ 2283 if (folio_maybe_mapped_shared(folio)) 2284 goto out; 2285 2286 if (!folio_trylock(folio)) 2287 goto out; 2288 2289 /* 2290 * If user want to discard part-pages of THP, split it so MADV_FREE 2291 * will deactivate only them. 2292 */ 2293 if (next - addr != HPAGE_PMD_SIZE) { 2294 folio_get(folio); 2295 spin_unlock(ptl); 2296 split_folio(folio); 2297 folio_unlock(folio); 2298 folio_put(folio); 2299 goto out_unlocked; 2300 } 2301 2302 if (folio_test_dirty(folio)) 2303 folio_clear_dirty(folio); 2304 folio_unlock(folio); 2305 2306 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) { 2307 pmdp_invalidate(vma, addr, pmd); 2308 orig_pmd = pmd_mkold(orig_pmd); 2309 orig_pmd = pmd_mkclean(orig_pmd); 2310 2311 set_pmd_at(mm, addr, pmd, orig_pmd); 2312 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 2313 } 2314 2315 folio_mark_lazyfree(folio); 2316 ret = true; 2317 out: 2318 spin_unlock(ptl); 2319 out_unlocked: 2320 return ret; 2321 } 2322 2323 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd) 2324 { 2325 pgtable_t pgtable; 2326 2327 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2328 pte_free(mm, pgtable); 2329 mm_dec_nr_ptes(mm); 2330 } 2331 2332 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2333 pmd_t *pmd, unsigned long addr) 2334 { 2335 pmd_t orig_pmd; 2336 spinlock_t *ptl; 2337 2338 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2339 2340 ptl = __pmd_trans_huge_lock(pmd, vma); 2341 if (!ptl) 2342 return 0; 2343 /* 2344 * For architectures like ppc64 we look at deposited pgtable 2345 * when calling pmdp_huge_get_and_clear. So do the 2346 * pgtable_trans_huge_withdraw after finishing pmdp related 2347 * operations. 2348 */ 2349 orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd, 2350 tlb->fullmm); 2351 arch_check_zapped_pmd(vma, orig_pmd); 2352 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 2353 if (!vma_is_dax(vma) && vma_is_special_huge(vma)) { 2354 if (arch_needs_pgtable_deposit()) 2355 zap_deposited_table(tlb->mm, pmd); 2356 spin_unlock(ptl); 2357 } else if (is_huge_zero_pmd(orig_pmd)) { 2358 if (!vma_is_dax(vma) || arch_needs_pgtable_deposit()) 2359 zap_deposited_table(tlb->mm, pmd); 2360 spin_unlock(ptl); 2361 } else { 2362 struct folio *folio = NULL; 2363 int flush_needed = 1; 2364 2365 if (pmd_present(orig_pmd)) { 2366 struct page *page = pmd_page(orig_pmd); 2367 2368 folio = page_folio(page); 2369 folio_remove_rmap_pmd(folio, page, vma); 2370 WARN_ON_ONCE(folio_mapcount(folio) < 0); 2371 VM_BUG_ON_PAGE(!PageHead(page), page); 2372 } else if (pmd_is_valid_softleaf(orig_pmd)) { 2373 const softleaf_t entry = softleaf_from_pmd(orig_pmd); 2374 2375 folio = softleaf_to_folio(entry); 2376 flush_needed = 0; 2377 2378 if (!thp_migration_supported()) 2379 WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!"); 2380 } 2381 2382 if (folio_test_anon(folio)) { 2383 zap_deposited_table(tlb->mm, pmd); 2384 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); 2385 } else { 2386 if (arch_needs_pgtable_deposit()) 2387 zap_deposited_table(tlb->mm, pmd); 2388 add_mm_counter(tlb->mm, mm_counter_file(folio), 2389 -HPAGE_PMD_NR); 2390 2391 /* 2392 * Use flush_needed to indicate whether the PMD entry 2393 * is present, instead of checking pmd_present() again. 2394 */ 2395 if (flush_needed && pmd_young(orig_pmd) && 2396 likely(vma_has_recency(vma))) 2397 folio_mark_accessed(folio); 2398 } 2399 2400 if (folio_is_device_private(folio)) { 2401 folio_remove_rmap_pmd(folio, &folio->page, vma); 2402 WARN_ON_ONCE(folio_mapcount(folio) < 0); 2403 folio_put(folio); 2404 } 2405 2406 spin_unlock(ptl); 2407 if (flush_needed) 2408 tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE); 2409 } 2410 return 1; 2411 } 2412 2413 #ifndef pmd_move_must_withdraw 2414 static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl, 2415 spinlock_t *old_pmd_ptl, 2416 struct vm_area_struct *vma) 2417 { 2418 /* 2419 * With split pmd lock we also need to move preallocated 2420 * PTE page table if new_pmd is on different PMD page table. 2421 * 2422 * We also don't deposit and withdraw tables for file pages. 2423 */ 2424 return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma); 2425 } 2426 #endif 2427 2428 static pmd_t move_soft_dirty_pmd(pmd_t pmd) 2429 { 2430 if (pgtable_supports_soft_dirty()) { 2431 if (unlikely(pmd_is_migration_entry(pmd))) 2432 pmd = pmd_swp_mksoft_dirty(pmd); 2433 else if (pmd_present(pmd)) 2434 pmd = pmd_mksoft_dirty(pmd); 2435 } 2436 2437 return pmd; 2438 } 2439 2440 static pmd_t clear_uffd_wp_pmd(pmd_t pmd) 2441 { 2442 if (pmd_none(pmd)) 2443 return pmd; 2444 if (pmd_present(pmd)) 2445 pmd = pmd_clear_uffd_wp(pmd); 2446 else 2447 pmd = pmd_swp_clear_uffd_wp(pmd); 2448 2449 return pmd; 2450 } 2451 2452 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, 2453 unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd) 2454 { 2455 spinlock_t *old_ptl, *new_ptl; 2456 pmd_t pmd; 2457 struct mm_struct *mm = vma->vm_mm; 2458 bool force_flush = false; 2459 2460 /* 2461 * The destination pmd shouldn't be established, free_pgtables() 2462 * should have released it; but move_page_tables() might have already 2463 * inserted a page table, if racing against shmem/file collapse. 2464 */ 2465 if (!pmd_none(*new_pmd)) { 2466 VM_BUG_ON(pmd_trans_huge(*new_pmd)); 2467 return false; 2468 } 2469 2470 /* 2471 * We don't have to worry about the ordering of src and dst 2472 * ptlocks because exclusive mmap_lock prevents deadlock. 2473 */ 2474 old_ptl = __pmd_trans_huge_lock(old_pmd, vma); 2475 if (old_ptl) { 2476 new_ptl = pmd_lockptr(mm, new_pmd); 2477 if (new_ptl != old_ptl) 2478 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 2479 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); 2480 if (pmd_present(pmd)) 2481 force_flush = true; 2482 VM_BUG_ON(!pmd_none(*new_pmd)); 2483 2484 if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) { 2485 pgtable_t pgtable; 2486 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); 2487 pgtable_trans_huge_deposit(mm, new_pmd, pgtable); 2488 } 2489 pmd = move_soft_dirty_pmd(pmd); 2490 if (vma_has_uffd_without_event_remap(vma)) 2491 pmd = clear_uffd_wp_pmd(pmd); 2492 set_pmd_at(mm, new_addr, new_pmd, pmd); 2493 if (force_flush) 2494 flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE); 2495 if (new_ptl != old_ptl) 2496 spin_unlock(new_ptl); 2497 spin_unlock(old_ptl); 2498 return true; 2499 } 2500 return false; 2501 } 2502 2503 static void change_non_present_huge_pmd(struct mm_struct *mm, 2504 unsigned long addr, pmd_t *pmd, bool uffd_wp, 2505 bool uffd_wp_resolve) 2506 { 2507 softleaf_t entry = softleaf_from_pmd(*pmd); 2508 const struct folio *folio = softleaf_to_folio(entry); 2509 pmd_t newpmd; 2510 2511 VM_WARN_ON(!pmd_is_valid_softleaf(*pmd)); 2512 if (softleaf_is_migration_write(entry)) { 2513 /* 2514 * A protection check is difficult so 2515 * just be safe and disable write 2516 */ 2517 if (folio_test_anon(folio)) 2518 entry = make_readable_exclusive_migration_entry(swp_offset(entry)); 2519 else 2520 entry = make_readable_migration_entry(swp_offset(entry)); 2521 newpmd = swp_entry_to_pmd(entry); 2522 if (pmd_swp_soft_dirty(*pmd)) 2523 newpmd = pmd_swp_mksoft_dirty(newpmd); 2524 } else if (softleaf_is_device_private_write(entry)) { 2525 entry = make_readable_device_private_entry(swp_offset(entry)); 2526 newpmd = swp_entry_to_pmd(entry); 2527 } else { 2528 newpmd = *pmd; 2529 } 2530 2531 if (uffd_wp) 2532 newpmd = pmd_swp_mkuffd_wp(newpmd); 2533 else if (uffd_wp_resolve) 2534 newpmd = pmd_swp_clear_uffd_wp(newpmd); 2535 if (!pmd_same(*pmd, newpmd)) 2536 set_pmd_at(mm, addr, pmd, newpmd); 2537 } 2538 2539 /* 2540 * Returns 2541 * - 0 if PMD could not be locked 2542 * - 1 if PMD was locked but protections unchanged and TLB flush unnecessary 2543 * or if prot_numa but THP migration is not supported 2544 * - HPAGE_PMD_NR if protections changed and TLB flush necessary 2545 */ 2546 int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2547 pmd_t *pmd, unsigned long addr, pgprot_t newprot, 2548 unsigned long cp_flags) 2549 { 2550 struct mm_struct *mm = vma->vm_mm; 2551 spinlock_t *ptl; 2552 pmd_t oldpmd, entry; 2553 bool prot_numa = cp_flags & MM_CP_PROT_NUMA; 2554 bool uffd_wp = cp_flags & MM_CP_UFFD_WP; 2555 bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; 2556 int ret = 1; 2557 2558 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2559 2560 if (prot_numa && !thp_migration_supported()) 2561 return 1; 2562 2563 ptl = __pmd_trans_huge_lock(pmd, vma); 2564 if (!ptl) 2565 return 0; 2566 2567 if (thp_migration_supported() && pmd_is_valid_softleaf(*pmd)) { 2568 change_non_present_huge_pmd(mm, addr, pmd, uffd_wp, 2569 uffd_wp_resolve); 2570 goto unlock; 2571 } 2572 2573 if (prot_numa) { 2574 2575 /* 2576 * Avoid trapping faults against the zero page. The read-only 2577 * data is likely to be read-cached on the local CPU and 2578 * local/remote hits to the zero page are not interesting. 2579 */ 2580 if (is_huge_zero_pmd(*pmd)) 2581 goto unlock; 2582 2583 if (pmd_protnone(*pmd)) 2584 goto unlock; 2585 2586 if (!folio_can_map_prot_numa(pmd_folio(*pmd), vma, 2587 vma_is_single_threaded_private(vma))) 2588 goto unlock; 2589 } 2590 /* 2591 * In case prot_numa, we are under mmap_read_lock(mm). It's critical 2592 * to not clear pmd intermittently to avoid race with MADV_DONTNEED 2593 * which is also under mmap_read_lock(mm): 2594 * 2595 * CPU0: CPU1: 2596 * change_huge_pmd(prot_numa=1) 2597 * pmdp_huge_get_and_clear_notify() 2598 * madvise_dontneed() 2599 * zap_pmd_range() 2600 * pmd_trans_huge(*pmd) == 0 (without ptl) 2601 * // skip the pmd 2602 * set_pmd_at(); 2603 * // pmd is re-established 2604 * 2605 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it 2606 * which may break userspace. 2607 * 2608 * pmdp_invalidate_ad() is required to make sure we don't miss 2609 * dirty/young flags set by hardware. 2610 */ 2611 oldpmd = pmdp_invalidate_ad(vma, addr, pmd); 2612 2613 entry = pmd_modify(oldpmd, newprot); 2614 if (uffd_wp) 2615 entry = pmd_mkuffd_wp(entry); 2616 else if (uffd_wp_resolve) 2617 /* 2618 * Leave the write bit to be handled by PF interrupt 2619 * handler, then things like COW could be properly 2620 * handled. 2621 */ 2622 entry = pmd_clear_uffd_wp(entry); 2623 2624 /* See change_pte_range(). */ 2625 if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) && 2626 can_change_pmd_writable(vma, addr, entry)) 2627 entry = pmd_mkwrite(entry, vma); 2628 2629 ret = HPAGE_PMD_NR; 2630 set_pmd_at(mm, addr, pmd, entry); 2631 2632 if (huge_pmd_needs_flush(oldpmd, entry)) 2633 tlb_flush_pmd_range(tlb, addr, HPAGE_PMD_SIZE); 2634 unlock: 2635 spin_unlock(ptl); 2636 return ret; 2637 } 2638 2639 /* 2640 * Returns: 2641 * 2642 * - 0: if pud leaf changed from under us 2643 * - 1: if pud can be skipped 2644 * - HPAGE_PUD_NR: if pud was successfully processed 2645 */ 2646 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2647 int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 2648 pud_t *pudp, unsigned long addr, pgprot_t newprot, 2649 unsigned long cp_flags) 2650 { 2651 struct mm_struct *mm = vma->vm_mm; 2652 pud_t oldpud, entry; 2653 spinlock_t *ptl; 2654 2655 tlb_change_page_size(tlb, HPAGE_PUD_SIZE); 2656 2657 /* NUMA balancing doesn't apply to dax */ 2658 if (cp_flags & MM_CP_PROT_NUMA) 2659 return 1; 2660 2661 /* 2662 * Huge entries on userfault-wp only works with anonymous, while we 2663 * don't have anonymous PUDs yet. 2664 */ 2665 if (WARN_ON_ONCE(cp_flags & MM_CP_UFFD_WP_ALL)) 2666 return 1; 2667 2668 ptl = __pud_trans_huge_lock(pudp, vma); 2669 if (!ptl) 2670 return 0; 2671 2672 /* 2673 * Can't clear PUD or it can race with concurrent zapping. See 2674 * change_huge_pmd(). 2675 */ 2676 oldpud = pudp_invalidate(vma, addr, pudp); 2677 entry = pud_modify(oldpud, newprot); 2678 set_pud_at(mm, addr, pudp, entry); 2679 tlb_flush_pud_range(tlb, addr, HPAGE_PUD_SIZE); 2680 2681 spin_unlock(ptl); 2682 return HPAGE_PUD_NR; 2683 } 2684 #endif 2685 2686 #ifdef CONFIG_USERFAULTFD 2687 /* 2688 * The PT lock for src_pmd and dst_vma/src_vma (for reading) are locked by 2689 * the caller, but it must return after releasing the page_table_lock. 2690 * Just move the page from src_pmd to dst_pmd if possible. 2691 * Return zero if succeeded in moving the page, -EAGAIN if it needs to be 2692 * repeated by the caller, or other errors in case of failure. 2693 */ 2694 int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval, 2695 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, 2696 unsigned long dst_addr, unsigned long src_addr) 2697 { 2698 pmd_t _dst_pmd, src_pmdval; 2699 struct page *src_page; 2700 struct folio *src_folio; 2701 spinlock_t *src_ptl, *dst_ptl; 2702 pgtable_t src_pgtable; 2703 struct mmu_notifier_range range; 2704 int err = 0; 2705 2706 src_pmdval = *src_pmd; 2707 src_ptl = pmd_lockptr(mm, src_pmd); 2708 2709 lockdep_assert_held(src_ptl); 2710 vma_assert_locked(src_vma); 2711 vma_assert_locked(dst_vma); 2712 2713 /* Sanity checks before the operation */ 2714 if (WARN_ON_ONCE(!pmd_none(dst_pmdval)) || WARN_ON_ONCE(src_addr & ~HPAGE_PMD_MASK) || 2715 WARN_ON_ONCE(dst_addr & ~HPAGE_PMD_MASK)) { 2716 spin_unlock(src_ptl); 2717 return -EINVAL; 2718 } 2719 2720 if (!pmd_trans_huge(src_pmdval)) { 2721 spin_unlock(src_ptl); 2722 if (pmd_is_migration_entry(src_pmdval)) { 2723 pmd_migration_entry_wait(mm, &src_pmdval); 2724 return -EAGAIN; 2725 } 2726 return -ENOENT; 2727 } 2728 2729 src_page = pmd_page(src_pmdval); 2730 2731 if (!is_huge_zero_pmd(src_pmdval)) { 2732 if (unlikely(!PageAnonExclusive(src_page))) { 2733 spin_unlock(src_ptl); 2734 return -EBUSY; 2735 } 2736 2737 src_folio = page_folio(src_page); 2738 folio_get(src_folio); 2739 } else 2740 src_folio = NULL; 2741 2742 spin_unlock(src_ptl); 2743 2744 flush_cache_range(src_vma, src_addr, src_addr + HPAGE_PMD_SIZE); 2745 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, src_addr, 2746 src_addr + HPAGE_PMD_SIZE); 2747 mmu_notifier_invalidate_range_start(&range); 2748 2749 if (src_folio) 2750 folio_lock(src_folio); 2751 2752 dst_ptl = pmd_lockptr(mm, dst_pmd); 2753 double_pt_lock(src_ptl, dst_ptl); 2754 if (unlikely(!pmd_same(*src_pmd, src_pmdval) || 2755 !pmd_same(*dst_pmd, dst_pmdval))) { 2756 err = -EAGAIN; 2757 goto unlock_ptls; 2758 } 2759 if (src_folio) { 2760 if (folio_maybe_dma_pinned(src_folio) || 2761 !PageAnonExclusive(&src_folio->page)) { 2762 err = -EBUSY; 2763 goto unlock_ptls; 2764 } 2765 2766 if (WARN_ON_ONCE(!folio_test_head(src_folio)) || 2767 WARN_ON_ONCE(!folio_test_anon(src_folio))) { 2768 err = -EBUSY; 2769 goto unlock_ptls; 2770 } 2771 2772 src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); 2773 /* Folio got pinned from under us. Put it back and fail the move. */ 2774 if (folio_maybe_dma_pinned(src_folio)) { 2775 set_pmd_at(mm, src_addr, src_pmd, src_pmdval); 2776 err = -EBUSY; 2777 goto unlock_ptls; 2778 } 2779 2780 folio_move_anon_rmap(src_folio, dst_vma); 2781 src_folio->index = linear_page_index(dst_vma, dst_addr); 2782 2783 _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); 2784 /* Follow mremap() behavior and treat the entry dirty after the move */ 2785 _dst_pmd = pmd_mkwrite(pmd_mkdirty(_dst_pmd), dst_vma); 2786 } else { 2787 src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); 2788 _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); 2789 } 2790 set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd); 2791 2792 src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd); 2793 pgtable_trans_huge_deposit(mm, dst_pmd, src_pgtable); 2794 unlock_ptls: 2795 double_pt_unlock(src_ptl, dst_ptl); 2796 /* unblock rmap walks */ 2797 if (src_folio) 2798 folio_unlock(src_folio); 2799 mmu_notifier_invalidate_range_end(&range); 2800 if (src_folio) 2801 folio_put(src_folio); 2802 return err; 2803 } 2804 #endif /* CONFIG_USERFAULTFD */ 2805 2806 /* 2807 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise. 2808 * 2809 * Note that if it returns page table lock pointer, this routine returns without 2810 * unlocking page table lock. So callers must unlock it. 2811 */ 2812 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) 2813 { 2814 spinlock_t *ptl; 2815 2816 ptl = pmd_lock(vma->vm_mm, pmd); 2817 if (likely(pmd_is_huge(*pmd))) 2818 return ptl; 2819 spin_unlock(ptl); 2820 return NULL; 2821 } 2822 2823 /* 2824 * Returns page table lock pointer if a given pud maps a thp, NULL otherwise. 2825 * 2826 * Note that if it returns page table lock pointer, this routine returns without 2827 * unlocking page table lock. So callers must unlock it. 2828 */ 2829 spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma) 2830 { 2831 spinlock_t *ptl; 2832 2833 ptl = pud_lock(vma->vm_mm, pud); 2834 if (likely(pud_trans_huge(*pud))) 2835 return ptl; 2836 spin_unlock(ptl); 2837 return NULL; 2838 } 2839 2840 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2841 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 2842 pud_t *pud, unsigned long addr) 2843 { 2844 spinlock_t *ptl; 2845 pud_t orig_pud; 2846 2847 ptl = __pud_trans_huge_lock(pud, vma); 2848 if (!ptl) 2849 return 0; 2850 2851 orig_pud = pudp_huge_get_and_clear_full(vma, addr, pud, tlb->fullmm); 2852 arch_check_zapped_pud(vma, orig_pud); 2853 tlb_remove_pud_tlb_entry(tlb, pud, addr); 2854 if (!vma_is_dax(vma) && vma_is_special_huge(vma)) { 2855 spin_unlock(ptl); 2856 /* No zero page support yet */ 2857 } else { 2858 struct page *page = NULL; 2859 struct folio *folio; 2860 2861 /* No support for anonymous PUD pages or migration yet */ 2862 VM_WARN_ON_ONCE(vma_is_anonymous(vma) || 2863 !pud_present(orig_pud)); 2864 2865 page = pud_page(orig_pud); 2866 folio = page_folio(page); 2867 folio_remove_rmap_pud(folio, page, vma); 2868 add_mm_counter(tlb->mm, mm_counter_file(folio), -HPAGE_PUD_NR); 2869 2870 spin_unlock(ptl); 2871 tlb_remove_page_size(tlb, page, HPAGE_PUD_SIZE); 2872 } 2873 return 1; 2874 } 2875 2876 static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud, 2877 unsigned long haddr) 2878 { 2879 struct folio *folio; 2880 struct page *page; 2881 pud_t old_pud; 2882 2883 VM_BUG_ON(haddr & ~HPAGE_PUD_MASK); 2884 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2885 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma); 2886 VM_BUG_ON(!pud_trans_huge(*pud)); 2887 2888 count_vm_event(THP_SPLIT_PUD); 2889 2890 old_pud = pudp_huge_clear_flush(vma, haddr, pud); 2891 2892 if (!vma_is_dax(vma)) 2893 return; 2894 2895 page = pud_page(old_pud); 2896 folio = page_folio(page); 2897 2898 if (!folio_test_dirty(folio) && pud_dirty(old_pud)) 2899 folio_mark_dirty(folio); 2900 if (!folio_test_referenced(folio) && pud_young(old_pud)) 2901 folio_set_referenced(folio); 2902 folio_remove_rmap_pud(folio, page, vma); 2903 folio_put(folio); 2904 add_mm_counter(vma->vm_mm, mm_counter_file(folio), 2905 -HPAGE_PUD_NR); 2906 } 2907 2908 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 2909 unsigned long address) 2910 { 2911 spinlock_t *ptl; 2912 struct mmu_notifier_range range; 2913 2914 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 2915 address & HPAGE_PUD_MASK, 2916 (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE); 2917 mmu_notifier_invalidate_range_start(&range); 2918 ptl = pud_lock(vma->vm_mm, pud); 2919 if (unlikely(!pud_trans_huge(*pud))) 2920 goto out; 2921 __split_huge_pud_locked(vma, pud, range.start); 2922 2923 out: 2924 spin_unlock(ptl); 2925 mmu_notifier_invalidate_range_end(&range); 2926 } 2927 #else 2928 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 2929 unsigned long address) 2930 { 2931 } 2932 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 2933 2934 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 2935 unsigned long haddr, pmd_t *pmd) 2936 { 2937 struct mm_struct *mm = vma->vm_mm; 2938 pgtable_t pgtable; 2939 pmd_t _pmd, old_pmd; 2940 unsigned long addr; 2941 pte_t *pte; 2942 int i; 2943 2944 /* 2945 * Leave pmd empty until pte is filled note that it is fine to delay 2946 * notification until mmu_notifier_invalidate_range_end() as we are 2947 * replacing a zero pmd write protected page with a zero pte write 2948 * protected page. 2949 * 2950 * See Documentation/mm/mmu_notifier.rst 2951 */ 2952 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); 2953 2954 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2955 pmd_populate(mm, &_pmd, pgtable); 2956 2957 pte = pte_offset_map(&_pmd, haddr); 2958 VM_BUG_ON(!pte); 2959 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2960 pte_t entry; 2961 2962 entry = pfn_pte(my_zero_pfn(addr), vma->vm_page_prot); 2963 entry = pte_mkspecial(entry); 2964 if (pmd_uffd_wp(old_pmd)) 2965 entry = pte_mkuffd_wp(entry); 2966 VM_BUG_ON(!pte_none(ptep_get(pte))); 2967 set_pte_at(mm, addr, pte, entry); 2968 pte++; 2969 } 2970 pte_unmap(pte - 1); 2971 smp_wmb(); /* make pte visible before pmd */ 2972 pmd_populate(mm, pmd, pgtable); 2973 } 2974 2975 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 2976 unsigned long haddr, bool freeze) 2977 { 2978 struct mm_struct *mm = vma->vm_mm; 2979 struct folio *folio; 2980 struct page *page; 2981 pgtable_t pgtable; 2982 pmd_t old_pmd, _pmd; 2983 bool soft_dirty, uffd_wp = false, young = false, write = false; 2984 bool anon_exclusive = false, dirty = false; 2985 unsigned long addr; 2986 pte_t *pte; 2987 int i; 2988 2989 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 2990 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2991 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 2992 2993 VM_WARN_ON_ONCE(!pmd_is_valid_softleaf(*pmd) && !pmd_trans_huge(*pmd)); 2994 2995 count_vm_event(THP_SPLIT_PMD); 2996 2997 if (!vma_is_anonymous(vma)) { 2998 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); 2999 /* 3000 * We are going to unmap this huge page. So 3001 * just go ahead and zap it 3002 */ 3003 if (arch_needs_pgtable_deposit()) 3004 zap_deposited_table(mm, pmd); 3005 if (!vma_is_dax(vma) && vma_is_special_huge(vma)) 3006 return; 3007 if (unlikely(pmd_is_migration_entry(old_pmd))) { 3008 const softleaf_t old_entry = softleaf_from_pmd(old_pmd); 3009 3010 folio = softleaf_to_folio(old_entry); 3011 } else if (is_huge_zero_pmd(old_pmd)) { 3012 return; 3013 } else { 3014 page = pmd_page(old_pmd); 3015 folio = page_folio(page); 3016 if (!folio_test_dirty(folio) && pmd_dirty(old_pmd)) 3017 folio_mark_dirty(folio); 3018 if (!folio_test_referenced(folio) && pmd_young(old_pmd)) 3019 folio_set_referenced(folio); 3020 folio_remove_rmap_pmd(folio, page, vma); 3021 folio_put(folio); 3022 } 3023 add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR); 3024 return; 3025 } 3026 3027 if (is_huge_zero_pmd(*pmd)) { 3028 /* 3029 * FIXME: Do we want to invalidate secondary mmu by calling 3030 * mmu_notifier_arch_invalidate_secondary_tlbs() see comments below 3031 * inside __split_huge_pmd() ? 3032 * 3033 * We are going from a zero huge page write protected to zero 3034 * small page also write protected so it does not seems useful 3035 * to invalidate secondary mmu at this time. 3036 */ 3037 return __split_huge_zero_page_pmd(vma, haddr, pmd); 3038 } 3039 3040 if (pmd_is_migration_entry(*pmd)) { 3041 softleaf_t entry; 3042 3043 old_pmd = *pmd; 3044 entry = softleaf_from_pmd(old_pmd); 3045 page = softleaf_to_page(entry); 3046 folio = page_folio(page); 3047 3048 soft_dirty = pmd_swp_soft_dirty(old_pmd); 3049 uffd_wp = pmd_swp_uffd_wp(old_pmd); 3050 3051 write = softleaf_is_migration_write(entry); 3052 if (PageAnon(page)) 3053 anon_exclusive = softleaf_is_migration_read_exclusive(entry); 3054 young = softleaf_is_migration_young(entry); 3055 dirty = softleaf_is_migration_dirty(entry); 3056 } else if (pmd_is_device_private_entry(*pmd)) { 3057 softleaf_t entry; 3058 3059 old_pmd = *pmd; 3060 entry = softleaf_from_pmd(old_pmd); 3061 page = softleaf_to_page(entry); 3062 folio = page_folio(page); 3063 3064 soft_dirty = pmd_swp_soft_dirty(old_pmd); 3065 uffd_wp = pmd_swp_uffd_wp(old_pmd); 3066 3067 write = softleaf_is_device_private_write(entry); 3068 anon_exclusive = PageAnonExclusive(page); 3069 3070 /* 3071 * Device private THP should be treated the same as regular 3072 * folios w.r.t anon exclusive handling. See the comments for 3073 * folio handling and anon_exclusive below. 3074 */ 3075 if (freeze && anon_exclusive && 3076 folio_try_share_anon_rmap_pmd(folio, page)) 3077 freeze = false; 3078 if (!freeze) { 3079 rmap_t rmap_flags = RMAP_NONE; 3080 3081 folio_ref_add(folio, HPAGE_PMD_NR - 1); 3082 if (anon_exclusive) 3083 rmap_flags |= RMAP_EXCLUSIVE; 3084 3085 folio_add_anon_rmap_ptes(folio, page, HPAGE_PMD_NR, 3086 vma, haddr, rmap_flags); 3087 } 3088 } else { 3089 /* 3090 * Up to this point the pmd is present and huge and userland has 3091 * the whole access to the hugepage during the split (which 3092 * happens in place). If we overwrite the pmd with the not-huge 3093 * version pointing to the pte here (which of course we could if 3094 * all CPUs were bug free), userland could trigger a small page 3095 * size TLB miss on the small sized TLB while the hugepage TLB 3096 * entry is still established in the huge TLB. Some CPU doesn't 3097 * like that. See 3098 * http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum 3099 * 383 on page 105. Intel should be safe but is also warns that 3100 * it's only safe if the permission and cache attributes of the 3101 * two entries loaded in the two TLB is identical (which should 3102 * be the case here). But it is generally safer to never allow 3103 * small and huge TLB entries for the same virtual address to be 3104 * loaded simultaneously. So instead of doing "pmd_populate(); 3105 * flush_pmd_tlb_range();" we first mark the current pmd 3106 * notpresent (atomically because here the pmd_trans_huge must 3107 * remain set at all times on the pmd until the split is 3108 * complete for this pmd), then we flush the SMP TLB and finally 3109 * we write the non-huge version of the pmd entry with 3110 * pmd_populate. 3111 */ 3112 old_pmd = pmdp_invalidate(vma, haddr, pmd); 3113 page = pmd_page(old_pmd); 3114 folio = page_folio(page); 3115 if (pmd_dirty(old_pmd)) { 3116 dirty = true; 3117 folio_set_dirty(folio); 3118 } 3119 write = pmd_write(old_pmd); 3120 young = pmd_young(old_pmd); 3121 soft_dirty = pmd_soft_dirty(old_pmd); 3122 uffd_wp = pmd_uffd_wp(old_pmd); 3123 3124 VM_WARN_ON_FOLIO(!folio_ref_count(folio), folio); 3125 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); 3126 3127 /* 3128 * Without "freeze", we'll simply split the PMD, propagating the 3129 * PageAnonExclusive() flag for each PTE by setting it for 3130 * each subpage -- no need to (temporarily) clear. 3131 * 3132 * With "freeze" we want to replace mapped pages by 3133 * migration entries right away. This is only possible if we 3134 * managed to clear PageAnonExclusive() -- see 3135 * set_pmd_migration_entry(). 3136 * 3137 * In case we cannot clear PageAnonExclusive(), split the PMD 3138 * only and let try_to_migrate_one() fail later. 3139 * 3140 * See folio_try_share_anon_rmap_pmd(): invalidate PMD first. 3141 */ 3142 anon_exclusive = PageAnonExclusive(page); 3143 if (freeze && anon_exclusive && 3144 folio_try_share_anon_rmap_pmd(folio, page)) 3145 freeze = false; 3146 if (!freeze) { 3147 rmap_t rmap_flags = RMAP_NONE; 3148 3149 folio_ref_add(folio, HPAGE_PMD_NR - 1); 3150 if (anon_exclusive) 3151 rmap_flags |= RMAP_EXCLUSIVE; 3152 folio_add_anon_rmap_ptes(folio, page, HPAGE_PMD_NR, 3153 vma, haddr, rmap_flags); 3154 } 3155 } 3156 3157 /* 3158 * Withdraw the table only after we mark the pmd entry invalid. 3159 * This's critical for some architectures (Power). 3160 */ 3161 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 3162 pmd_populate(mm, &_pmd, pgtable); 3163 3164 pte = pte_offset_map(&_pmd, haddr); 3165 VM_BUG_ON(!pte); 3166 3167 /* 3168 * Note that NUMA hinting access restrictions are not transferred to 3169 * avoid any possibility of altering permissions across VMAs. 3170 */ 3171 if (freeze || pmd_is_migration_entry(old_pmd)) { 3172 pte_t entry; 3173 swp_entry_t swp_entry; 3174 3175 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 3176 if (write) 3177 swp_entry = make_writable_migration_entry( 3178 page_to_pfn(page + i)); 3179 else if (anon_exclusive) 3180 swp_entry = make_readable_exclusive_migration_entry( 3181 page_to_pfn(page + i)); 3182 else 3183 swp_entry = make_readable_migration_entry( 3184 page_to_pfn(page + i)); 3185 if (young) 3186 swp_entry = make_migration_entry_young(swp_entry); 3187 if (dirty) 3188 swp_entry = make_migration_entry_dirty(swp_entry); 3189 entry = swp_entry_to_pte(swp_entry); 3190 if (soft_dirty) 3191 entry = pte_swp_mksoft_dirty(entry); 3192 if (uffd_wp) 3193 entry = pte_swp_mkuffd_wp(entry); 3194 VM_WARN_ON(!pte_none(ptep_get(pte + i))); 3195 set_pte_at(mm, addr, pte + i, entry); 3196 } 3197 } else if (pmd_is_device_private_entry(old_pmd)) { 3198 pte_t entry; 3199 swp_entry_t swp_entry; 3200 3201 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 3202 /* 3203 * anon_exclusive was already propagated to the relevant 3204 * pages corresponding to the pte entries when freeze 3205 * is false. 3206 */ 3207 if (write) 3208 swp_entry = make_writable_device_private_entry( 3209 page_to_pfn(page + i)); 3210 else 3211 swp_entry = make_readable_device_private_entry( 3212 page_to_pfn(page + i)); 3213 /* 3214 * Young and dirty bits are not progated via swp_entry 3215 */ 3216 entry = swp_entry_to_pte(swp_entry); 3217 if (soft_dirty) 3218 entry = pte_swp_mksoft_dirty(entry); 3219 if (uffd_wp) 3220 entry = pte_swp_mkuffd_wp(entry); 3221 VM_WARN_ON(!pte_none(ptep_get(pte + i))); 3222 set_pte_at(mm, addr, pte + i, entry); 3223 } 3224 } else { 3225 pte_t entry; 3226 3227 entry = mk_pte(page, READ_ONCE(vma->vm_page_prot)); 3228 if (write) 3229 entry = pte_mkwrite(entry, vma); 3230 if (!young) 3231 entry = pte_mkold(entry); 3232 /* NOTE: this may set soft-dirty too on some archs */ 3233 if (dirty) 3234 entry = pte_mkdirty(entry); 3235 if (soft_dirty) 3236 entry = pte_mksoft_dirty(entry); 3237 if (uffd_wp) 3238 entry = pte_mkuffd_wp(entry); 3239 3240 for (i = 0; i < HPAGE_PMD_NR; i++) 3241 VM_WARN_ON(!pte_none(ptep_get(pte + i))); 3242 3243 set_ptes(mm, haddr, pte, entry, HPAGE_PMD_NR); 3244 } 3245 pte_unmap(pte); 3246 3247 if (!pmd_is_migration_entry(*pmd)) 3248 folio_remove_rmap_pmd(folio, page, vma); 3249 if (freeze) 3250 put_page(page); 3251 3252 smp_wmb(); /* make pte visible before pmd */ 3253 pmd_populate(mm, pmd, pgtable); 3254 } 3255 3256 void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, 3257 pmd_t *pmd, bool freeze) 3258 { 3259 VM_WARN_ON_ONCE(!IS_ALIGNED(address, HPAGE_PMD_SIZE)); 3260 if (pmd_trans_huge(*pmd) || pmd_is_valid_softleaf(*pmd)) 3261 __split_huge_pmd_locked(vma, pmd, address, freeze); 3262 } 3263 3264 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 3265 unsigned long address, bool freeze) 3266 { 3267 spinlock_t *ptl; 3268 struct mmu_notifier_range range; 3269 3270 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 3271 address & HPAGE_PMD_MASK, 3272 (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE); 3273 mmu_notifier_invalidate_range_start(&range); 3274 ptl = pmd_lock(vma->vm_mm, pmd); 3275 split_huge_pmd_locked(vma, range.start, pmd, freeze); 3276 spin_unlock(ptl); 3277 mmu_notifier_invalidate_range_end(&range); 3278 } 3279 3280 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, 3281 bool freeze) 3282 { 3283 pmd_t *pmd = mm_find_pmd(vma->vm_mm, address); 3284 3285 if (!pmd) 3286 return; 3287 3288 __split_huge_pmd(vma, pmd, address, freeze); 3289 } 3290 3291 static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address) 3292 { 3293 /* 3294 * If the new address isn't hpage aligned and it could previously 3295 * contain an hugepage: check if we need to split an huge pmd. 3296 */ 3297 if (!IS_ALIGNED(address, HPAGE_PMD_SIZE) && 3298 range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE), 3299 ALIGN(address, HPAGE_PMD_SIZE))) 3300 split_huge_pmd_address(vma, address, false); 3301 } 3302 3303 void vma_adjust_trans_huge(struct vm_area_struct *vma, 3304 unsigned long start, 3305 unsigned long end, 3306 struct vm_area_struct *next) 3307 { 3308 /* Check if we need to split start first. */ 3309 split_huge_pmd_if_needed(vma, start); 3310 3311 /* Check if we need to split end next. */ 3312 split_huge_pmd_if_needed(vma, end); 3313 3314 /* If we're incrementing next->vm_start, we might need to split it. */ 3315 if (next) 3316 split_huge_pmd_if_needed(next, end); 3317 } 3318 3319 static void unmap_folio(struct folio *folio) 3320 { 3321 enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SYNC | 3322 TTU_BATCH_FLUSH; 3323 3324 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 3325 3326 if (folio_test_pmd_mappable(folio)) 3327 ttu_flags |= TTU_SPLIT_HUGE_PMD; 3328 3329 /* 3330 * Anon pages need migration entries to preserve them, but file 3331 * pages can simply be left unmapped, then faulted back on demand. 3332 * If that is ever changed (perhaps for mlock), update remap_page(). 3333 */ 3334 if (folio_test_anon(folio)) 3335 try_to_migrate(folio, ttu_flags); 3336 else 3337 try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK); 3338 3339 try_to_unmap_flush(); 3340 } 3341 3342 static bool __discard_anon_folio_pmd_locked(struct vm_area_struct *vma, 3343 unsigned long addr, pmd_t *pmdp, 3344 struct folio *folio) 3345 { 3346 struct mm_struct *mm = vma->vm_mm; 3347 int ref_count, map_count; 3348 pmd_t orig_pmd = *pmdp; 3349 3350 if (pmd_dirty(orig_pmd)) 3351 folio_set_dirty(folio); 3352 if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { 3353 folio_set_swapbacked(folio); 3354 return false; 3355 } 3356 3357 orig_pmd = pmdp_huge_clear_flush(vma, addr, pmdp); 3358 3359 /* 3360 * Syncing against concurrent GUP-fast: 3361 * - clear PMD; barrier; read refcount 3362 * - inc refcount; barrier; read PMD 3363 */ 3364 smp_mb(); 3365 3366 ref_count = folio_ref_count(folio); 3367 map_count = folio_mapcount(folio); 3368 3369 /* 3370 * Order reads for folio refcount and dirty flag 3371 * (see comments in __remove_mapping()). 3372 */ 3373 smp_rmb(); 3374 3375 /* 3376 * If the folio or its PMD is redirtied at this point, or if there 3377 * are unexpected references, we will give up to discard this folio 3378 * and remap it. 3379 * 3380 * The only folio refs must be one from isolation plus the rmap(s). 3381 */ 3382 if (pmd_dirty(orig_pmd)) 3383 folio_set_dirty(folio); 3384 if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) { 3385 folio_set_swapbacked(folio); 3386 set_pmd_at(mm, addr, pmdp, orig_pmd); 3387 return false; 3388 } 3389 3390 if (ref_count != map_count + 1) { 3391 set_pmd_at(mm, addr, pmdp, orig_pmd); 3392 return false; 3393 } 3394 3395 folio_remove_rmap_pmd(folio, pmd_page(orig_pmd), vma); 3396 zap_deposited_table(mm, pmdp); 3397 add_mm_counter(mm, MM_ANONPAGES, -HPAGE_PMD_NR); 3398 if (vma->vm_flags & VM_LOCKED) 3399 mlock_drain_local(); 3400 folio_put(folio); 3401 3402 return true; 3403 } 3404 3405 bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr, 3406 pmd_t *pmdp, struct folio *folio) 3407 { 3408 VM_WARN_ON_FOLIO(!folio_test_pmd_mappable(folio), folio); 3409 VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); 3410 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); 3411 VM_WARN_ON_FOLIO(folio_test_swapbacked(folio), folio); 3412 VM_WARN_ON_ONCE(!IS_ALIGNED(addr, HPAGE_PMD_SIZE)); 3413 3414 return __discard_anon_folio_pmd_locked(vma, addr, pmdp, folio); 3415 } 3416 3417 static void remap_page(struct folio *folio, unsigned long nr, int flags) 3418 { 3419 int i = 0; 3420 3421 /* If unmap_folio() uses try_to_migrate() on file, remove this check */ 3422 if (!folio_test_anon(folio)) 3423 return; 3424 for (;;) { 3425 remove_migration_ptes(folio, folio, RMP_LOCKED | flags); 3426 i += folio_nr_pages(folio); 3427 if (i >= nr) 3428 break; 3429 folio = folio_next(folio); 3430 } 3431 } 3432 3433 static void lru_add_split_folio(struct folio *folio, struct folio *new_folio, 3434 struct lruvec *lruvec, struct list_head *list) 3435 { 3436 VM_BUG_ON_FOLIO(folio_test_lru(new_folio), folio); 3437 lockdep_assert_held(&lruvec->lru_lock); 3438 3439 if (folio_is_device_private(folio)) 3440 return; 3441 3442 if (list) { 3443 /* page reclaim is reclaiming a huge page */ 3444 VM_WARN_ON(folio_test_lru(folio)); 3445 folio_get(new_folio); 3446 list_add_tail(&new_folio->lru, list); 3447 } else { 3448 /* head is still on lru (and we have it frozen) */ 3449 VM_WARN_ON(!folio_test_lru(folio)); 3450 if (folio_test_unevictable(folio)) 3451 new_folio->mlock_count = 0; 3452 else 3453 list_add_tail(&new_folio->lru, &folio->lru); 3454 folio_set_lru(new_folio); 3455 } 3456 } 3457 3458 static bool page_range_has_hwpoisoned(struct page *page, long nr_pages) 3459 { 3460 for (; nr_pages; page++, nr_pages--) 3461 if (PageHWPoison(page)) 3462 return true; 3463 return false; 3464 } 3465 3466 /* 3467 * It splits @folio into @new_order folios and copies the @folio metadata to 3468 * all the resulting folios. 3469 */ 3470 static void __split_folio_to_order(struct folio *folio, int old_order, 3471 int new_order) 3472 { 3473 /* Scan poisoned pages when split a poisoned folio to large folios */ 3474 const bool handle_hwpoison = folio_test_has_hwpoisoned(folio) && new_order; 3475 long new_nr_pages = 1 << new_order; 3476 long nr_pages = 1 << old_order; 3477 long i; 3478 3479 folio_clear_has_hwpoisoned(folio); 3480 3481 /* Check first new_nr_pages since the loop below skips them */ 3482 if (handle_hwpoison && 3483 page_range_has_hwpoisoned(folio_page(folio, 0), new_nr_pages)) 3484 folio_set_has_hwpoisoned(folio); 3485 /* 3486 * Skip the first new_nr_pages, since the new folio from them have all 3487 * the flags from the original folio. 3488 */ 3489 for (i = new_nr_pages; i < nr_pages; i += new_nr_pages) { 3490 struct page *new_head = &folio->page + i; 3491 /* 3492 * Careful: new_folio is not a "real" folio before we cleared PageTail. 3493 * Don't pass it around before clear_compound_head(). 3494 */ 3495 struct folio *new_folio = (struct folio *)new_head; 3496 3497 VM_BUG_ON_PAGE(atomic_read(&new_folio->_mapcount) != -1, new_head); 3498 3499 /* 3500 * Clone page flags before unfreezing refcount. 3501 * 3502 * After successful get_page_unless_zero() might follow flags change, 3503 * for example lock_page() which set PG_waiters. 3504 * 3505 * Note that for mapped sub-pages of an anonymous THP, 3506 * PG_anon_exclusive has been cleared in unmap_folio() and is stored in 3507 * the migration entry instead from where remap_page() will restore it. 3508 * We can still have PG_anon_exclusive set on effectively unmapped and 3509 * unreferenced sub-pages of an anonymous THP: we can simply drop 3510 * PG_anon_exclusive (-> PG_mappedtodisk) for these here. 3511 */ 3512 new_folio->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP; 3513 new_folio->flags.f |= (folio->flags.f & 3514 ((1L << PG_referenced) | 3515 (1L << PG_swapbacked) | 3516 (1L << PG_swapcache) | 3517 (1L << PG_mlocked) | 3518 (1L << PG_uptodate) | 3519 (1L << PG_active) | 3520 (1L << PG_workingset) | 3521 (1L << PG_locked) | 3522 (1L << PG_unevictable) | 3523 #ifdef CONFIG_ARCH_USES_PG_ARCH_2 3524 (1L << PG_arch_2) | 3525 #endif 3526 #ifdef CONFIG_ARCH_USES_PG_ARCH_3 3527 (1L << PG_arch_3) | 3528 #endif 3529 (1L << PG_dirty) | 3530 LRU_GEN_MASK | LRU_REFS_MASK)); 3531 3532 if (handle_hwpoison && 3533 page_range_has_hwpoisoned(new_head, new_nr_pages)) 3534 folio_set_has_hwpoisoned(new_folio); 3535 3536 new_folio->mapping = folio->mapping; 3537 new_folio->index = folio->index + i; 3538 3539 if (folio_test_swapcache(folio)) 3540 new_folio->swap.val = folio->swap.val + i; 3541 3542 /* Page flags must be visible before we make the page non-compound. */ 3543 smp_wmb(); 3544 3545 /* 3546 * Clear PageTail before unfreezing page refcount. 3547 * 3548 * After successful get_page_unless_zero() might follow put_page() 3549 * which needs correct compound_head(). 3550 */ 3551 clear_compound_head(new_head); 3552 if (new_order) { 3553 prep_compound_page(new_head, new_order); 3554 folio_set_large_rmappable(new_folio); 3555 } 3556 3557 if (folio_test_young(folio)) 3558 folio_set_young(new_folio); 3559 if (folio_test_idle(folio)) 3560 folio_set_idle(new_folio); 3561 #ifdef CONFIG_MEMCG 3562 new_folio->memcg_data = folio->memcg_data; 3563 #endif 3564 3565 folio_xchg_last_cpupid(new_folio, folio_last_cpupid(folio)); 3566 } 3567 3568 if (new_order) 3569 folio_set_order(folio, new_order); 3570 else 3571 ClearPageCompound(&folio->page); 3572 } 3573 3574 /** 3575 * __split_unmapped_folio() - splits an unmapped @folio to lower order folios in 3576 * two ways: uniform split or non-uniform split. 3577 * @folio: the to-be-split folio 3578 * @new_order: the smallest order of the after split folios (since buddy 3579 * allocator like split generates folios with orders from @folio's 3580 * order - 1 to new_order). 3581 * @split_at: in buddy allocator like split, the folio containing @split_at 3582 * will be split until its order becomes @new_order. 3583 * @xas: xa_state pointing to folio->mapping->i_pages and locked by caller 3584 * @mapping: @folio->mapping 3585 * @split_type: if the split is uniform or not (buddy allocator like split) 3586 * 3587 * 3588 * 1. uniform split: the given @folio into multiple @new_order small folios, 3589 * where all small folios have the same order. This is done when 3590 * split_type is SPLIT_TYPE_UNIFORM. 3591 * 2. buddy allocator like (non-uniform) split: the given @folio is split into 3592 * half and one of the half (containing the given page) is split into half 3593 * until the given @folio's order becomes @new_order. This is done when 3594 * split_type is SPLIT_TYPE_NON_UNIFORM. 3595 * 3596 * The high level flow for these two methods are: 3597 * 3598 * 1. uniform split: @xas is split with no expectation of failure and a single 3599 * __split_folio_to_order() is called to split the @folio into @new_order 3600 * along with stats update. 3601 * 2. non-uniform split: folio_order - @new_order calls to 3602 * __split_folio_to_order() are expected to be made in a for loop to split 3603 * the @folio to one lower order at a time. The folio containing @split_at 3604 * is split in each iteration. @xas is split into half in each iteration and 3605 * can fail. A failed @xas split leaves split folios as is without merging 3606 * them back. 3607 * 3608 * After splitting, the caller's folio reference will be transferred to the 3609 * folio containing @split_at. The caller needs to unlock and/or free 3610 * after-split folios if necessary. 3611 * 3612 * Return: 0 - successful, <0 - failed (if -ENOMEM is returned, @folio might be 3613 * split but not to @new_order, the caller needs to check) 3614 */ 3615 static int __split_unmapped_folio(struct folio *folio, int new_order, 3616 struct page *split_at, struct xa_state *xas, 3617 struct address_space *mapping, enum split_type split_type) 3618 { 3619 const bool is_anon = folio_test_anon(folio); 3620 int old_order = folio_order(folio); 3621 int start_order = split_type == SPLIT_TYPE_UNIFORM ? new_order : old_order - 1; 3622 int split_order; 3623 3624 /* 3625 * split to new_order one order at a time. For uniform split, 3626 * folio is split to new_order directly. 3627 */ 3628 for (split_order = start_order; 3629 split_order >= new_order; 3630 split_order--) { 3631 int nr_new_folios = 1UL << (old_order - split_order); 3632 3633 /* order-1 anonymous folio is not supported */ 3634 if (is_anon && split_order == 1) 3635 continue; 3636 3637 if (mapping) { 3638 /* 3639 * uniform split has xas_split_alloc() called before 3640 * irq is disabled to allocate enough memory, whereas 3641 * non-uniform split can handle ENOMEM. 3642 */ 3643 if (split_type == SPLIT_TYPE_UNIFORM) 3644 xas_split(xas, folio, old_order); 3645 else { 3646 xas_set_order(xas, folio->index, split_order); 3647 xas_try_split(xas, folio, old_order); 3648 if (xas_error(xas)) 3649 return xas_error(xas); 3650 } 3651 } 3652 3653 folio_split_memcg_refs(folio, old_order, split_order); 3654 split_page_owner(&folio->page, old_order, split_order); 3655 pgalloc_tag_split(folio, old_order, split_order); 3656 __split_folio_to_order(folio, old_order, split_order); 3657 3658 if (is_anon) { 3659 mod_mthp_stat(old_order, MTHP_STAT_NR_ANON, -1); 3660 mod_mthp_stat(split_order, MTHP_STAT_NR_ANON, nr_new_folios); 3661 } 3662 /* 3663 * If uniform split, the process is complete. 3664 * If non-uniform, continue splitting the folio at @split_at 3665 * as long as the next @split_order is >= @new_order. 3666 */ 3667 folio = page_folio(split_at); 3668 old_order = split_order; 3669 } 3670 3671 return 0; 3672 } 3673 3674 /** 3675 * folio_check_splittable() - check if a folio can be split to a given order 3676 * @folio: folio to be split 3677 * @new_order: the smallest order of the after split folios (since buddy 3678 * allocator like split generates folios with orders from @folio's 3679 * order - 1 to new_order). 3680 * @split_type: uniform or non-uniform split 3681 * 3682 * folio_check_splittable() checks if @folio can be split to @new_order using 3683 * @split_type method. The truncated folio check must come first. 3684 * 3685 * Context: folio must be locked. 3686 * 3687 * Return: 0 - @folio can be split to @new_order, otherwise an error number is 3688 * returned. 3689 */ 3690 int folio_check_splittable(struct folio *folio, unsigned int new_order, 3691 enum split_type split_type) 3692 { 3693 VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); 3694 /* 3695 * Folios that just got truncated cannot get split. Signal to the 3696 * caller that there was a race. 3697 * 3698 * TODO: this will also currently refuse folios without a mapping in the 3699 * swapcache (shmem or to-be-anon folios). 3700 */ 3701 if (!folio->mapping && !folio_test_anon(folio)) 3702 return -EBUSY; 3703 3704 if (folio_test_anon(folio)) { 3705 /* order-1 is not supported for anonymous THP. */ 3706 if (new_order == 1) 3707 return -EINVAL; 3708 } else if (split_type == SPLIT_TYPE_NON_UNIFORM || new_order) { 3709 if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && 3710 !mapping_large_folio_support(folio->mapping)) { 3711 /* 3712 * We can always split a folio down to a single page 3713 * (new_order == 0) uniformly. 3714 * 3715 * For any other scenario 3716 * a) uniform split targeting a large folio 3717 * (new_order > 0) 3718 * b) any non-uniform split 3719 * we must confirm that the file system supports large 3720 * folios. 3721 * 3722 * Note that we might still have THPs in such 3723 * mappings, which is created from khugepaged when 3724 * CONFIG_READ_ONLY_THP_FOR_FS is enabled. But in that 3725 * case, the mapping does not actually support large 3726 * folios properly. 3727 */ 3728 return -EINVAL; 3729 } 3730 } 3731 3732 /* 3733 * swapcache folio could only be split to order 0 3734 * 3735 * non-uniform split creates after-split folios with orders from 3736 * folio_order(folio) - 1 to new_order, making it not suitable for any 3737 * swapcache folio split. Only uniform split to order-0 can be used 3738 * here. 3739 */ 3740 if ((split_type == SPLIT_TYPE_NON_UNIFORM || new_order) && folio_test_swapcache(folio)) { 3741 return -EINVAL; 3742 } 3743 3744 if (is_huge_zero_folio(folio)) 3745 return -EINVAL; 3746 3747 if (folio_test_writeback(folio)) 3748 return -EBUSY; 3749 3750 return 0; 3751 } 3752 3753 /* Number of folio references from the pagecache or the swapcache. */ 3754 static unsigned int folio_cache_ref_count(const struct folio *folio) 3755 { 3756 if (folio_test_anon(folio) && !folio_test_swapcache(folio)) 3757 return 0; 3758 return folio_nr_pages(folio); 3759 } 3760 3761 static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int new_order, 3762 struct page *split_at, struct xa_state *xas, 3763 struct address_space *mapping, bool do_lru, 3764 struct list_head *list, enum split_type split_type, 3765 pgoff_t end, int *nr_shmem_dropped) 3766 { 3767 struct folio *end_folio = folio_next(folio); 3768 struct folio *new_folio, *next; 3769 int old_order = folio_order(folio); 3770 int ret = 0; 3771 struct deferred_split *ds_queue; 3772 3773 VM_WARN_ON_ONCE(!mapping && end); 3774 /* Prevent deferred_split_scan() touching ->_refcount */ 3775 ds_queue = folio_split_queue_lock(folio); 3776 if (folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) { 3777 struct swap_cluster_info *ci = NULL; 3778 struct lruvec *lruvec; 3779 3780 if (old_order > 1) { 3781 if (!list_empty(&folio->_deferred_list)) { 3782 ds_queue->split_queue_len--; 3783 /* 3784 * Reinitialize page_deferred_list after removing the 3785 * page from the split_queue, otherwise a subsequent 3786 * split will see list corruption when checking the 3787 * page_deferred_list. 3788 */ 3789 list_del_init(&folio->_deferred_list); 3790 } 3791 if (folio_test_partially_mapped(folio)) { 3792 folio_clear_partially_mapped(folio); 3793 mod_mthp_stat(old_order, 3794 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 3795 } 3796 } 3797 split_queue_unlock(ds_queue); 3798 if (mapping) { 3799 int nr = folio_nr_pages(folio); 3800 3801 if (folio_test_pmd_mappable(folio) && 3802 new_order < HPAGE_PMD_ORDER) { 3803 if (folio_test_swapbacked(folio)) { 3804 lruvec_stat_mod_folio(folio, 3805 NR_SHMEM_THPS, -nr); 3806 } else { 3807 lruvec_stat_mod_folio(folio, 3808 NR_FILE_THPS, -nr); 3809 filemap_nr_thps_dec(mapping); 3810 } 3811 } 3812 } 3813 3814 if (folio_test_swapcache(folio)) { 3815 if (mapping) { 3816 VM_WARN_ON_ONCE_FOLIO(mapping, folio); 3817 return -EINVAL; 3818 } 3819 3820 ci = swap_cluster_get_and_lock(folio); 3821 } 3822 3823 /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ 3824 if (do_lru) 3825 lruvec = folio_lruvec_lock(folio); 3826 3827 ret = __split_unmapped_folio(folio, new_order, split_at, xas, 3828 mapping, split_type); 3829 3830 /* 3831 * Unfreeze after-split folios and put them back to the right 3832 * list. @folio should be kept frozon until page cache 3833 * entries are updated with all the other after-split folios 3834 * to prevent others seeing stale page cache entries. 3835 * As a result, new_folio starts from the next folio of 3836 * @folio. 3837 */ 3838 for (new_folio = folio_next(folio); new_folio != end_folio; 3839 new_folio = next) { 3840 unsigned long nr_pages = folio_nr_pages(new_folio); 3841 3842 next = folio_next(new_folio); 3843 3844 zone_device_private_split_cb(folio, new_folio); 3845 3846 folio_ref_unfreeze(new_folio, 3847 folio_cache_ref_count(new_folio) + 1); 3848 3849 if (do_lru) 3850 lru_add_split_folio(folio, new_folio, lruvec, list); 3851 3852 /* 3853 * Anonymous folio with swap cache. 3854 * NOTE: shmem in swap cache is not supported yet. 3855 */ 3856 if (ci) { 3857 __swap_cache_replace_folio(ci, folio, new_folio); 3858 continue; 3859 } 3860 3861 /* Anonymous folio without swap cache */ 3862 if (!mapping) 3863 continue; 3864 3865 /* Add the new folio to the page cache. */ 3866 if (new_folio->index < end) { 3867 __xa_store(&mapping->i_pages, new_folio->index, 3868 new_folio, 0); 3869 continue; 3870 } 3871 3872 VM_WARN_ON_ONCE(!nr_shmem_dropped); 3873 /* Drop folio beyond EOF: ->index >= end */ 3874 if (shmem_mapping(mapping) && nr_shmem_dropped) 3875 *nr_shmem_dropped += nr_pages; 3876 else if (folio_test_clear_dirty(new_folio)) 3877 folio_account_cleaned( 3878 new_folio, inode_to_wb(mapping->host)); 3879 __filemap_remove_folio(new_folio, NULL); 3880 folio_put_refs(new_folio, nr_pages); 3881 } 3882 3883 zone_device_private_split_cb(folio, NULL); 3884 /* 3885 * Unfreeze @folio only after all page cache entries, which 3886 * used to point to it, have been updated with new folios. 3887 * Otherwise, a parallel folio_try_get() can grab @folio 3888 * and its caller can see stale page cache entries. 3889 */ 3890 folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1); 3891 3892 if (do_lru) 3893 unlock_page_lruvec(lruvec); 3894 3895 if (ci) 3896 swap_cluster_unlock(ci); 3897 } else { 3898 split_queue_unlock(ds_queue); 3899 return -EAGAIN; 3900 } 3901 3902 return ret; 3903 } 3904 3905 /** 3906 * __folio_split() - split a folio at @split_at to a @new_order folio 3907 * @folio: folio to split 3908 * @new_order: the order of the new folio 3909 * @split_at: a page within the new folio 3910 * @lock_at: a page within @folio to be left locked to caller 3911 * @list: after-split folios will be put on it if non NULL 3912 * @split_type: perform uniform split or not (non-uniform split) 3913 * 3914 * It calls __split_unmapped_folio() to perform uniform and non-uniform split. 3915 * It is in charge of checking whether the split is supported or not and 3916 * preparing @folio for __split_unmapped_folio(). 3917 * 3918 * After splitting, the after-split folio containing @lock_at remains locked 3919 * and others are unlocked: 3920 * 1. for uniform split, @lock_at points to one of @folio's subpages; 3921 * 2. for buddy allocator like (non-uniform) split, @lock_at points to @folio. 3922 * 3923 * Return: 0 - successful, <0 - failed (if -ENOMEM is returned, @folio might be 3924 * split but not to @new_order, the caller needs to check) 3925 */ 3926 static int __folio_split(struct folio *folio, unsigned int new_order, 3927 struct page *split_at, struct page *lock_at, 3928 struct list_head *list, enum split_type split_type) 3929 { 3930 XA_STATE(xas, &folio->mapping->i_pages, folio->index); 3931 struct folio *end_folio = folio_next(folio); 3932 bool is_anon = folio_test_anon(folio); 3933 struct address_space *mapping = NULL; 3934 struct anon_vma *anon_vma = NULL; 3935 int old_order = folio_order(folio); 3936 struct folio *new_folio, *next; 3937 int nr_shmem_dropped = 0; 3938 int remap_flags = 0; 3939 int ret; 3940 pgoff_t end = 0; 3941 3942 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); 3943 VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio); 3944 3945 if (folio != page_folio(split_at) || folio != page_folio(lock_at)) { 3946 ret = -EINVAL; 3947 goto out; 3948 } 3949 3950 if (new_order >= old_order) { 3951 ret = -EINVAL; 3952 goto out; 3953 } 3954 3955 ret = folio_check_splittable(folio, new_order, split_type); 3956 if (ret) { 3957 VM_WARN_ONCE(ret == -EINVAL, "Tried to split an unsplittable folio"); 3958 goto out; 3959 } 3960 3961 if (is_anon) { 3962 /* 3963 * The caller does not necessarily hold an mmap_lock that would 3964 * prevent the anon_vma disappearing so we first we take a 3965 * reference to it and then lock the anon_vma for write. This 3966 * is similar to folio_lock_anon_vma_read except the write lock 3967 * is taken to serialise against parallel split or collapse 3968 * operations. 3969 */ 3970 anon_vma = folio_get_anon_vma(folio); 3971 if (!anon_vma) { 3972 ret = -EBUSY; 3973 goto out; 3974 } 3975 anon_vma_lock_write(anon_vma); 3976 mapping = NULL; 3977 } else { 3978 unsigned int min_order; 3979 gfp_t gfp; 3980 3981 mapping = folio->mapping; 3982 min_order = mapping_min_folio_order(folio->mapping); 3983 if (new_order < min_order) { 3984 ret = -EINVAL; 3985 goto out; 3986 } 3987 3988 gfp = current_gfp_context(mapping_gfp_mask(mapping) & 3989 GFP_RECLAIM_MASK); 3990 3991 if (!filemap_release_folio(folio, gfp)) { 3992 ret = -EBUSY; 3993 goto out; 3994 } 3995 3996 if (split_type == SPLIT_TYPE_UNIFORM) { 3997 xas_set_order(&xas, folio->index, new_order); 3998 xas_split_alloc(&xas, folio, old_order, gfp); 3999 if (xas_error(&xas)) { 4000 ret = xas_error(&xas); 4001 goto out; 4002 } 4003 } 4004 4005 anon_vma = NULL; 4006 i_mmap_lock_read(mapping); 4007 4008 /* 4009 *__split_unmapped_folio() may need to trim off pages beyond 4010 * EOF: but on 32-bit, i_size_read() takes an irq-unsafe 4011 * seqlock, which cannot be nested inside the page tree lock. 4012 * So note end now: i_size itself may be changed at any moment, 4013 * but folio lock is good enough to serialize the trimming. 4014 */ 4015 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); 4016 if (shmem_mapping(mapping)) 4017 end = shmem_fallocend(mapping->host, end); 4018 } 4019 4020 /* 4021 * Racy check if we can split the page, before unmap_folio() will 4022 * split PMDs 4023 */ 4024 if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) { 4025 ret = -EAGAIN; 4026 goto out_unlock; 4027 } 4028 4029 unmap_folio(folio); 4030 4031 /* block interrupt reentry in xa_lock and spinlock */ 4032 local_irq_disable(); 4033 if (mapping) { 4034 /* 4035 * Check if the folio is present in page cache. 4036 * We assume all tail are present too, if folio is there. 4037 */ 4038 xas_lock(&xas); 4039 xas_reset(&xas); 4040 if (xas_load(&xas) != folio) { 4041 ret = -EAGAIN; 4042 goto fail; 4043 } 4044 } 4045 4046 ret = __folio_freeze_and_split_unmapped(folio, new_order, split_at, &xas, mapping, 4047 true, list, split_type, end, &nr_shmem_dropped); 4048 fail: 4049 if (mapping) 4050 xas_unlock(&xas); 4051 4052 local_irq_enable(); 4053 4054 if (nr_shmem_dropped) 4055 shmem_uncharge(mapping->host, nr_shmem_dropped); 4056 4057 if (!ret && is_anon && !folio_is_device_private(folio)) 4058 remap_flags = RMP_USE_SHARED_ZEROPAGE; 4059 4060 remap_page(folio, 1 << old_order, remap_flags); 4061 4062 /* 4063 * Unlock all after-split folios except the one containing 4064 * @lock_at page. If @folio is not split, it will be kept locked. 4065 */ 4066 for (new_folio = folio; new_folio != end_folio; new_folio = next) { 4067 next = folio_next(new_folio); 4068 if (new_folio == page_folio(lock_at)) 4069 continue; 4070 4071 folio_unlock(new_folio); 4072 /* 4073 * Subpages may be freed if there wasn't any mapping 4074 * like if add_to_swap() is running on a lru page that 4075 * had its mapping zapped. And freeing these pages 4076 * requires taking the lru_lock so we do the put_page 4077 * of the tail pages after the split is complete. 4078 */ 4079 free_folio_and_swap_cache(new_folio); 4080 } 4081 4082 out_unlock: 4083 if (anon_vma) { 4084 anon_vma_unlock_write(anon_vma); 4085 put_anon_vma(anon_vma); 4086 } 4087 if (mapping) 4088 i_mmap_unlock_read(mapping); 4089 out: 4090 xas_destroy(&xas); 4091 if (old_order == HPAGE_PMD_ORDER) 4092 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); 4093 count_mthp_stat(old_order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED); 4094 return ret; 4095 } 4096 4097 /** 4098 * folio_split_unmapped() - split a large anon folio that is already unmapped 4099 * @folio: folio to split 4100 * @new_order: the order of folios after split 4101 * 4102 * This function is a helper for splitting folios that have already been 4103 * unmapped. The use case is that the device or the CPU can refuse to migrate 4104 * THP pages in the middle of migration, due to allocation issues on either 4105 * side. 4106 * 4107 * anon_vma_lock is not required to be held, mmap_read_lock() or 4108 * mmap_write_lock() should be held. @folio is expected to be locked by the 4109 * caller. device-private and non device-private folios are supported along 4110 * with folios that are in the swapcache. @folio should also be unmapped and 4111 * isolated from LRU (if applicable) 4112 * 4113 * Upon return, the folio is not remapped, split folios are not added to LRU, 4114 * free_folio_and_swap_cache() is not called, and new folios remain locked. 4115 * 4116 * Return: 0 on success, -EAGAIN if the folio cannot be split (e.g., due to 4117 * insufficient reference count or extra pins). 4118 */ 4119 int folio_split_unmapped(struct folio *folio, unsigned int new_order) 4120 { 4121 int ret = 0; 4122 4123 VM_WARN_ON_ONCE_FOLIO(folio_mapped(folio), folio); 4124 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); 4125 VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio); 4126 VM_WARN_ON_ONCE_FOLIO(!folio_test_anon(folio), folio); 4127 4128 if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) 4129 return -EAGAIN; 4130 4131 local_irq_disable(); 4132 ret = __folio_freeze_and_split_unmapped(folio, new_order, &folio->page, NULL, 4133 NULL, false, NULL, SPLIT_TYPE_UNIFORM, 4134 0, NULL); 4135 local_irq_enable(); 4136 return ret; 4137 } 4138 4139 /* 4140 * This function splits a large folio into smaller folios of order @new_order. 4141 * @page can point to any page of the large folio to split. The split operation 4142 * does not change the position of @page. 4143 * 4144 * Prerequisites: 4145 * 4146 * 1) The caller must hold a reference on the @page's owning folio, also known 4147 * as the large folio. 4148 * 4149 * 2) The large folio must be locked. 4150 * 4151 * 3) The folio must not be pinned. Any unexpected folio references, including 4152 * GUP pins, will result in the folio not getting split; instead, the caller 4153 * will receive an -EAGAIN. 4154 * 4155 * 4) @new_order > 1, usually. Splitting to order-1 anonymous folios is not 4156 * supported for non-file-backed folios, because folio->_deferred_list, which 4157 * is used by partially mapped folios, is stored in subpage 2, but an order-1 4158 * folio only has subpages 0 and 1. File-backed order-1 folios are supported, 4159 * since they do not use _deferred_list. 4160 * 4161 * After splitting, the caller's folio reference will be transferred to @page, 4162 * resulting in a raised refcount of @page after this call. The other pages may 4163 * be freed if they are not mapped. 4164 * 4165 * If @list is null, tail pages will be added to LRU list, otherwise, to @list. 4166 * 4167 * Pages in @new_order will inherit the mapping, flags, and so on from the 4168 * huge page. 4169 * 4170 * Returns 0 if the huge page was split successfully. 4171 * 4172 * Returns -EAGAIN if the folio has unexpected reference (e.g., GUP) or if 4173 * the folio was concurrently removed from the page cache. 4174 * 4175 * Returns -EBUSY when trying to split the huge zeropage, if the folio is 4176 * under writeback, if fs-specific folio metadata cannot currently be 4177 * released, or if some unexpected race happened (e.g., anon VMA disappeared, 4178 * truncation). 4179 * 4180 * Callers should ensure that the order respects the address space mapping 4181 * min-order if one is set for non-anonymous folios. 4182 * 4183 * Returns -EINVAL when trying to split to an order that is incompatible 4184 * with the folio. Splitting to order 0 is compatible with all folios. 4185 */ 4186 int __split_huge_page_to_list_to_order(struct page *page, struct list_head *list, 4187 unsigned int new_order) 4188 { 4189 struct folio *folio = page_folio(page); 4190 4191 return __folio_split(folio, new_order, &folio->page, page, list, 4192 SPLIT_TYPE_UNIFORM); 4193 } 4194 4195 /** 4196 * folio_split() - split a folio at @split_at to a @new_order folio 4197 * @folio: folio to split 4198 * @new_order: the order of the new folio 4199 * @split_at: a page within the new folio 4200 * @list: after-split folios are added to @list if not null, otherwise to LRU 4201 * list 4202 * 4203 * It has the same prerequisites and returns as 4204 * split_huge_page_to_list_to_order(). 4205 * 4206 * Split a folio at @split_at to a new_order folio, leave the 4207 * remaining subpages of the original folio as large as possible. For example, 4208 * in the case of splitting an order-9 folio at its third order-3 subpages to 4209 * an order-3 folio, there are 2^(9-3)=64 order-3 subpages in the order-9 folio. 4210 * After the split, there will be a group of folios with different orders and 4211 * the new folio containing @split_at is marked in bracket: 4212 * [order-4, {order-3}, order-3, order-5, order-6, order-7, order-8]. 4213 * 4214 * After split, folio is left locked for caller. 4215 * 4216 * Return: 0 - successful, <0 - failed (if -ENOMEM is returned, @folio might be 4217 * split but not to @new_order, the caller needs to check) 4218 */ 4219 int folio_split(struct folio *folio, unsigned int new_order, 4220 struct page *split_at, struct list_head *list) 4221 { 4222 return __folio_split(folio, new_order, split_at, &folio->page, list, 4223 SPLIT_TYPE_NON_UNIFORM); 4224 } 4225 4226 /** 4227 * min_order_for_split() - get the minimum order @folio can be split to 4228 * @folio: folio to split 4229 * 4230 * min_order_for_split() tells the minimum order @folio can be split to. 4231 * If a file-backed folio is truncated, 0 will be returned. Any subsequent 4232 * split attempt should get -EBUSY from split checking code. 4233 * 4234 * Return: @folio's minimum order for split 4235 */ 4236 unsigned int min_order_for_split(struct folio *folio) 4237 { 4238 if (folio_test_anon(folio)) 4239 return 0; 4240 4241 /* 4242 * If the folio got truncated, we don't know the previous mapping and 4243 * consequently the old min order. But it doesn't matter, as any split 4244 * attempt will immediately fail with -EBUSY as the folio cannot get 4245 * split until freed. 4246 */ 4247 if (!folio->mapping) 4248 return 0; 4249 4250 return mapping_min_folio_order(folio->mapping); 4251 } 4252 4253 int split_folio_to_list(struct folio *folio, struct list_head *list) 4254 { 4255 return split_huge_page_to_list_to_order(&folio->page, list, 0); 4256 } 4257 4258 /* 4259 * __folio_unqueue_deferred_split() is not to be called directly: 4260 * the folio_unqueue_deferred_split() inline wrapper in mm/internal.h 4261 * limits its calls to those folios which may have a _deferred_list for 4262 * queueing THP splits, and that list is (racily observed to be) non-empty. 4263 * 4264 * It is unsafe to call folio_unqueue_deferred_split() until folio refcount is 4265 * zero: because even when split_queue_lock is held, a non-empty _deferred_list 4266 * might be in use on deferred_split_scan()'s unlocked on-stack list. 4267 * 4268 * If memory cgroups are enabled, split_queue_lock is in the mem_cgroup: it is 4269 * therefore important to unqueue deferred split before changing folio memcg. 4270 */ 4271 bool __folio_unqueue_deferred_split(struct folio *folio) 4272 { 4273 struct deferred_split *ds_queue; 4274 unsigned long flags; 4275 bool unqueued = false; 4276 4277 WARN_ON_ONCE(folio_ref_count(folio)); 4278 WARN_ON_ONCE(!mem_cgroup_disabled() && !folio_memcg_charged(folio)); 4279 4280 ds_queue = folio_split_queue_lock_irqsave(folio, &flags); 4281 if (!list_empty(&folio->_deferred_list)) { 4282 ds_queue->split_queue_len--; 4283 if (folio_test_partially_mapped(folio)) { 4284 folio_clear_partially_mapped(folio); 4285 mod_mthp_stat(folio_order(folio), 4286 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 4287 } 4288 list_del_init(&folio->_deferred_list); 4289 unqueued = true; 4290 } 4291 split_queue_unlock_irqrestore(ds_queue, flags); 4292 4293 return unqueued; /* useful for debug warnings */ 4294 } 4295 4296 /* partially_mapped=false won't clear PG_partially_mapped folio flag */ 4297 void deferred_split_folio(struct folio *folio, bool partially_mapped) 4298 { 4299 struct deferred_split *ds_queue; 4300 unsigned long flags; 4301 4302 /* 4303 * Order 1 folios have no space for a deferred list, but we also 4304 * won't waste much memory by not adding them to the deferred list. 4305 */ 4306 if (folio_order(folio) <= 1) 4307 return; 4308 4309 if (!partially_mapped && !split_underused_thp) 4310 return; 4311 4312 /* 4313 * Exclude swapcache: originally to avoid a corrupt deferred split 4314 * queue. Nowadays that is fully prevented by memcg1_swapout(); 4315 * but if page reclaim is already handling the same folio, it is 4316 * unnecessary to handle it again in the shrinker, so excluding 4317 * swapcache here may still be a useful optimization. 4318 */ 4319 if (folio_test_swapcache(folio)) 4320 return; 4321 4322 ds_queue = folio_split_queue_lock_irqsave(folio, &flags); 4323 if (partially_mapped) { 4324 if (!folio_test_partially_mapped(folio)) { 4325 folio_set_partially_mapped(folio); 4326 if (folio_test_pmd_mappable(folio)) 4327 count_vm_event(THP_DEFERRED_SPLIT_PAGE); 4328 count_mthp_stat(folio_order(folio), MTHP_STAT_SPLIT_DEFERRED); 4329 mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, 1); 4330 4331 } 4332 } else { 4333 /* partially mapped folios cannot become non-partially mapped */ 4334 VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio); 4335 } 4336 if (list_empty(&folio->_deferred_list)) { 4337 struct mem_cgroup *memcg; 4338 4339 memcg = folio_split_queue_memcg(folio, ds_queue); 4340 list_add_tail(&folio->_deferred_list, &ds_queue->split_queue); 4341 ds_queue->split_queue_len++; 4342 if (memcg) 4343 set_shrinker_bit(memcg, folio_nid(folio), 4344 shrinker_id(deferred_split_shrinker)); 4345 } 4346 split_queue_unlock_irqrestore(ds_queue, flags); 4347 } 4348 4349 static unsigned long deferred_split_count(struct shrinker *shrink, 4350 struct shrink_control *sc) 4351 { 4352 struct pglist_data *pgdata = NODE_DATA(sc->nid); 4353 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 4354 4355 #ifdef CONFIG_MEMCG 4356 if (sc->memcg) 4357 ds_queue = &sc->memcg->deferred_split_queue; 4358 #endif 4359 return READ_ONCE(ds_queue->split_queue_len); 4360 } 4361 4362 static bool thp_underused(struct folio *folio) 4363 { 4364 int num_zero_pages = 0, num_filled_pages = 0; 4365 int i; 4366 4367 if (khugepaged_max_ptes_none == HPAGE_PMD_NR - 1) 4368 return false; 4369 4370 if (folio_contain_hwpoisoned_page(folio)) 4371 return false; 4372 4373 for (i = 0; i < folio_nr_pages(folio); i++) { 4374 if (pages_identical(folio_page(folio, i), ZERO_PAGE(0))) { 4375 if (++num_zero_pages > khugepaged_max_ptes_none) 4376 return true; 4377 } else { 4378 /* 4379 * Another path for early exit once the number 4380 * of non-zero filled pages exceeds threshold. 4381 */ 4382 if (++num_filled_pages >= HPAGE_PMD_NR - khugepaged_max_ptes_none) 4383 return false; 4384 } 4385 } 4386 return false; 4387 } 4388 4389 static unsigned long deferred_split_scan(struct shrinker *shrink, 4390 struct shrink_control *sc) 4391 { 4392 struct deferred_split *ds_queue; 4393 unsigned long flags; 4394 struct folio *folio, *next; 4395 int split = 0, i; 4396 struct folio_batch fbatch; 4397 4398 folio_batch_init(&fbatch); 4399 4400 retry: 4401 ds_queue = split_queue_lock_irqsave(sc->nid, sc->memcg, &flags); 4402 /* Take pin on all head pages to avoid freeing them under us */ 4403 list_for_each_entry_safe(folio, next, &ds_queue->split_queue, 4404 _deferred_list) { 4405 if (folio_try_get(folio)) { 4406 folio_batch_add(&fbatch, folio); 4407 } else if (folio_test_partially_mapped(folio)) { 4408 /* We lost race with folio_put() */ 4409 folio_clear_partially_mapped(folio); 4410 mod_mthp_stat(folio_order(folio), 4411 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 4412 } 4413 list_del_init(&folio->_deferred_list); 4414 ds_queue->split_queue_len--; 4415 if (!--sc->nr_to_scan) 4416 break; 4417 if (!folio_batch_space(&fbatch)) 4418 break; 4419 } 4420 split_queue_unlock_irqrestore(ds_queue, flags); 4421 4422 for (i = 0; i < folio_batch_count(&fbatch); i++) { 4423 bool did_split = false; 4424 bool underused = false; 4425 struct deferred_split *fqueue; 4426 4427 folio = fbatch.folios[i]; 4428 if (!folio_test_partially_mapped(folio)) { 4429 /* 4430 * See try_to_map_unused_to_zeropage(): we cannot 4431 * optimize zero-filled pages after splitting an 4432 * mlocked folio. 4433 */ 4434 if (folio_test_mlocked(folio)) 4435 goto next; 4436 underused = thp_underused(folio); 4437 if (!underused) 4438 goto next; 4439 } 4440 if (!folio_trylock(folio)) 4441 goto next; 4442 if (!split_folio(folio)) { 4443 did_split = true; 4444 if (underused) 4445 count_vm_event(THP_UNDERUSED_SPLIT_PAGE); 4446 split++; 4447 } 4448 folio_unlock(folio); 4449 next: 4450 if (did_split || !folio_test_partially_mapped(folio)) 4451 continue; 4452 /* 4453 * Only add back to the queue if folio is partially mapped. 4454 * If thp_underused returns false, or if split_folio fails 4455 * in the case it was underused, then consider it used and 4456 * don't add it back to split_queue. 4457 */ 4458 fqueue = folio_split_queue_lock_irqsave(folio, &flags); 4459 if (list_empty(&folio->_deferred_list)) { 4460 list_add_tail(&folio->_deferred_list, &fqueue->split_queue); 4461 fqueue->split_queue_len++; 4462 } 4463 split_queue_unlock_irqrestore(fqueue, flags); 4464 } 4465 folios_put(&fbatch); 4466 4467 if (sc->nr_to_scan && !list_empty(&ds_queue->split_queue)) { 4468 cond_resched(); 4469 goto retry; 4470 } 4471 4472 /* 4473 * Stop shrinker if we didn't split any page, but the queue is empty. 4474 * This can happen if pages were freed under us. 4475 */ 4476 if (!split && list_empty(&ds_queue->split_queue)) 4477 return SHRINK_STOP; 4478 return split; 4479 } 4480 4481 #ifdef CONFIG_MEMCG 4482 void reparent_deferred_split_queue(struct mem_cgroup *memcg) 4483 { 4484 struct mem_cgroup *parent = parent_mem_cgroup(memcg); 4485 struct deferred_split *ds_queue = &memcg->deferred_split_queue; 4486 struct deferred_split *parent_ds_queue = &parent->deferred_split_queue; 4487 int nid; 4488 4489 spin_lock_irq(&ds_queue->split_queue_lock); 4490 spin_lock_nested(&parent_ds_queue->split_queue_lock, SINGLE_DEPTH_NESTING); 4491 4492 if (!ds_queue->split_queue_len) 4493 goto unlock; 4494 4495 list_splice_tail_init(&ds_queue->split_queue, &parent_ds_queue->split_queue); 4496 parent_ds_queue->split_queue_len += ds_queue->split_queue_len; 4497 ds_queue->split_queue_len = 0; 4498 4499 for_each_node(nid) 4500 set_shrinker_bit(parent, nid, shrinker_id(deferred_split_shrinker)); 4501 4502 unlock: 4503 spin_unlock(&parent_ds_queue->split_queue_lock); 4504 spin_unlock_irq(&ds_queue->split_queue_lock); 4505 } 4506 #endif 4507 4508 #ifdef CONFIG_DEBUG_FS 4509 static void split_huge_pages_all(void) 4510 { 4511 struct zone *zone; 4512 struct page *page; 4513 struct folio *folio; 4514 unsigned long pfn, max_zone_pfn; 4515 unsigned long total = 0, split = 0; 4516 4517 pr_debug("Split all THPs\n"); 4518 for_each_zone(zone) { 4519 if (!managed_zone(zone)) 4520 continue; 4521 max_zone_pfn = zone_end_pfn(zone); 4522 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 4523 int nr_pages; 4524 4525 page = pfn_to_online_page(pfn); 4526 if (!page || PageTail(page)) 4527 continue; 4528 folio = page_folio(page); 4529 if (!folio_try_get(folio)) 4530 continue; 4531 4532 if (unlikely(page_folio(page) != folio)) 4533 goto next; 4534 4535 if (zone != folio_zone(folio)) 4536 goto next; 4537 4538 if (!folio_test_large(folio) 4539 || folio_test_hugetlb(folio) 4540 || !folio_test_lru(folio)) 4541 goto next; 4542 4543 total++; 4544 folio_lock(folio); 4545 nr_pages = folio_nr_pages(folio); 4546 if (!split_folio(folio)) 4547 split++; 4548 pfn += nr_pages - 1; 4549 folio_unlock(folio); 4550 next: 4551 folio_put(folio); 4552 cond_resched(); 4553 } 4554 } 4555 4556 pr_debug("%lu of %lu THP split\n", split, total); 4557 } 4558 4559 static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma) 4560 { 4561 return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) || 4562 is_vm_hugetlb_page(vma); 4563 } 4564 4565 static int split_huge_pages_pid(int pid, unsigned long vaddr_start, 4566 unsigned long vaddr_end, unsigned int new_order, 4567 long in_folio_offset) 4568 { 4569 int ret = 0; 4570 struct task_struct *task; 4571 struct mm_struct *mm; 4572 unsigned long total = 0, split = 0; 4573 unsigned long addr; 4574 4575 vaddr_start &= PAGE_MASK; 4576 vaddr_end &= PAGE_MASK; 4577 4578 task = find_get_task_by_vpid(pid); 4579 if (!task) { 4580 ret = -ESRCH; 4581 goto out; 4582 } 4583 4584 /* Find the mm_struct */ 4585 mm = get_task_mm(task); 4586 put_task_struct(task); 4587 4588 if (!mm) { 4589 ret = -EINVAL; 4590 goto out; 4591 } 4592 4593 pr_debug("Split huge pages in pid: %d, vaddr: [0x%lx - 0x%lx], new_order: %u, in_folio_offset: %ld\n", 4594 pid, vaddr_start, vaddr_end, new_order, in_folio_offset); 4595 4596 mmap_read_lock(mm); 4597 /* 4598 * always increase addr by PAGE_SIZE, since we could have a PTE page 4599 * table filled with PTE-mapped THPs, each of which is distinct. 4600 */ 4601 for (addr = vaddr_start; addr < vaddr_end; addr += PAGE_SIZE) { 4602 struct vm_area_struct *vma = vma_lookup(mm, addr); 4603 struct folio_walk fw; 4604 struct folio *folio; 4605 struct address_space *mapping; 4606 unsigned int target_order = new_order; 4607 4608 if (!vma) 4609 break; 4610 4611 /* skip special VMA and hugetlb VMA */ 4612 if (vma_not_suitable_for_thp_split(vma)) { 4613 addr = vma->vm_end; 4614 continue; 4615 } 4616 4617 folio = folio_walk_start(&fw, vma, addr, 0); 4618 if (!folio) 4619 continue; 4620 4621 if (!is_transparent_hugepage(folio)) 4622 goto next; 4623 4624 if (!folio_test_anon(folio)) { 4625 mapping = folio->mapping; 4626 target_order = max(new_order, 4627 mapping_min_folio_order(mapping)); 4628 } 4629 4630 if (target_order >= folio_order(folio)) 4631 goto next; 4632 4633 total++; 4634 /* 4635 * For folios with private, split_huge_page_to_list_to_order() 4636 * will try to drop it before split and then check if the folio 4637 * can be split or not. So skip the check here. 4638 */ 4639 if (!folio_test_private(folio) && 4640 folio_expected_ref_count(folio) != folio_ref_count(folio)) 4641 goto next; 4642 4643 if (!folio_trylock(folio)) 4644 goto next; 4645 folio_get(folio); 4646 folio_walk_end(&fw, vma); 4647 4648 if (!folio_test_anon(folio) && folio->mapping != mapping) 4649 goto unlock; 4650 4651 if (in_folio_offset < 0 || 4652 in_folio_offset >= folio_nr_pages(folio)) { 4653 if (!split_folio_to_order(folio, target_order)) 4654 split++; 4655 } else { 4656 struct page *split_at = folio_page(folio, 4657 in_folio_offset); 4658 if (!folio_split(folio, target_order, split_at, NULL)) 4659 split++; 4660 } 4661 4662 unlock: 4663 4664 folio_unlock(folio); 4665 folio_put(folio); 4666 4667 cond_resched(); 4668 continue; 4669 next: 4670 folio_walk_end(&fw, vma); 4671 cond_resched(); 4672 } 4673 mmap_read_unlock(mm); 4674 mmput(mm); 4675 4676 pr_debug("%lu of %lu THP split\n", split, total); 4677 4678 out: 4679 return ret; 4680 } 4681 4682 static int split_huge_pages_in_file(const char *file_path, pgoff_t off_start, 4683 pgoff_t off_end, unsigned int new_order, 4684 long in_folio_offset) 4685 { 4686 struct filename *file; 4687 struct file *candidate; 4688 struct address_space *mapping; 4689 int ret = -EINVAL; 4690 pgoff_t index; 4691 int nr_pages = 1; 4692 unsigned long total = 0, split = 0; 4693 unsigned int min_order; 4694 unsigned int target_order; 4695 4696 file = getname_kernel(file_path); 4697 if (IS_ERR(file)) 4698 return ret; 4699 4700 candidate = file_open_name(file, O_RDONLY, 0); 4701 if (IS_ERR(candidate)) 4702 goto out; 4703 4704 pr_debug("split file-backed THPs in file: %s, page offset: [0x%lx - 0x%lx], new_order: %u, in_folio_offset: %ld\n", 4705 file_path, off_start, off_end, new_order, in_folio_offset); 4706 4707 mapping = candidate->f_mapping; 4708 min_order = mapping_min_folio_order(mapping); 4709 target_order = max(new_order, min_order); 4710 4711 for (index = off_start; index < off_end; index += nr_pages) { 4712 struct folio *folio = filemap_get_folio(mapping, index); 4713 4714 nr_pages = 1; 4715 if (IS_ERR(folio)) 4716 continue; 4717 4718 if (!folio_test_large(folio)) 4719 goto next; 4720 4721 total++; 4722 nr_pages = folio_nr_pages(folio); 4723 4724 if (target_order >= folio_order(folio)) 4725 goto next; 4726 4727 if (!folio_trylock(folio)) 4728 goto next; 4729 4730 if (folio->mapping != mapping) 4731 goto unlock; 4732 4733 if (in_folio_offset < 0 || in_folio_offset >= nr_pages) { 4734 if (!split_folio_to_order(folio, target_order)) 4735 split++; 4736 } else { 4737 struct page *split_at = folio_page(folio, 4738 in_folio_offset); 4739 if (!folio_split(folio, target_order, split_at, NULL)) 4740 split++; 4741 } 4742 4743 unlock: 4744 folio_unlock(folio); 4745 next: 4746 folio_put(folio); 4747 cond_resched(); 4748 } 4749 4750 filp_close(candidate, NULL); 4751 ret = 0; 4752 4753 pr_debug("%lu of %lu file-backed THP split\n", split, total); 4754 out: 4755 putname(file); 4756 return ret; 4757 } 4758 4759 #define MAX_INPUT_BUF_SZ 255 4760 4761 static ssize_t split_huge_pages_write(struct file *file, const char __user *buf, 4762 size_t count, loff_t *ppops) 4763 { 4764 static DEFINE_MUTEX(split_debug_mutex); 4765 ssize_t ret; 4766 /* 4767 * hold pid, start_vaddr, end_vaddr, new_order or 4768 * file_path, off_start, off_end, new_order 4769 */ 4770 char input_buf[MAX_INPUT_BUF_SZ]; 4771 int pid; 4772 unsigned long vaddr_start, vaddr_end; 4773 unsigned int new_order = 0; 4774 long in_folio_offset = -1; 4775 4776 ret = mutex_lock_interruptible(&split_debug_mutex); 4777 if (ret) 4778 return ret; 4779 4780 ret = -EFAULT; 4781 4782 memset(input_buf, 0, MAX_INPUT_BUF_SZ); 4783 if (copy_from_user(input_buf, buf, min_t(size_t, count, MAX_INPUT_BUF_SZ))) 4784 goto out; 4785 4786 input_buf[MAX_INPUT_BUF_SZ - 1] = '\0'; 4787 4788 if (input_buf[0] == '/') { 4789 char *tok; 4790 char *tok_buf = input_buf; 4791 char file_path[MAX_INPUT_BUF_SZ]; 4792 pgoff_t off_start = 0, off_end = 0; 4793 size_t input_len = strlen(input_buf); 4794 4795 tok = strsep(&tok_buf, ","); 4796 if (tok && tok_buf) { 4797 strscpy(file_path, tok); 4798 } else { 4799 ret = -EINVAL; 4800 goto out; 4801 } 4802 4803 ret = sscanf(tok_buf, "0x%lx,0x%lx,%d,%ld", &off_start, &off_end, 4804 &new_order, &in_folio_offset); 4805 if (ret != 2 && ret != 3 && ret != 4) { 4806 ret = -EINVAL; 4807 goto out; 4808 } 4809 ret = split_huge_pages_in_file(file_path, off_start, off_end, 4810 new_order, in_folio_offset); 4811 if (!ret) 4812 ret = input_len; 4813 4814 goto out; 4815 } 4816 4817 ret = sscanf(input_buf, "%d,0x%lx,0x%lx,%d,%ld", &pid, &vaddr_start, 4818 &vaddr_end, &new_order, &in_folio_offset); 4819 if (ret == 1 && pid == 1) { 4820 split_huge_pages_all(); 4821 ret = strlen(input_buf); 4822 goto out; 4823 } else if (ret != 3 && ret != 4 && ret != 5) { 4824 ret = -EINVAL; 4825 goto out; 4826 } 4827 4828 ret = split_huge_pages_pid(pid, vaddr_start, vaddr_end, new_order, 4829 in_folio_offset); 4830 if (!ret) 4831 ret = strlen(input_buf); 4832 out: 4833 mutex_unlock(&split_debug_mutex); 4834 return ret; 4835 4836 } 4837 4838 static const struct file_operations split_huge_pages_fops = { 4839 .owner = THIS_MODULE, 4840 .write = split_huge_pages_write, 4841 }; 4842 4843 static int __init split_huge_pages_debugfs(void) 4844 { 4845 debugfs_create_file("split_huge_pages", 0200, NULL, NULL, 4846 &split_huge_pages_fops); 4847 return 0; 4848 } 4849 late_initcall(split_huge_pages_debugfs); 4850 #endif 4851 4852 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 4853 int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 4854 struct page *page) 4855 { 4856 struct folio *folio = page_folio(page); 4857 struct vm_area_struct *vma = pvmw->vma; 4858 struct mm_struct *mm = vma->vm_mm; 4859 unsigned long address = pvmw->address; 4860 bool anon_exclusive; 4861 pmd_t pmdval; 4862 swp_entry_t entry; 4863 pmd_t pmdswp; 4864 4865 if (!(pvmw->pmd && !pvmw->pte)) 4866 return 0; 4867 4868 flush_cache_range(vma, address, address + HPAGE_PMD_SIZE); 4869 if (unlikely(!pmd_present(*pvmw->pmd))) 4870 pmdval = pmdp_huge_get_and_clear(vma->vm_mm, address, pvmw->pmd); 4871 else 4872 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); 4873 4874 /* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. */ 4875 anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(page); 4876 if (anon_exclusive && folio_try_share_anon_rmap_pmd(folio, page)) { 4877 set_pmd_at(mm, address, pvmw->pmd, pmdval); 4878 return -EBUSY; 4879 } 4880 4881 if (pmd_dirty(pmdval)) 4882 folio_mark_dirty(folio); 4883 if (pmd_write(pmdval)) 4884 entry = make_writable_migration_entry(page_to_pfn(page)); 4885 else if (anon_exclusive) 4886 entry = make_readable_exclusive_migration_entry(page_to_pfn(page)); 4887 else 4888 entry = make_readable_migration_entry(page_to_pfn(page)); 4889 if (pmd_young(pmdval)) 4890 entry = make_migration_entry_young(entry); 4891 if (pmd_dirty(pmdval)) 4892 entry = make_migration_entry_dirty(entry); 4893 pmdswp = swp_entry_to_pmd(entry); 4894 if (pmd_soft_dirty(pmdval)) 4895 pmdswp = pmd_swp_mksoft_dirty(pmdswp); 4896 if (pmd_uffd_wp(pmdval)) 4897 pmdswp = pmd_swp_mkuffd_wp(pmdswp); 4898 set_pmd_at(mm, address, pvmw->pmd, pmdswp); 4899 folio_remove_rmap_pmd(folio, page, vma); 4900 folio_put(folio); 4901 trace_set_migration_pmd(address, pmd_val(pmdswp)); 4902 4903 return 0; 4904 } 4905 4906 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) 4907 { 4908 struct folio *folio = page_folio(new); 4909 struct vm_area_struct *vma = pvmw->vma; 4910 struct mm_struct *mm = vma->vm_mm; 4911 unsigned long address = pvmw->address; 4912 unsigned long haddr = address & HPAGE_PMD_MASK; 4913 pmd_t pmde; 4914 softleaf_t entry; 4915 4916 if (!(pvmw->pmd && !pvmw->pte)) 4917 return; 4918 4919 entry = softleaf_from_pmd(*pvmw->pmd); 4920 folio_get(folio); 4921 pmde = folio_mk_pmd(folio, READ_ONCE(vma->vm_page_prot)); 4922 4923 if (pmd_swp_soft_dirty(*pvmw->pmd)) 4924 pmde = pmd_mksoft_dirty(pmde); 4925 if (softleaf_is_migration_write(entry)) 4926 pmde = pmd_mkwrite(pmde, vma); 4927 if (pmd_swp_uffd_wp(*pvmw->pmd)) 4928 pmde = pmd_mkuffd_wp(pmde); 4929 if (!softleaf_is_migration_young(entry)) 4930 pmde = pmd_mkold(pmde); 4931 /* NOTE: this may contain setting soft-dirty on some archs */ 4932 if (folio_test_dirty(folio) && softleaf_is_migration_dirty(entry)) 4933 pmde = pmd_mkdirty(pmde); 4934 4935 if (folio_is_device_private(folio)) { 4936 swp_entry_t entry; 4937 4938 if (pmd_write(pmde)) 4939 entry = make_writable_device_private_entry( 4940 page_to_pfn(new)); 4941 else 4942 entry = make_readable_device_private_entry( 4943 page_to_pfn(new)); 4944 pmde = swp_entry_to_pmd(entry); 4945 4946 if (pmd_swp_soft_dirty(*pvmw->pmd)) 4947 pmde = pmd_swp_mksoft_dirty(pmde); 4948 if (pmd_swp_uffd_wp(*pvmw->pmd)) 4949 pmde = pmd_swp_mkuffd_wp(pmde); 4950 } 4951 4952 if (folio_test_anon(folio)) { 4953 rmap_t rmap_flags = RMAP_NONE; 4954 4955 if (!softleaf_is_migration_read(entry)) 4956 rmap_flags |= RMAP_EXCLUSIVE; 4957 4958 folio_add_anon_rmap_pmd(folio, new, vma, haddr, rmap_flags); 4959 } else { 4960 folio_add_file_rmap_pmd(folio, new, vma); 4961 } 4962 VM_BUG_ON(pmd_write(pmde) && folio_test_anon(folio) && !PageAnonExclusive(new)); 4963 set_pmd_at(mm, haddr, pvmw->pmd, pmde); 4964 4965 /* No need to invalidate - it was non-present before */ 4966 update_mmu_cache_pmd(vma, address, pvmw->pmd); 4967 trace_remove_migration_pmd(address, pmd_val(pmde)); 4968 } 4969 #endif 4970