1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2009 Red Hat, Inc. 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/mm.h> 9 #include <linux/sched.h> 10 #include <linux/sched/mm.h> 11 #include <linux/sched/numa_balancing.h> 12 #include <linux/highmem.h> 13 #include <linux/hugetlb.h> 14 #include <linux/mmu_notifier.h> 15 #include <linux/rmap.h> 16 #include <linux/swap.h> 17 #include <linux/shrinker.h> 18 #include <linux/mm_inline.h> 19 #include <linux/swapops.h> 20 #include <linux/backing-dev.h> 21 #include <linux/dax.h> 22 #include <linux/mm_types.h> 23 #include <linux/khugepaged.h> 24 #include <linux/freezer.h> 25 #include <linux/pfn_t.h> 26 #include <linux/mman.h> 27 #include <linux/memremap.h> 28 #include <linux/pagemap.h> 29 #include <linux/debugfs.h> 30 #include <linux/migrate.h> 31 #include <linux/hashtable.h> 32 #include <linux/userfaultfd_k.h> 33 #include <linux/page_idle.h> 34 #include <linux/shmem_fs.h> 35 #include <linux/oom.h> 36 #include <linux/numa.h> 37 #include <linux/page_owner.h> 38 #include <linux/sched/sysctl.h> 39 #include <linux/memory-tiers.h> 40 #include <linux/compat.h> 41 #include <linux/pgalloc_tag.h> 42 #include <linux/pagewalk.h> 43 44 #include <asm/tlb.h> 45 #include <asm/pgalloc.h> 46 #include "internal.h" 47 #include "swap.h" 48 49 #define CREATE_TRACE_POINTS 50 #include <trace/events/thp.h> 51 52 /* 53 * By default, transparent hugepage support is disabled in order to avoid 54 * risking an increased memory footprint for applications that are not 55 * guaranteed to benefit from it. When transparent hugepage support is 56 * enabled, it is for all mappings, and khugepaged scans all mappings. 57 * Defrag is invoked by khugepaged hugepage allocations and by page faults 58 * for all hugepage allocations. 59 */ 60 unsigned long transparent_hugepage_flags __read_mostly = 61 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS 62 (1<<TRANSPARENT_HUGEPAGE_FLAG)| 63 #endif 64 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE 65 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| 66 #endif 67 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)| 68 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| 69 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 70 71 static struct shrinker *deferred_split_shrinker; 72 static unsigned long deferred_split_count(struct shrinker *shrink, 73 struct shrink_control *sc); 74 static unsigned long deferred_split_scan(struct shrinker *shrink, 75 struct shrink_control *sc); 76 static bool split_underused_thp = true; 77 78 static atomic_t huge_zero_refcount; 79 struct folio *huge_zero_folio __read_mostly; 80 unsigned long huge_zero_pfn __read_mostly = ~0UL; 81 unsigned long huge_anon_orders_always __read_mostly; 82 unsigned long huge_anon_orders_madvise __read_mostly; 83 unsigned long huge_anon_orders_inherit __read_mostly; 84 static bool anon_orders_configured __initdata; 85 86 static inline bool file_thp_enabled(struct vm_area_struct *vma) 87 { 88 struct inode *inode; 89 90 if (!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS)) 91 return false; 92 93 if (!vma->vm_file) 94 return false; 95 96 inode = file_inode(vma->vm_file); 97 98 return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); 99 } 100 101 unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, 102 unsigned long vm_flags, 103 unsigned long tva_flags, 104 unsigned long orders) 105 { 106 bool smaps = tva_flags & TVA_SMAPS; 107 bool in_pf = tva_flags & TVA_IN_PF; 108 bool enforce_sysfs = tva_flags & TVA_ENFORCE_SYSFS; 109 unsigned long supported_orders; 110 111 /* Check the intersection of requested and supported orders. */ 112 if (vma_is_anonymous(vma)) 113 supported_orders = THP_ORDERS_ALL_ANON; 114 else if (vma_is_special_huge(vma)) 115 supported_orders = THP_ORDERS_ALL_SPECIAL; 116 else 117 supported_orders = THP_ORDERS_ALL_FILE_DEFAULT; 118 119 orders &= supported_orders; 120 if (!orders) 121 return 0; 122 123 if (!vma->vm_mm) /* vdso */ 124 return 0; 125 126 if (thp_disabled_by_hw() || vma_thp_disabled(vma, vm_flags)) 127 return 0; 128 129 /* khugepaged doesn't collapse DAX vma, but page fault is fine. */ 130 if (vma_is_dax(vma)) 131 return in_pf ? orders : 0; 132 133 /* 134 * khugepaged special VMA and hugetlb VMA. 135 * Must be checked after dax since some dax mappings may have 136 * VM_MIXEDMAP set. 137 */ 138 if (!in_pf && !smaps && (vm_flags & VM_NO_KHUGEPAGED)) 139 return 0; 140 141 /* 142 * Check alignment for file vma and size for both file and anon vma by 143 * filtering out the unsuitable orders. 144 * 145 * Skip the check for page fault. Huge fault does the check in fault 146 * handlers. 147 */ 148 if (!in_pf) { 149 int order = highest_order(orders); 150 unsigned long addr; 151 152 while (orders) { 153 addr = vma->vm_end - (PAGE_SIZE << order); 154 if (thp_vma_suitable_order(vma, addr, order)) 155 break; 156 order = next_order(&orders, order); 157 } 158 159 if (!orders) 160 return 0; 161 } 162 163 /* 164 * Enabled via shmem mount options or sysfs settings. 165 * Must be done before hugepage flags check since shmem has its 166 * own flags. 167 */ 168 if (!in_pf && shmem_file(vma->vm_file)) 169 return shmem_allowable_huge_orders(file_inode(vma->vm_file), 170 vma, vma->vm_pgoff, 0, 171 !enforce_sysfs); 172 173 if (!vma_is_anonymous(vma)) { 174 /* 175 * Enforce sysfs THP requirements as necessary. Anonymous vmas 176 * were already handled in thp_vma_allowable_orders(). 177 */ 178 if (enforce_sysfs && 179 (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) && 180 !hugepage_global_always()))) 181 return 0; 182 183 /* 184 * Trust that ->huge_fault() handlers know what they are doing 185 * in fault path. 186 */ 187 if (((in_pf || smaps)) && vma->vm_ops->huge_fault) 188 return orders; 189 /* Only regular file is valid in collapse path */ 190 if (((!in_pf || smaps)) && file_thp_enabled(vma)) 191 return orders; 192 return 0; 193 } 194 195 if (vma_is_temporary_stack(vma)) 196 return 0; 197 198 /* 199 * THPeligible bit of smaps should show 1 for proper VMAs even 200 * though anon_vma is not initialized yet. 201 * 202 * Allow page fault since anon_vma may be not initialized until 203 * the first page fault. 204 */ 205 if (!vma->anon_vma) 206 return (smaps || in_pf) ? orders : 0; 207 208 return orders; 209 } 210 211 static bool get_huge_zero_page(void) 212 { 213 struct folio *zero_folio; 214 retry: 215 if (likely(atomic_inc_not_zero(&huge_zero_refcount))) 216 return true; 217 218 zero_folio = folio_alloc((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE, 219 HPAGE_PMD_ORDER); 220 if (!zero_folio) { 221 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); 222 return false; 223 } 224 /* Ensure zero folio won't have large_rmappable flag set. */ 225 folio_clear_large_rmappable(zero_folio); 226 preempt_disable(); 227 if (cmpxchg(&huge_zero_folio, NULL, zero_folio)) { 228 preempt_enable(); 229 folio_put(zero_folio); 230 goto retry; 231 } 232 WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio)); 233 234 /* We take additional reference here. It will be put back by shrinker */ 235 atomic_set(&huge_zero_refcount, 2); 236 preempt_enable(); 237 count_vm_event(THP_ZERO_PAGE_ALLOC); 238 return true; 239 } 240 241 static void put_huge_zero_page(void) 242 { 243 /* 244 * Counter should never go to zero here. Only shrinker can put 245 * last reference. 246 */ 247 BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); 248 } 249 250 struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) 251 { 252 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 253 return READ_ONCE(huge_zero_folio); 254 255 if (!get_huge_zero_page()) 256 return NULL; 257 258 if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 259 put_huge_zero_page(); 260 261 return READ_ONCE(huge_zero_folio); 262 } 263 264 void mm_put_huge_zero_folio(struct mm_struct *mm) 265 { 266 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 267 put_huge_zero_page(); 268 } 269 270 static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink, 271 struct shrink_control *sc) 272 { 273 /* we can free zero page only if last reference remains */ 274 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; 275 } 276 277 static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink, 278 struct shrink_control *sc) 279 { 280 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { 281 struct folio *zero_folio = xchg(&huge_zero_folio, NULL); 282 BUG_ON(zero_folio == NULL); 283 WRITE_ONCE(huge_zero_pfn, ~0UL); 284 folio_put(zero_folio); 285 return HPAGE_PMD_NR; 286 } 287 288 return 0; 289 } 290 291 static struct shrinker *huge_zero_page_shrinker; 292 293 #ifdef CONFIG_SYSFS 294 static ssize_t enabled_show(struct kobject *kobj, 295 struct kobj_attribute *attr, char *buf) 296 { 297 const char *output; 298 299 if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags)) 300 output = "[always] madvise never"; 301 else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 302 &transparent_hugepage_flags)) 303 output = "always [madvise] never"; 304 else 305 output = "always madvise [never]"; 306 307 return sysfs_emit(buf, "%s\n", output); 308 } 309 310 static ssize_t enabled_store(struct kobject *kobj, 311 struct kobj_attribute *attr, 312 const char *buf, size_t count) 313 { 314 ssize_t ret = count; 315 316 if (sysfs_streq(buf, "always")) { 317 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 318 set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 319 } else if (sysfs_streq(buf, "madvise")) { 320 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 321 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 322 } else if (sysfs_streq(buf, "never")) { 323 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 324 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 325 } else 326 ret = -EINVAL; 327 328 if (ret > 0) { 329 int err = start_stop_khugepaged(); 330 if (err) 331 ret = err; 332 } 333 return ret; 334 } 335 336 static struct kobj_attribute enabled_attr = __ATTR_RW(enabled); 337 338 ssize_t single_hugepage_flag_show(struct kobject *kobj, 339 struct kobj_attribute *attr, char *buf, 340 enum transparent_hugepage_flag flag) 341 { 342 return sysfs_emit(buf, "%d\n", 343 !!test_bit(flag, &transparent_hugepage_flags)); 344 } 345 346 ssize_t single_hugepage_flag_store(struct kobject *kobj, 347 struct kobj_attribute *attr, 348 const char *buf, size_t count, 349 enum transparent_hugepage_flag flag) 350 { 351 unsigned long value; 352 int ret; 353 354 ret = kstrtoul(buf, 10, &value); 355 if (ret < 0) 356 return ret; 357 if (value > 1) 358 return -EINVAL; 359 360 if (value) 361 set_bit(flag, &transparent_hugepage_flags); 362 else 363 clear_bit(flag, &transparent_hugepage_flags); 364 365 return count; 366 } 367 368 static ssize_t defrag_show(struct kobject *kobj, 369 struct kobj_attribute *attr, char *buf) 370 { 371 const char *output; 372 373 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, 374 &transparent_hugepage_flags)) 375 output = "[always] defer defer+madvise madvise never"; 376 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, 377 &transparent_hugepage_flags)) 378 output = "always [defer] defer+madvise madvise never"; 379 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, 380 &transparent_hugepage_flags)) 381 output = "always defer [defer+madvise] madvise never"; 382 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, 383 &transparent_hugepage_flags)) 384 output = "always defer defer+madvise [madvise] never"; 385 else 386 output = "always defer defer+madvise madvise [never]"; 387 388 return sysfs_emit(buf, "%s\n", output); 389 } 390 391 static ssize_t defrag_store(struct kobject *kobj, 392 struct kobj_attribute *attr, 393 const char *buf, size_t count) 394 { 395 if (sysfs_streq(buf, "always")) { 396 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 397 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 398 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 399 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 400 } else if (sysfs_streq(buf, "defer+madvise")) { 401 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 402 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 403 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 404 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 405 } else if (sysfs_streq(buf, "defer")) { 406 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 407 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 408 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 409 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 410 } else if (sysfs_streq(buf, "madvise")) { 411 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 412 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 413 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 414 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 415 } else if (sysfs_streq(buf, "never")) { 416 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 417 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 418 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 419 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 420 } else 421 return -EINVAL; 422 423 return count; 424 } 425 static struct kobj_attribute defrag_attr = __ATTR_RW(defrag); 426 427 static ssize_t use_zero_page_show(struct kobject *kobj, 428 struct kobj_attribute *attr, char *buf) 429 { 430 return single_hugepage_flag_show(kobj, attr, buf, 431 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 432 } 433 static ssize_t use_zero_page_store(struct kobject *kobj, 434 struct kobj_attribute *attr, const char *buf, size_t count) 435 { 436 return single_hugepage_flag_store(kobj, attr, buf, count, 437 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 438 } 439 static struct kobj_attribute use_zero_page_attr = __ATTR_RW(use_zero_page); 440 441 static ssize_t hpage_pmd_size_show(struct kobject *kobj, 442 struct kobj_attribute *attr, char *buf) 443 { 444 return sysfs_emit(buf, "%lu\n", HPAGE_PMD_SIZE); 445 } 446 static struct kobj_attribute hpage_pmd_size_attr = 447 __ATTR_RO(hpage_pmd_size); 448 449 static ssize_t split_underused_thp_show(struct kobject *kobj, 450 struct kobj_attribute *attr, char *buf) 451 { 452 return sysfs_emit(buf, "%d\n", split_underused_thp); 453 } 454 455 static ssize_t split_underused_thp_store(struct kobject *kobj, 456 struct kobj_attribute *attr, 457 const char *buf, size_t count) 458 { 459 int err = kstrtobool(buf, &split_underused_thp); 460 461 if (err < 0) 462 return err; 463 464 return count; 465 } 466 467 static struct kobj_attribute split_underused_thp_attr = __ATTR( 468 shrink_underused, 0644, split_underused_thp_show, split_underused_thp_store); 469 470 static struct attribute *hugepage_attr[] = { 471 &enabled_attr.attr, 472 &defrag_attr.attr, 473 &use_zero_page_attr.attr, 474 &hpage_pmd_size_attr.attr, 475 #ifdef CONFIG_SHMEM 476 &shmem_enabled_attr.attr, 477 #endif 478 &split_underused_thp_attr.attr, 479 NULL, 480 }; 481 482 static const struct attribute_group hugepage_attr_group = { 483 .attrs = hugepage_attr, 484 }; 485 486 static void hugepage_exit_sysfs(struct kobject *hugepage_kobj); 487 static void thpsize_release(struct kobject *kobj); 488 static DEFINE_SPINLOCK(huge_anon_orders_lock); 489 static LIST_HEAD(thpsize_list); 490 491 static ssize_t anon_enabled_show(struct kobject *kobj, 492 struct kobj_attribute *attr, char *buf) 493 { 494 int order = to_thpsize(kobj)->order; 495 const char *output; 496 497 if (test_bit(order, &huge_anon_orders_always)) 498 output = "[always] inherit madvise never"; 499 else if (test_bit(order, &huge_anon_orders_inherit)) 500 output = "always [inherit] madvise never"; 501 else if (test_bit(order, &huge_anon_orders_madvise)) 502 output = "always inherit [madvise] never"; 503 else 504 output = "always inherit madvise [never]"; 505 506 return sysfs_emit(buf, "%s\n", output); 507 } 508 509 static ssize_t anon_enabled_store(struct kobject *kobj, 510 struct kobj_attribute *attr, 511 const char *buf, size_t count) 512 { 513 int order = to_thpsize(kobj)->order; 514 ssize_t ret = count; 515 516 if (sysfs_streq(buf, "always")) { 517 spin_lock(&huge_anon_orders_lock); 518 clear_bit(order, &huge_anon_orders_inherit); 519 clear_bit(order, &huge_anon_orders_madvise); 520 set_bit(order, &huge_anon_orders_always); 521 spin_unlock(&huge_anon_orders_lock); 522 } else if (sysfs_streq(buf, "inherit")) { 523 spin_lock(&huge_anon_orders_lock); 524 clear_bit(order, &huge_anon_orders_always); 525 clear_bit(order, &huge_anon_orders_madvise); 526 set_bit(order, &huge_anon_orders_inherit); 527 spin_unlock(&huge_anon_orders_lock); 528 } else if (sysfs_streq(buf, "madvise")) { 529 spin_lock(&huge_anon_orders_lock); 530 clear_bit(order, &huge_anon_orders_always); 531 clear_bit(order, &huge_anon_orders_inherit); 532 set_bit(order, &huge_anon_orders_madvise); 533 spin_unlock(&huge_anon_orders_lock); 534 } else if (sysfs_streq(buf, "never")) { 535 spin_lock(&huge_anon_orders_lock); 536 clear_bit(order, &huge_anon_orders_always); 537 clear_bit(order, &huge_anon_orders_inherit); 538 clear_bit(order, &huge_anon_orders_madvise); 539 spin_unlock(&huge_anon_orders_lock); 540 } else 541 ret = -EINVAL; 542 543 if (ret > 0) { 544 int err; 545 546 err = start_stop_khugepaged(); 547 if (err) 548 ret = err; 549 } 550 return ret; 551 } 552 553 static struct kobj_attribute anon_enabled_attr = 554 __ATTR(enabled, 0644, anon_enabled_show, anon_enabled_store); 555 556 static struct attribute *anon_ctrl_attrs[] = { 557 &anon_enabled_attr.attr, 558 NULL, 559 }; 560 561 static const struct attribute_group anon_ctrl_attr_grp = { 562 .attrs = anon_ctrl_attrs, 563 }; 564 565 static struct attribute *file_ctrl_attrs[] = { 566 #ifdef CONFIG_SHMEM 567 &thpsize_shmem_enabled_attr.attr, 568 #endif 569 NULL, 570 }; 571 572 static const struct attribute_group file_ctrl_attr_grp = { 573 .attrs = file_ctrl_attrs, 574 }; 575 576 static struct attribute *any_ctrl_attrs[] = { 577 NULL, 578 }; 579 580 static const struct attribute_group any_ctrl_attr_grp = { 581 .attrs = any_ctrl_attrs, 582 }; 583 584 static const struct kobj_type thpsize_ktype = { 585 .release = &thpsize_release, 586 .sysfs_ops = &kobj_sysfs_ops, 587 }; 588 589 DEFINE_PER_CPU(struct mthp_stat, mthp_stats) = {{{0}}}; 590 591 static unsigned long sum_mthp_stat(int order, enum mthp_stat_item item) 592 { 593 unsigned long sum = 0; 594 int cpu; 595 596 for_each_possible_cpu(cpu) { 597 struct mthp_stat *this = &per_cpu(mthp_stats, cpu); 598 599 sum += this->stats[order][item]; 600 } 601 602 return sum; 603 } 604 605 #define DEFINE_MTHP_STAT_ATTR(_name, _index) \ 606 static ssize_t _name##_show(struct kobject *kobj, \ 607 struct kobj_attribute *attr, char *buf) \ 608 { \ 609 int order = to_thpsize(kobj)->order; \ 610 \ 611 return sysfs_emit(buf, "%lu\n", sum_mthp_stat(order, _index)); \ 612 } \ 613 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 614 615 DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC); 616 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK); 617 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); 618 DEFINE_MTHP_STAT_ATTR(zswpout, MTHP_STAT_ZSWPOUT); 619 DEFINE_MTHP_STAT_ATTR(swpin, MTHP_STAT_SWPIN); 620 DEFINE_MTHP_STAT_ATTR(swpout, MTHP_STAT_SWPOUT); 621 DEFINE_MTHP_STAT_ATTR(swpout_fallback, MTHP_STAT_SWPOUT_FALLBACK); 622 #ifdef CONFIG_SHMEM 623 DEFINE_MTHP_STAT_ATTR(shmem_alloc, MTHP_STAT_SHMEM_ALLOC); 624 DEFINE_MTHP_STAT_ATTR(shmem_fallback, MTHP_STAT_SHMEM_FALLBACK); 625 DEFINE_MTHP_STAT_ATTR(shmem_fallback_charge, MTHP_STAT_SHMEM_FALLBACK_CHARGE); 626 #endif 627 DEFINE_MTHP_STAT_ATTR(split, MTHP_STAT_SPLIT); 628 DEFINE_MTHP_STAT_ATTR(split_failed, MTHP_STAT_SPLIT_FAILED); 629 DEFINE_MTHP_STAT_ATTR(split_deferred, MTHP_STAT_SPLIT_DEFERRED); 630 DEFINE_MTHP_STAT_ATTR(nr_anon, MTHP_STAT_NR_ANON); 631 DEFINE_MTHP_STAT_ATTR(nr_anon_partially_mapped, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED); 632 633 static struct attribute *anon_stats_attrs[] = { 634 &anon_fault_alloc_attr.attr, 635 &anon_fault_fallback_attr.attr, 636 &anon_fault_fallback_charge_attr.attr, 637 #ifndef CONFIG_SHMEM 638 &zswpout_attr.attr, 639 &swpin_attr.attr, 640 &swpout_attr.attr, 641 &swpout_fallback_attr.attr, 642 #endif 643 &split_deferred_attr.attr, 644 &nr_anon_attr.attr, 645 &nr_anon_partially_mapped_attr.attr, 646 NULL, 647 }; 648 649 static struct attribute_group anon_stats_attr_grp = { 650 .name = "stats", 651 .attrs = anon_stats_attrs, 652 }; 653 654 static struct attribute *file_stats_attrs[] = { 655 #ifdef CONFIG_SHMEM 656 &shmem_alloc_attr.attr, 657 &shmem_fallback_attr.attr, 658 &shmem_fallback_charge_attr.attr, 659 #endif 660 NULL, 661 }; 662 663 static struct attribute_group file_stats_attr_grp = { 664 .name = "stats", 665 .attrs = file_stats_attrs, 666 }; 667 668 static struct attribute *any_stats_attrs[] = { 669 #ifdef CONFIG_SHMEM 670 &zswpout_attr.attr, 671 &swpin_attr.attr, 672 &swpout_attr.attr, 673 &swpout_fallback_attr.attr, 674 #endif 675 &split_attr.attr, 676 &split_failed_attr.attr, 677 NULL, 678 }; 679 680 static struct attribute_group any_stats_attr_grp = { 681 .name = "stats", 682 .attrs = any_stats_attrs, 683 }; 684 685 static int sysfs_add_group(struct kobject *kobj, 686 const struct attribute_group *grp) 687 { 688 int ret = -ENOENT; 689 690 /* 691 * If the group is named, try to merge first, assuming the subdirectory 692 * was already created. This avoids the warning emitted by 693 * sysfs_create_group() if the directory already exists. 694 */ 695 if (grp->name) 696 ret = sysfs_merge_group(kobj, grp); 697 if (ret) 698 ret = sysfs_create_group(kobj, grp); 699 700 return ret; 701 } 702 703 static struct thpsize *thpsize_create(int order, struct kobject *parent) 704 { 705 unsigned long size = (PAGE_SIZE << order) / SZ_1K; 706 struct thpsize *thpsize; 707 int ret = -ENOMEM; 708 709 thpsize = kzalloc(sizeof(*thpsize), GFP_KERNEL); 710 if (!thpsize) 711 goto err; 712 713 thpsize->order = order; 714 715 ret = kobject_init_and_add(&thpsize->kobj, &thpsize_ktype, parent, 716 "hugepages-%lukB", size); 717 if (ret) { 718 kfree(thpsize); 719 goto err; 720 } 721 722 723 ret = sysfs_add_group(&thpsize->kobj, &any_ctrl_attr_grp); 724 if (ret) 725 goto err_put; 726 727 ret = sysfs_add_group(&thpsize->kobj, &any_stats_attr_grp); 728 if (ret) 729 goto err_put; 730 731 if (BIT(order) & THP_ORDERS_ALL_ANON) { 732 ret = sysfs_add_group(&thpsize->kobj, &anon_ctrl_attr_grp); 733 if (ret) 734 goto err_put; 735 736 ret = sysfs_add_group(&thpsize->kobj, &anon_stats_attr_grp); 737 if (ret) 738 goto err_put; 739 } 740 741 if (BIT(order) & THP_ORDERS_ALL_FILE_DEFAULT) { 742 ret = sysfs_add_group(&thpsize->kobj, &file_ctrl_attr_grp); 743 if (ret) 744 goto err_put; 745 746 ret = sysfs_add_group(&thpsize->kobj, &file_stats_attr_grp); 747 if (ret) 748 goto err_put; 749 } 750 751 return thpsize; 752 err_put: 753 kobject_put(&thpsize->kobj); 754 err: 755 return ERR_PTR(ret); 756 } 757 758 static void thpsize_release(struct kobject *kobj) 759 { 760 kfree(to_thpsize(kobj)); 761 } 762 763 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj) 764 { 765 int err; 766 struct thpsize *thpsize; 767 unsigned long orders; 768 int order; 769 770 /* 771 * Default to setting PMD-sized THP to inherit the global setting and 772 * disable all other sizes. powerpc's PMD_ORDER isn't a compile-time 773 * constant so we have to do this here. 774 */ 775 if (!anon_orders_configured) 776 huge_anon_orders_inherit = BIT(PMD_ORDER); 777 778 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj); 779 if (unlikely(!*hugepage_kobj)) { 780 pr_err("failed to create transparent hugepage kobject\n"); 781 return -ENOMEM; 782 } 783 784 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group); 785 if (err) { 786 pr_err("failed to register transparent hugepage group\n"); 787 goto delete_obj; 788 } 789 790 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group); 791 if (err) { 792 pr_err("failed to register transparent hugepage group\n"); 793 goto remove_hp_group; 794 } 795 796 orders = THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_FILE_DEFAULT; 797 order = highest_order(orders); 798 while (orders) { 799 thpsize = thpsize_create(order, *hugepage_kobj); 800 if (IS_ERR(thpsize)) { 801 pr_err("failed to create thpsize for order %d\n", order); 802 err = PTR_ERR(thpsize); 803 goto remove_all; 804 } 805 list_add(&thpsize->node, &thpsize_list); 806 order = next_order(&orders, order); 807 } 808 809 return 0; 810 811 remove_all: 812 hugepage_exit_sysfs(*hugepage_kobj); 813 return err; 814 remove_hp_group: 815 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group); 816 delete_obj: 817 kobject_put(*hugepage_kobj); 818 return err; 819 } 820 821 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj) 822 { 823 struct thpsize *thpsize, *tmp; 824 825 list_for_each_entry_safe(thpsize, tmp, &thpsize_list, node) { 826 list_del(&thpsize->node); 827 kobject_put(&thpsize->kobj); 828 } 829 830 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group); 831 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group); 832 kobject_put(hugepage_kobj); 833 } 834 #else 835 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj) 836 { 837 return 0; 838 } 839 840 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) 841 { 842 } 843 #endif /* CONFIG_SYSFS */ 844 845 static int __init thp_shrinker_init(void) 846 { 847 huge_zero_page_shrinker = shrinker_alloc(0, "thp-zero"); 848 if (!huge_zero_page_shrinker) 849 return -ENOMEM; 850 851 deferred_split_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE | 852 SHRINKER_MEMCG_AWARE | 853 SHRINKER_NONSLAB, 854 "thp-deferred_split"); 855 if (!deferred_split_shrinker) { 856 shrinker_free(huge_zero_page_shrinker); 857 return -ENOMEM; 858 } 859 860 huge_zero_page_shrinker->count_objects = shrink_huge_zero_page_count; 861 huge_zero_page_shrinker->scan_objects = shrink_huge_zero_page_scan; 862 shrinker_register(huge_zero_page_shrinker); 863 864 deferred_split_shrinker->count_objects = deferred_split_count; 865 deferred_split_shrinker->scan_objects = deferred_split_scan; 866 shrinker_register(deferred_split_shrinker); 867 868 return 0; 869 } 870 871 static void __init thp_shrinker_exit(void) 872 { 873 shrinker_free(huge_zero_page_shrinker); 874 shrinker_free(deferred_split_shrinker); 875 } 876 877 static int __init hugepage_init(void) 878 { 879 int err; 880 struct kobject *hugepage_kobj; 881 882 if (!has_transparent_hugepage()) { 883 transparent_hugepage_flags = 1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED; 884 return -EINVAL; 885 } 886 887 /* 888 * hugepages can't be allocated by the buddy allocator 889 */ 890 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER > MAX_PAGE_ORDER); 891 892 err = hugepage_init_sysfs(&hugepage_kobj); 893 if (err) 894 goto err_sysfs; 895 896 err = khugepaged_init(); 897 if (err) 898 goto err_slab; 899 900 err = thp_shrinker_init(); 901 if (err) 902 goto err_shrinker; 903 904 /* 905 * By default disable transparent hugepages on smaller systems, 906 * where the extra memory used could hurt more than TLB overhead 907 * is likely to save. The admin can still enable it through /sys. 908 */ 909 if (totalram_pages() < (512 << (20 - PAGE_SHIFT))) { 910 transparent_hugepage_flags = 0; 911 return 0; 912 } 913 914 err = start_stop_khugepaged(); 915 if (err) 916 goto err_khugepaged; 917 918 return 0; 919 err_khugepaged: 920 thp_shrinker_exit(); 921 err_shrinker: 922 khugepaged_destroy(); 923 err_slab: 924 hugepage_exit_sysfs(hugepage_kobj); 925 err_sysfs: 926 return err; 927 } 928 subsys_initcall(hugepage_init); 929 930 static int __init setup_transparent_hugepage(char *str) 931 { 932 int ret = 0; 933 if (!str) 934 goto out; 935 if (!strcmp(str, "always")) { 936 set_bit(TRANSPARENT_HUGEPAGE_FLAG, 937 &transparent_hugepage_flags); 938 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 939 &transparent_hugepage_flags); 940 ret = 1; 941 } else if (!strcmp(str, "madvise")) { 942 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 943 &transparent_hugepage_flags); 944 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 945 &transparent_hugepage_flags); 946 ret = 1; 947 } else if (!strcmp(str, "never")) { 948 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 949 &transparent_hugepage_flags); 950 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 951 &transparent_hugepage_flags); 952 ret = 1; 953 } 954 out: 955 if (!ret) 956 pr_warn("transparent_hugepage= cannot parse, ignored\n"); 957 return ret; 958 } 959 __setup("transparent_hugepage=", setup_transparent_hugepage); 960 961 static char str_dup[PAGE_SIZE] __initdata; 962 static int __init setup_thp_anon(char *str) 963 { 964 char *token, *range, *policy, *subtoken; 965 unsigned long always, inherit, madvise; 966 char *start_size, *end_size; 967 int start, end, nr; 968 char *p; 969 970 if (!str || strlen(str) + 1 > PAGE_SIZE) 971 goto err; 972 strscpy(str_dup, str); 973 974 always = huge_anon_orders_always; 975 madvise = huge_anon_orders_madvise; 976 inherit = huge_anon_orders_inherit; 977 p = str_dup; 978 while ((token = strsep(&p, ";")) != NULL) { 979 range = strsep(&token, ":"); 980 policy = token; 981 982 if (!policy) 983 goto err; 984 985 while ((subtoken = strsep(&range, ",")) != NULL) { 986 if (strchr(subtoken, '-')) { 987 start_size = strsep(&subtoken, "-"); 988 end_size = subtoken; 989 990 start = get_order_from_str(start_size, THP_ORDERS_ALL_ANON); 991 end = get_order_from_str(end_size, THP_ORDERS_ALL_ANON); 992 } else { 993 start_size = end_size = subtoken; 994 start = end = get_order_from_str(subtoken, 995 THP_ORDERS_ALL_ANON); 996 } 997 998 if (start == -EINVAL) { 999 pr_err("invalid size %s in thp_anon boot parameter\n", start_size); 1000 goto err; 1001 } 1002 1003 if (end == -EINVAL) { 1004 pr_err("invalid size %s in thp_anon boot parameter\n", end_size); 1005 goto err; 1006 } 1007 1008 if (start < 0 || end < 0 || start > end) 1009 goto err; 1010 1011 nr = end - start + 1; 1012 if (!strcmp(policy, "always")) { 1013 bitmap_set(&always, start, nr); 1014 bitmap_clear(&inherit, start, nr); 1015 bitmap_clear(&madvise, start, nr); 1016 } else if (!strcmp(policy, "madvise")) { 1017 bitmap_set(&madvise, start, nr); 1018 bitmap_clear(&inherit, start, nr); 1019 bitmap_clear(&always, start, nr); 1020 } else if (!strcmp(policy, "inherit")) { 1021 bitmap_set(&inherit, start, nr); 1022 bitmap_clear(&madvise, start, nr); 1023 bitmap_clear(&always, start, nr); 1024 } else if (!strcmp(policy, "never")) { 1025 bitmap_clear(&inherit, start, nr); 1026 bitmap_clear(&madvise, start, nr); 1027 bitmap_clear(&always, start, nr); 1028 } else { 1029 pr_err("invalid policy %s in thp_anon boot parameter\n", policy); 1030 goto err; 1031 } 1032 } 1033 } 1034 1035 huge_anon_orders_always = always; 1036 huge_anon_orders_madvise = madvise; 1037 huge_anon_orders_inherit = inherit; 1038 anon_orders_configured = true; 1039 return 1; 1040 1041 err: 1042 pr_warn("thp_anon=%s: error parsing string, ignoring setting\n", str); 1043 return 0; 1044 } 1045 __setup("thp_anon=", setup_thp_anon); 1046 1047 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 1048 { 1049 if (likely(vma->vm_flags & VM_WRITE)) 1050 pmd = pmd_mkwrite(pmd, vma); 1051 return pmd; 1052 } 1053 1054 #ifdef CONFIG_MEMCG 1055 static inline 1056 struct deferred_split *get_deferred_split_queue(struct folio *folio) 1057 { 1058 struct mem_cgroup *memcg = folio_memcg(folio); 1059 struct pglist_data *pgdat = NODE_DATA(folio_nid(folio)); 1060 1061 if (memcg) 1062 return &memcg->deferred_split_queue; 1063 else 1064 return &pgdat->deferred_split_queue; 1065 } 1066 #else 1067 static inline 1068 struct deferred_split *get_deferred_split_queue(struct folio *folio) 1069 { 1070 struct pglist_data *pgdat = NODE_DATA(folio_nid(folio)); 1071 1072 return &pgdat->deferred_split_queue; 1073 } 1074 #endif 1075 1076 static inline bool is_transparent_hugepage(const struct folio *folio) 1077 { 1078 if (!folio_test_large(folio)) 1079 return false; 1080 1081 return is_huge_zero_folio(folio) || 1082 folio_test_large_rmappable(folio); 1083 } 1084 1085 static unsigned long __thp_get_unmapped_area(struct file *filp, 1086 unsigned long addr, unsigned long len, 1087 loff_t off, unsigned long flags, unsigned long size, 1088 vm_flags_t vm_flags) 1089 { 1090 loff_t off_end = off + len; 1091 loff_t off_align = round_up(off, size); 1092 unsigned long len_pad, ret, off_sub; 1093 1094 if (!IS_ENABLED(CONFIG_64BIT) || in_compat_syscall()) 1095 return 0; 1096 1097 if (off_end <= off_align || (off_end - off_align) < size) 1098 return 0; 1099 1100 len_pad = len + size; 1101 if (len_pad < len || (off + len_pad) < off) 1102 return 0; 1103 1104 ret = mm_get_unmapped_area_vmflags(current->mm, filp, addr, len_pad, 1105 off >> PAGE_SHIFT, flags, vm_flags); 1106 1107 /* 1108 * The failure might be due to length padding. The caller will retry 1109 * without the padding. 1110 */ 1111 if (IS_ERR_VALUE(ret)) 1112 return 0; 1113 1114 /* 1115 * Do not try to align to THP boundary if allocation at the address 1116 * hint succeeds. 1117 */ 1118 if (ret == addr) 1119 return addr; 1120 1121 off_sub = (off - ret) & (size - 1); 1122 1123 if (test_bit(MMF_TOPDOWN, ¤t->mm->flags) && !off_sub) 1124 return ret + size; 1125 1126 ret += off_sub; 1127 return ret; 1128 } 1129 1130 unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, 1131 unsigned long len, unsigned long pgoff, unsigned long flags, 1132 vm_flags_t vm_flags) 1133 { 1134 unsigned long ret; 1135 loff_t off = (loff_t)pgoff << PAGE_SHIFT; 1136 1137 ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, vm_flags); 1138 if (ret) 1139 return ret; 1140 1141 return mm_get_unmapped_area_vmflags(current->mm, filp, addr, len, pgoff, flags, 1142 vm_flags); 1143 } 1144 1145 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, 1146 unsigned long len, unsigned long pgoff, unsigned long flags) 1147 { 1148 return thp_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0); 1149 } 1150 EXPORT_SYMBOL_GPL(thp_get_unmapped_area); 1151 1152 static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma, 1153 unsigned long addr) 1154 { 1155 gfp_t gfp = vma_thp_gfp_mask(vma); 1156 const int order = HPAGE_PMD_ORDER; 1157 struct folio *folio; 1158 1159 folio = vma_alloc_folio(gfp, order, vma, addr & HPAGE_PMD_MASK); 1160 1161 if (unlikely(!folio)) { 1162 count_vm_event(THP_FAULT_FALLBACK); 1163 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK); 1164 return NULL; 1165 } 1166 1167 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 1168 if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) { 1169 folio_put(folio); 1170 count_vm_event(THP_FAULT_FALLBACK); 1171 count_vm_event(THP_FAULT_FALLBACK_CHARGE); 1172 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK); 1173 count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); 1174 return NULL; 1175 } 1176 folio_throttle_swaprate(folio, gfp); 1177 1178 /* 1179 * When a folio is not zeroed during allocation (__GFP_ZERO not used) 1180 * or user folios require special handling, folio_zero_user() is used to 1181 * make sure that the page corresponding to the faulting address will be 1182 * hot in the cache after zeroing. 1183 */ 1184 if (user_alloc_needs_zeroing()) 1185 folio_zero_user(folio, addr); 1186 /* 1187 * The memory barrier inside __folio_mark_uptodate makes sure that 1188 * folio_zero_user writes become visible before the set_pmd_at() 1189 * write. 1190 */ 1191 __folio_mark_uptodate(folio); 1192 return folio; 1193 } 1194 1195 static void map_anon_folio_pmd(struct folio *folio, pmd_t *pmd, 1196 struct vm_area_struct *vma, unsigned long haddr) 1197 { 1198 pmd_t entry; 1199 1200 entry = mk_huge_pmd(&folio->page, vma->vm_page_prot); 1201 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1202 folio_add_new_anon_rmap(folio, vma, haddr, RMAP_EXCLUSIVE); 1203 folio_add_lru_vma(folio, vma); 1204 set_pmd_at(vma->vm_mm, haddr, pmd, entry); 1205 update_mmu_cache_pmd(vma, haddr, pmd); 1206 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1207 count_vm_event(THP_FAULT_ALLOC); 1208 count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_ALLOC); 1209 count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); 1210 } 1211 1212 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf) 1213 { 1214 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1215 struct vm_area_struct *vma = vmf->vma; 1216 struct folio *folio; 1217 pgtable_t pgtable; 1218 vm_fault_t ret = 0; 1219 1220 folio = vma_alloc_anon_folio_pmd(vma, vmf->address); 1221 if (unlikely(!folio)) 1222 return VM_FAULT_FALLBACK; 1223 1224 pgtable = pte_alloc_one(vma->vm_mm); 1225 if (unlikely(!pgtable)) { 1226 ret = VM_FAULT_OOM; 1227 goto release; 1228 } 1229 1230 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1231 if (unlikely(!pmd_none(*vmf->pmd))) { 1232 goto unlock_release; 1233 } else { 1234 ret = check_stable_address_space(vma->vm_mm); 1235 if (ret) 1236 goto unlock_release; 1237 1238 /* Deliver the page fault to userland */ 1239 if (userfaultfd_missing(vma)) { 1240 spin_unlock(vmf->ptl); 1241 folio_put(folio); 1242 pte_free(vma->vm_mm, pgtable); 1243 ret = handle_userfault(vmf, VM_UFFD_MISSING); 1244 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 1245 return ret; 1246 } 1247 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); 1248 map_anon_folio_pmd(folio, vmf->pmd, vma, haddr); 1249 mm_inc_nr_ptes(vma->vm_mm); 1250 deferred_split_folio(folio, false); 1251 spin_unlock(vmf->ptl); 1252 } 1253 1254 return 0; 1255 unlock_release: 1256 spin_unlock(vmf->ptl); 1257 release: 1258 if (pgtable) 1259 pte_free(vma->vm_mm, pgtable); 1260 folio_put(folio); 1261 return ret; 1262 1263 } 1264 1265 /* 1266 * always: directly stall for all thp allocations 1267 * defer: wake kswapd and fail if not immediately available 1268 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise 1269 * fail if not immediately available 1270 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately 1271 * available 1272 * never: never stall for any thp allocation 1273 */ 1274 gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma) 1275 { 1276 const bool vma_madvised = vma && (vma->vm_flags & VM_HUGEPAGE); 1277 1278 /* Always do synchronous compaction */ 1279 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) 1280 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY); 1281 1282 /* Kick kcompactd and fail quickly */ 1283 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) 1284 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM; 1285 1286 /* Synchronous compaction if madvised, otherwise kick kcompactd */ 1287 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags)) 1288 return GFP_TRANSHUGE_LIGHT | 1289 (vma_madvised ? __GFP_DIRECT_RECLAIM : 1290 __GFP_KSWAPD_RECLAIM); 1291 1292 /* Only do synchronous compaction if madvised */ 1293 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags)) 1294 return GFP_TRANSHUGE_LIGHT | 1295 (vma_madvised ? __GFP_DIRECT_RECLAIM : 0); 1296 1297 return GFP_TRANSHUGE_LIGHT; 1298 } 1299 1300 /* Caller must hold page table lock. */ 1301 static void set_huge_zero_folio(pgtable_t pgtable, struct mm_struct *mm, 1302 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, 1303 struct folio *zero_folio) 1304 { 1305 pmd_t entry; 1306 if (!pmd_none(*pmd)) 1307 return; 1308 entry = mk_pmd(&zero_folio->page, vma->vm_page_prot); 1309 entry = pmd_mkhuge(entry); 1310 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1311 set_pmd_at(mm, haddr, pmd, entry); 1312 mm_inc_nr_ptes(mm); 1313 } 1314 1315 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) 1316 { 1317 struct vm_area_struct *vma = vmf->vma; 1318 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1319 vm_fault_t ret; 1320 1321 if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER)) 1322 return VM_FAULT_FALLBACK; 1323 ret = vmf_anon_prepare(vmf); 1324 if (ret) 1325 return ret; 1326 khugepaged_enter_vma(vma, vma->vm_flags); 1327 1328 if (!(vmf->flags & FAULT_FLAG_WRITE) && 1329 !mm_forbids_zeropage(vma->vm_mm) && 1330 transparent_hugepage_use_zero_page()) { 1331 pgtable_t pgtable; 1332 struct folio *zero_folio; 1333 vm_fault_t ret; 1334 1335 pgtable = pte_alloc_one(vma->vm_mm); 1336 if (unlikely(!pgtable)) 1337 return VM_FAULT_OOM; 1338 zero_folio = mm_get_huge_zero_folio(vma->vm_mm); 1339 if (unlikely(!zero_folio)) { 1340 pte_free(vma->vm_mm, pgtable); 1341 count_vm_event(THP_FAULT_FALLBACK); 1342 return VM_FAULT_FALLBACK; 1343 } 1344 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1345 ret = 0; 1346 if (pmd_none(*vmf->pmd)) { 1347 ret = check_stable_address_space(vma->vm_mm); 1348 if (ret) { 1349 spin_unlock(vmf->ptl); 1350 pte_free(vma->vm_mm, pgtable); 1351 } else if (userfaultfd_missing(vma)) { 1352 spin_unlock(vmf->ptl); 1353 pte_free(vma->vm_mm, pgtable); 1354 ret = handle_userfault(vmf, VM_UFFD_MISSING); 1355 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 1356 } else { 1357 set_huge_zero_folio(pgtable, vma->vm_mm, vma, 1358 haddr, vmf->pmd, zero_folio); 1359 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1360 spin_unlock(vmf->ptl); 1361 } 1362 } else { 1363 spin_unlock(vmf->ptl); 1364 pte_free(vma->vm_mm, pgtable); 1365 } 1366 return ret; 1367 } 1368 1369 return __do_huge_pmd_anonymous_page(vmf); 1370 } 1371 1372 static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 1373 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write, 1374 pgtable_t pgtable) 1375 { 1376 struct mm_struct *mm = vma->vm_mm; 1377 pmd_t entry; 1378 spinlock_t *ptl; 1379 1380 ptl = pmd_lock(mm, pmd); 1381 if (!pmd_none(*pmd)) { 1382 if (write) { 1383 if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) { 1384 WARN_ON_ONCE(!is_huge_zero_pmd(*pmd)); 1385 goto out_unlock; 1386 } 1387 entry = pmd_mkyoung(*pmd); 1388 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1389 if (pmdp_set_access_flags(vma, addr, pmd, entry, 1)) 1390 update_mmu_cache_pmd(vma, addr, pmd); 1391 } 1392 1393 goto out_unlock; 1394 } 1395 1396 entry = pmd_mkhuge(pfn_t_pmd(pfn, prot)); 1397 if (pfn_t_devmap(pfn)) 1398 entry = pmd_mkdevmap(entry); 1399 else 1400 entry = pmd_mkspecial(entry); 1401 if (write) { 1402 entry = pmd_mkyoung(pmd_mkdirty(entry)); 1403 entry = maybe_pmd_mkwrite(entry, vma); 1404 } 1405 1406 if (pgtable) { 1407 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1408 mm_inc_nr_ptes(mm); 1409 pgtable = NULL; 1410 } 1411 1412 set_pmd_at(mm, addr, pmd, entry); 1413 update_mmu_cache_pmd(vma, addr, pmd); 1414 1415 out_unlock: 1416 spin_unlock(ptl); 1417 if (pgtable) 1418 pte_free(mm, pgtable); 1419 } 1420 1421 /** 1422 * vmf_insert_pfn_pmd - insert a pmd size pfn 1423 * @vmf: Structure describing the fault 1424 * @pfn: pfn to insert 1425 * @write: whether it's a write fault 1426 * 1427 * Insert a pmd size pfn. See vmf_insert_pfn() for additional info. 1428 * 1429 * Return: vm_fault_t value. 1430 */ 1431 vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write) 1432 { 1433 unsigned long addr = vmf->address & PMD_MASK; 1434 struct vm_area_struct *vma = vmf->vma; 1435 pgprot_t pgprot = vma->vm_page_prot; 1436 pgtable_t pgtable = NULL; 1437 1438 /* 1439 * If we had pmd_special, we could avoid all these restrictions, 1440 * but we need to be consistent with PTEs and architectures that 1441 * can't support a 'special' bit. 1442 */ 1443 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && 1444 !pfn_t_devmap(pfn)); 1445 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1446 (VM_PFNMAP|VM_MIXEDMAP)); 1447 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1448 1449 if (addr < vma->vm_start || addr >= vma->vm_end) 1450 return VM_FAULT_SIGBUS; 1451 1452 if (arch_needs_pgtable_deposit()) { 1453 pgtable = pte_alloc_one(vma->vm_mm); 1454 if (!pgtable) 1455 return VM_FAULT_OOM; 1456 } 1457 1458 track_pfn_insert(vma, &pgprot, pfn); 1459 1460 insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable); 1461 return VM_FAULT_NOPAGE; 1462 } 1463 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd); 1464 1465 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1466 static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma) 1467 { 1468 if (likely(vma->vm_flags & VM_WRITE)) 1469 pud = pud_mkwrite(pud); 1470 return pud; 1471 } 1472 1473 static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr, 1474 pud_t *pud, pfn_t pfn, bool write) 1475 { 1476 struct mm_struct *mm = vma->vm_mm; 1477 pgprot_t prot = vma->vm_page_prot; 1478 pud_t entry; 1479 spinlock_t *ptl; 1480 1481 ptl = pud_lock(mm, pud); 1482 if (!pud_none(*pud)) { 1483 if (write) { 1484 if (WARN_ON_ONCE(pud_pfn(*pud) != pfn_t_to_pfn(pfn))) 1485 goto out_unlock; 1486 entry = pud_mkyoung(*pud); 1487 entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma); 1488 if (pudp_set_access_flags(vma, addr, pud, entry, 1)) 1489 update_mmu_cache_pud(vma, addr, pud); 1490 } 1491 goto out_unlock; 1492 } 1493 1494 entry = pud_mkhuge(pfn_t_pud(pfn, prot)); 1495 if (pfn_t_devmap(pfn)) 1496 entry = pud_mkdevmap(entry); 1497 else 1498 entry = pud_mkspecial(entry); 1499 if (write) { 1500 entry = pud_mkyoung(pud_mkdirty(entry)); 1501 entry = maybe_pud_mkwrite(entry, vma); 1502 } 1503 set_pud_at(mm, addr, pud, entry); 1504 update_mmu_cache_pud(vma, addr, pud); 1505 1506 out_unlock: 1507 spin_unlock(ptl); 1508 } 1509 1510 /** 1511 * vmf_insert_pfn_pud - insert a pud size pfn 1512 * @vmf: Structure describing the fault 1513 * @pfn: pfn to insert 1514 * @write: whether it's a write fault 1515 * 1516 * Insert a pud size pfn. See vmf_insert_pfn() for additional info. 1517 * 1518 * Return: vm_fault_t value. 1519 */ 1520 vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write) 1521 { 1522 unsigned long addr = vmf->address & PUD_MASK; 1523 struct vm_area_struct *vma = vmf->vma; 1524 pgprot_t pgprot = vma->vm_page_prot; 1525 1526 /* 1527 * If we had pud_special, we could avoid all these restrictions, 1528 * but we need to be consistent with PTEs and architectures that 1529 * can't support a 'special' bit. 1530 */ 1531 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && 1532 !pfn_t_devmap(pfn)); 1533 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1534 (VM_PFNMAP|VM_MIXEDMAP)); 1535 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1536 1537 if (addr < vma->vm_start || addr >= vma->vm_end) 1538 return VM_FAULT_SIGBUS; 1539 1540 track_pfn_insert(vma, &pgprot, pfn); 1541 1542 insert_pfn_pud(vma, addr, vmf->pud, pfn, write); 1543 return VM_FAULT_NOPAGE; 1544 } 1545 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud); 1546 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1547 1548 void touch_pmd(struct vm_area_struct *vma, unsigned long addr, 1549 pmd_t *pmd, bool write) 1550 { 1551 pmd_t _pmd; 1552 1553 _pmd = pmd_mkyoung(*pmd); 1554 if (write) 1555 _pmd = pmd_mkdirty(_pmd); 1556 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, 1557 pmd, _pmd, write)) 1558 update_mmu_cache_pmd(vma, addr, pmd); 1559 } 1560 1561 struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, 1562 pmd_t *pmd, int flags, struct dev_pagemap **pgmap) 1563 { 1564 unsigned long pfn = pmd_pfn(*pmd); 1565 struct mm_struct *mm = vma->vm_mm; 1566 struct page *page; 1567 int ret; 1568 1569 assert_spin_locked(pmd_lockptr(mm, pmd)); 1570 1571 if (flags & FOLL_WRITE && !pmd_write(*pmd)) 1572 return NULL; 1573 1574 if (pmd_present(*pmd) && pmd_devmap(*pmd)) 1575 /* pass */; 1576 else 1577 return NULL; 1578 1579 if (flags & FOLL_TOUCH) 1580 touch_pmd(vma, addr, pmd, flags & FOLL_WRITE); 1581 1582 /* 1583 * device mapped pages can only be returned if the 1584 * caller will manage the page reference count. 1585 */ 1586 if (!(flags & (FOLL_GET | FOLL_PIN))) 1587 return ERR_PTR(-EEXIST); 1588 1589 pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT; 1590 *pgmap = get_dev_pagemap(pfn, *pgmap); 1591 if (!*pgmap) 1592 return ERR_PTR(-EFAULT); 1593 page = pfn_to_page(pfn); 1594 ret = try_grab_folio(page_folio(page), 1, flags); 1595 if (ret) 1596 page = ERR_PTR(ret); 1597 1598 return page; 1599 } 1600 1601 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1602 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 1603 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) 1604 { 1605 spinlock_t *dst_ptl, *src_ptl; 1606 struct page *src_page; 1607 struct folio *src_folio; 1608 pmd_t pmd; 1609 pgtable_t pgtable = NULL; 1610 int ret = -ENOMEM; 1611 1612 pmd = pmdp_get_lockless(src_pmd); 1613 if (unlikely(pmd_present(pmd) && pmd_special(pmd))) { 1614 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1615 src_ptl = pmd_lockptr(src_mm, src_pmd); 1616 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1617 /* 1618 * No need to recheck the pmd, it can't change with write 1619 * mmap lock held here. 1620 * 1621 * Meanwhile, making sure it's not a CoW VMA with writable 1622 * mapping, otherwise it means either the anon page wrongly 1623 * applied special bit, or we made the PRIVATE mapping be 1624 * able to wrongly write to the backend MMIO. 1625 */ 1626 VM_WARN_ON_ONCE(is_cow_mapping(src_vma->vm_flags) && pmd_write(pmd)); 1627 goto set_pmd; 1628 } 1629 1630 /* Skip if can be re-fill on fault */ 1631 if (!vma_is_anonymous(dst_vma)) 1632 return 0; 1633 1634 pgtable = pte_alloc_one(dst_mm); 1635 if (unlikely(!pgtable)) 1636 goto out; 1637 1638 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1639 src_ptl = pmd_lockptr(src_mm, src_pmd); 1640 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1641 1642 ret = -EAGAIN; 1643 pmd = *src_pmd; 1644 1645 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 1646 if (unlikely(is_swap_pmd(pmd))) { 1647 swp_entry_t entry = pmd_to_swp_entry(pmd); 1648 1649 VM_BUG_ON(!is_pmd_migration_entry(pmd)); 1650 if (!is_readable_migration_entry(entry)) { 1651 entry = make_readable_migration_entry( 1652 swp_offset(entry)); 1653 pmd = swp_entry_to_pmd(entry); 1654 if (pmd_swp_soft_dirty(*src_pmd)) 1655 pmd = pmd_swp_mksoft_dirty(pmd); 1656 if (pmd_swp_uffd_wp(*src_pmd)) 1657 pmd = pmd_swp_mkuffd_wp(pmd); 1658 set_pmd_at(src_mm, addr, src_pmd, pmd); 1659 } 1660 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1661 mm_inc_nr_ptes(dst_mm); 1662 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1663 if (!userfaultfd_wp(dst_vma)) 1664 pmd = pmd_swp_clear_uffd_wp(pmd); 1665 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1666 ret = 0; 1667 goto out_unlock; 1668 } 1669 #endif 1670 1671 if (unlikely(!pmd_trans_huge(pmd))) { 1672 pte_free(dst_mm, pgtable); 1673 goto out_unlock; 1674 } 1675 /* 1676 * When page table lock is held, the huge zero pmd should not be 1677 * under splitting since we don't split the page itself, only pmd to 1678 * a page table. 1679 */ 1680 if (is_huge_zero_pmd(pmd)) { 1681 /* 1682 * mm_get_huge_zero_folio() will never allocate a new 1683 * folio here, since we already have a zero page to 1684 * copy. It just takes a reference. 1685 */ 1686 mm_get_huge_zero_folio(dst_mm); 1687 goto out_zero_page; 1688 } 1689 1690 src_page = pmd_page(pmd); 1691 VM_BUG_ON_PAGE(!PageHead(src_page), src_page); 1692 src_folio = page_folio(src_page); 1693 1694 folio_get(src_folio); 1695 if (unlikely(folio_try_dup_anon_rmap_pmd(src_folio, src_page, src_vma))) { 1696 /* Page maybe pinned: split and retry the fault on PTEs. */ 1697 folio_put(src_folio); 1698 pte_free(dst_mm, pgtable); 1699 spin_unlock(src_ptl); 1700 spin_unlock(dst_ptl); 1701 __split_huge_pmd(src_vma, src_pmd, addr, false, NULL); 1702 return -EAGAIN; 1703 } 1704 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1705 out_zero_page: 1706 mm_inc_nr_ptes(dst_mm); 1707 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1708 pmdp_set_wrprotect(src_mm, addr, src_pmd); 1709 if (!userfaultfd_wp(dst_vma)) 1710 pmd = pmd_clear_uffd_wp(pmd); 1711 pmd = pmd_wrprotect(pmd); 1712 set_pmd: 1713 pmd = pmd_mkold(pmd); 1714 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1715 1716 ret = 0; 1717 out_unlock: 1718 spin_unlock(src_ptl); 1719 spin_unlock(dst_ptl); 1720 out: 1721 return ret; 1722 } 1723 1724 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1725 void touch_pud(struct vm_area_struct *vma, unsigned long addr, 1726 pud_t *pud, bool write) 1727 { 1728 pud_t _pud; 1729 1730 _pud = pud_mkyoung(*pud); 1731 if (write) 1732 _pud = pud_mkdirty(_pud); 1733 if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK, 1734 pud, _pud, write)) 1735 update_mmu_cache_pud(vma, addr, pud); 1736 } 1737 1738 int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1739 pud_t *dst_pud, pud_t *src_pud, unsigned long addr, 1740 struct vm_area_struct *vma) 1741 { 1742 spinlock_t *dst_ptl, *src_ptl; 1743 pud_t pud; 1744 int ret; 1745 1746 dst_ptl = pud_lock(dst_mm, dst_pud); 1747 src_ptl = pud_lockptr(src_mm, src_pud); 1748 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1749 1750 ret = -EAGAIN; 1751 pud = *src_pud; 1752 if (unlikely(!pud_trans_huge(pud) && !pud_devmap(pud))) 1753 goto out_unlock; 1754 1755 /* 1756 * TODO: once we support anonymous pages, use 1757 * folio_try_dup_anon_rmap_*() and split if duplicating fails. 1758 */ 1759 if (is_cow_mapping(vma->vm_flags) && pud_write(pud)) { 1760 pudp_set_wrprotect(src_mm, addr, src_pud); 1761 pud = pud_wrprotect(pud); 1762 } 1763 pud = pud_mkold(pud); 1764 set_pud_at(dst_mm, addr, dst_pud, pud); 1765 1766 ret = 0; 1767 out_unlock: 1768 spin_unlock(src_ptl); 1769 spin_unlock(dst_ptl); 1770 return ret; 1771 } 1772 1773 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud) 1774 { 1775 bool write = vmf->flags & FAULT_FLAG_WRITE; 1776 1777 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); 1778 if (unlikely(!pud_same(*vmf->pud, orig_pud))) 1779 goto unlock; 1780 1781 touch_pud(vmf->vma, vmf->address, vmf->pud, write); 1782 unlock: 1783 spin_unlock(vmf->ptl); 1784 } 1785 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1786 1787 void huge_pmd_set_accessed(struct vm_fault *vmf) 1788 { 1789 bool write = vmf->flags & FAULT_FLAG_WRITE; 1790 1791 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); 1792 if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd))) 1793 goto unlock; 1794 1795 touch_pmd(vmf->vma, vmf->address, vmf->pmd, write); 1796 1797 unlock: 1798 spin_unlock(vmf->ptl); 1799 } 1800 1801 static vm_fault_t do_huge_zero_wp_pmd(struct vm_fault *vmf) 1802 { 1803 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1804 struct vm_area_struct *vma = vmf->vma; 1805 struct mmu_notifier_range range; 1806 struct folio *folio; 1807 vm_fault_t ret = 0; 1808 1809 folio = vma_alloc_anon_folio_pmd(vma, vmf->address); 1810 if (unlikely(!folio)) 1811 return VM_FAULT_FALLBACK; 1812 1813 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, haddr, 1814 haddr + HPAGE_PMD_SIZE); 1815 mmu_notifier_invalidate_range_start(&range); 1816 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1817 if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) 1818 goto release; 1819 ret = check_stable_address_space(vma->vm_mm); 1820 if (ret) 1821 goto release; 1822 (void)pmdp_huge_clear_flush(vma, haddr, vmf->pmd); 1823 map_anon_folio_pmd(folio, vmf->pmd, vma, haddr); 1824 goto unlock; 1825 release: 1826 folio_put(folio); 1827 unlock: 1828 spin_unlock(vmf->ptl); 1829 mmu_notifier_invalidate_range_end(&range); 1830 return ret; 1831 } 1832 1833 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) 1834 { 1835 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; 1836 struct vm_area_struct *vma = vmf->vma; 1837 struct folio *folio; 1838 struct page *page; 1839 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1840 pmd_t orig_pmd = vmf->orig_pmd; 1841 1842 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); 1843 VM_BUG_ON_VMA(!vma->anon_vma, vma); 1844 1845 if (is_huge_zero_pmd(orig_pmd)) { 1846 vm_fault_t ret = do_huge_zero_wp_pmd(vmf); 1847 1848 if (!(ret & VM_FAULT_FALLBACK)) 1849 return ret; 1850 1851 /* Fallback to splitting PMD if THP cannot be allocated */ 1852 goto fallback; 1853 } 1854 1855 spin_lock(vmf->ptl); 1856 1857 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1858 spin_unlock(vmf->ptl); 1859 return 0; 1860 } 1861 1862 page = pmd_page(orig_pmd); 1863 folio = page_folio(page); 1864 VM_BUG_ON_PAGE(!PageHead(page), page); 1865 1866 /* Early check when only holding the PT lock. */ 1867 if (PageAnonExclusive(page)) 1868 goto reuse; 1869 1870 if (!folio_trylock(folio)) { 1871 folio_get(folio); 1872 spin_unlock(vmf->ptl); 1873 folio_lock(folio); 1874 spin_lock(vmf->ptl); 1875 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1876 spin_unlock(vmf->ptl); 1877 folio_unlock(folio); 1878 folio_put(folio); 1879 return 0; 1880 } 1881 folio_put(folio); 1882 } 1883 1884 /* Recheck after temporarily dropping the PT lock. */ 1885 if (PageAnonExclusive(page)) { 1886 folio_unlock(folio); 1887 goto reuse; 1888 } 1889 1890 /* 1891 * See do_wp_page(): we can only reuse the folio exclusively if 1892 * there are no additional references. Note that we always drain 1893 * the LRU cache immediately after adding a THP. 1894 */ 1895 if (folio_ref_count(folio) > 1896 1 + folio_test_swapcache(folio) * folio_nr_pages(folio)) 1897 goto unlock_fallback; 1898 if (folio_test_swapcache(folio)) 1899 folio_free_swap(folio); 1900 if (folio_ref_count(folio) == 1) { 1901 pmd_t entry; 1902 1903 folio_move_anon_rmap(folio, vma); 1904 SetPageAnonExclusive(page); 1905 folio_unlock(folio); 1906 reuse: 1907 if (unlikely(unshare)) { 1908 spin_unlock(vmf->ptl); 1909 return 0; 1910 } 1911 entry = pmd_mkyoung(orig_pmd); 1912 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1913 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) 1914 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1915 spin_unlock(vmf->ptl); 1916 return 0; 1917 } 1918 1919 unlock_fallback: 1920 folio_unlock(folio); 1921 spin_unlock(vmf->ptl); 1922 fallback: 1923 __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL); 1924 return VM_FAULT_FALLBACK; 1925 } 1926 1927 static inline bool can_change_pmd_writable(struct vm_area_struct *vma, 1928 unsigned long addr, pmd_t pmd) 1929 { 1930 struct page *page; 1931 1932 if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) 1933 return false; 1934 1935 /* Don't touch entries that are not even readable (NUMA hinting). */ 1936 if (pmd_protnone(pmd)) 1937 return false; 1938 1939 /* Do we need write faults for softdirty tracking? */ 1940 if (pmd_needs_soft_dirty_wp(vma, pmd)) 1941 return false; 1942 1943 /* Do we need write faults for uffd-wp tracking? */ 1944 if (userfaultfd_huge_pmd_wp(vma, pmd)) 1945 return false; 1946 1947 if (!(vma->vm_flags & VM_SHARED)) { 1948 /* See can_change_pte_writable(). */ 1949 page = vm_normal_page_pmd(vma, addr, pmd); 1950 return page && PageAnon(page) && PageAnonExclusive(page); 1951 } 1952 1953 /* See can_change_pte_writable(). */ 1954 return pmd_dirty(pmd); 1955 } 1956 1957 /* NUMA hinting page fault entry point for trans huge pmds */ 1958 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) 1959 { 1960 struct vm_area_struct *vma = vmf->vma; 1961 struct folio *folio; 1962 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1963 int nid = NUMA_NO_NODE; 1964 int target_nid, last_cpupid; 1965 pmd_t pmd, old_pmd; 1966 bool writable = false; 1967 int flags = 0; 1968 1969 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1970 old_pmd = pmdp_get(vmf->pmd); 1971 1972 if (unlikely(!pmd_same(old_pmd, vmf->orig_pmd))) { 1973 spin_unlock(vmf->ptl); 1974 return 0; 1975 } 1976 1977 pmd = pmd_modify(old_pmd, vma->vm_page_prot); 1978 1979 /* 1980 * Detect now whether the PMD could be writable; this information 1981 * is only valid while holding the PT lock. 1982 */ 1983 writable = pmd_write(pmd); 1984 if (!writable && vma_wants_manual_pte_write_upgrade(vma) && 1985 can_change_pmd_writable(vma, vmf->address, pmd)) 1986 writable = true; 1987 1988 folio = vm_normal_folio_pmd(vma, haddr, pmd); 1989 if (!folio) 1990 goto out_map; 1991 1992 nid = folio_nid(folio); 1993 1994 target_nid = numa_migrate_check(folio, vmf, haddr, &flags, writable, 1995 &last_cpupid); 1996 if (target_nid == NUMA_NO_NODE) 1997 goto out_map; 1998 if (migrate_misplaced_folio_prepare(folio, vma, target_nid)) { 1999 flags |= TNF_MIGRATE_FAIL; 2000 goto out_map; 2001 } 2002 /* The folio is isolated and isolation code holds a folio reference. */ 2003 spin_unlock(vmf->ptl); 2004 writable = false; 2005 2006 if (!migrate_misplaced_folio(folio, vma, target_nid)) { 2007 flags |= TNF_MIGRATED; 2008 nid = target_nid; 2009 task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags); 2010 return 0; 2011 } 2012 2013 flags |= TNF_MIGRATE_FAIL; 2014 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 2015 if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) { 2016 spin_unlock(vmf->ptl); 2017 return 0; 2018 } 2019 out_map: 2020 /* Restore the PMD */ 2021 pmd = pmd_modify(pmdp_get(vmf->pmd), vma->vm_page_prot); 2022 pmd = pmd_mkyoung(pmd); 2023 if (writable) 2024 pmd = pmd_mkwrite(pmd, vma); 2025 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); 2026 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 2027 spin_unlock(vmf->ptl); 2028 2029 if (nid != NUMA_NO_NODE) 2030 task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags); 2031 return 0; 2032 } 2033 2034 /* 2035 * Return true if we do MADV_FREE successfully on entire pmd page. 2036 * Otherwise, return false. 2037 */ 2038 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2039 pmd_t *pmd, unsigned long addr, unsigned long next) 2040 { 2041 spinlock_t *ptl; 2042 pmd_t orig_pmd; 2043 struct folio *folio; 2044 struct mm_struct *mm = tlb->mm; 2045 bool ret = false; 2046 2047 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2048 2049 ptl = pmd_trans_huge_lock(pmd, vma); 2050 if (!ptl) 2051 goto out_unlocked; 2052 2053 orig_pmd = *pmd; 2054 if (is_huge_zero_pmd(orig_pmd)) 2055 goto out; 2056 2057 if (unlikely(!pmd_present(orig_pmd))) { 2058 VM_BUG_ON(thp_migration_supported() && 2059 !is_pmd_migration_entry(orig_pmd)); 2060 goto out; 2061 } 2062 2063 folio = pmd_folio(orig_pmd); 2064 /* 2065 * If other processes are mapping this folio, we couldn't discard 2066 * the folio unless they all do MADV_FREE so let's skip the folio. 2067 */ 2068 if (folio_likely_mapped_shared(folio)) 2069 goto out; 2070 2071 if (!folio_trylock(folio)) 2072 goto out; 2073 2074 /* 2075 * If user want to discard part-pages of THP, split it so MADV_FREE 2076 * will deactivate only them. 2077 */ 2078 if (next - addr != HPAGE_PMD_SIZE) { 2079 folio_get(folio); 2080 spin_unlock(ptl); 2081 split_folio(folio); 2082 folio_unlock(folio); 2083 folio_put(folio); 2084 goto out_unlocked; 2085 } 2086 2087 if (folio_test_dirty(folio)) 2088 folio_clear_dirty(folio); 2089 folio_unlock(folio); 2090 2091 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) { 2092 pmdp_invalidate(vma, addr, pmd); 2093 orig_pmd = pmd_mkold(orig_pmd); 2094 orig_pmd = pmd_mkclean(orig_pmd); 2095 2096 set_pmd_at(mm, addr, pmd, orig_pmd); 2097 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 2098 } 2099 2100 folio_mark_lazyfree(folio); 2101 ret = true; 2102 out: 2103 spin_unlock(ptl); 2104 out_unlocked: 2105 return ret; 2106 } 2107 2108 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd) 2109 { 2110 pgtable_t pgtable; 2111 2112 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2113 pte_free(mm, pgtable); 2114 mm_dec_nr_ptes(mm); 2115 } 2116 2117 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2118 pmd_t *pmd, unsigned long addr) 2119 { 2120 pmd_t orig_pmd; 2121 spinlock_t *ptl; 2122 2123 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2124 2125 ptl = __pmd_trans_huge_lock(pmd, vma); 2126 if (!ptl) 2127 return 0; 2128 /* 2129 * For architectures like ppc64 we look at deposited pgtable 2130 * when calling pmdp_huge_get_and_clear. So do the 2131 * pgtable_trans_huge_withdraw after finishing pmdp related 2132 * operations. 2133 */ 2134 orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd, 2135 tlb->fullmm); 2136 arch_check_zapped_pmd(vma, orig_pmd); 2137 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 2138 if (vma_is_special_huge(vma)) { 2139 if (arch_needs_pgtable_deposit()) 2140 zap_deposited_table(tlb->mm, pmd); 2141 spin_unlock(ptl); 2142 } else if (is_huge_zero_pmd(orig_pmd)) { 2143 zap_deposited_table(tlb->mm, pmd); 2144 spin_unlock(ptl); 2145 } else { 2146 struct folio *folio = NULL; 2147 int flush_needed = 1; 2148 2149 if (pmd_present(orig_pmd)) { 2150 struct page *page = pmd_page(orig_pmd); 2151 2152 folio = page_folio(page); 2153 folio_remove_rmap_pmd(folio, page, vma); 2154 WARN_ON_ONCE(folio_mapcount(folio) < 0); 2155 VM_BUG_ON_PAGE(!PageHead(page), page); 2156 } else if (thp_migration_supported()) { 2157 swp_entry_t entry; 2158 2159 VM_BUG_ON(!is_pmd_migration_entry(orig_pmd)); 2160 entry = pmd_to_swp_entry(orig_pmd); 2161 folio = pfn_swap_entry_folio(entry); 2162 flush_needed = 0; 2163 } else 2164 WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!"); 2165 2166 if (folio_test_anon(folio)) { 2167 zap_deposited_table(tlb->mm, pmd); 2168 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); 2169 } else { 2170 if (arch_needs_pgtable_deposit()) 2171 zap_deposited_table(tlb->mm, pmd); 2172 add_mm_counter(tlb->mm, mm_counter_file(folio), 2173 -HPAGE_PMD_NR); 2174 } 2175 2176 spin_unlock(ptl); 2177 if (flush_needed) 2178 tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE); 2179 } 2180 return 1; 2181 } 2182 2183 #ifndef pmd_move_must_withdraw 2184 static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl, 2185 spinlock_t *old_pmd_ptl, 2186 struct vm_area_struct *vma) 2187 { 2188 /* 2189 * With split pmd lock we also need to move preallocated 2190 * PTE page table if new_pmd is on different PMD page table. 2191 * 2192 * We also don't deposit and withdraw tables for file pages. 2193 */ 2194 return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma); 2195 } 2196 #endif 2197 2198 static pmd_t move_soft_dirty_pmd(pmd_t pmd) 2199 { 2200 #ifdef CONFIG_MEM_SOFT_DIRTY 2201 if (unlikely(is_pmd_migration_entry(pmd))) 2202 pmd = pmd_swp_mksoft_dirty(pmd); 2203 else if (pmd_present(pmd)) 2204 pmd = pmd_mksoft_dirty(pmd); 2205 #endif 2206 return pmd; 2207 } 2208 2209 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, 2210 unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd) 2211 { 2212 spinlock_t *old_ptl, *new_ptl; 2213 pmd_t pmd; 2214 struct mm_struct *mm = vma->vm_mm; 2215 bool force_flush = false; 2216 2217 /* 2218 * The destination pmd shouldn't be established, free_pgtables() 2219 * should have released it; but move_page_tables() might have already 2220 * inserted a page table, if racing against shmem/file collapse. 2221 */ 2222 if (!pmd_none(*new_pmd)) { 2223 VM_BUG_ON(pmd_trans_huge(*new_pmd)); 2224 return false; 2225 } 2226 2227 /* 2228 * We don't have to worry about the ordering of src and dst 2229 * ptlocks because exclusive mmap_lock prevents deadlock. 2230 */ 2231 old_ptl = __pmd_trans_huge_lock(old_pmd, vma); 2232 if (old_ptl) { 2233 new_ptl = pmd_lockptr(mm, new_pmd); 2234 if (new_ptl != old_ptl) 2235 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 2236 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); 2237 if (pmd_present(pmd)) 2238 force_flush = true; 2239 VM_BUG_ON(!pmd_none(*new_pmd)); 2240 2241 if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) { 2242 pgtable_t pgtable; 2243 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); 2244 pgtable_trans_huge_deposit(mm, new_pmd, pgtable); 2245 } 2246 pmd = move_soft_dirty_pmd(pmd); 2247 set_pmd_at(mm, new_addr, new_pmd, pmd); 2248 if (force_flush) 2249 flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE); 2250 if (new_ptl != old_ptl) 2251 spin_unlock(new_ptl); 2252 spin_unlock(old_ptl); 2253 return true; 2254 } 2255 return false; 2256 } 2257 2258 /* 2259 * Returns 2260 * - 0 if PMD could not be locked 2261 * - 1 if PMD was locked but protections unchanged and TLB flush unnecessary 2262 * or if prot_numa but THP migration is not supported 2263 * - HPAGE_PMD_NR if protections changed and TLB flush necessary 2264 */ 2265 int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2266 pmd_t *pmd, unsigned long addr, pgprot_t newprot, 2267 unsigned long cp_flags) 2268 { 2269 struct mm_struct *mm = vma->vm_mm; 2270 spinlock_t *ptl; 2271 pmd_t oldpmd, entry; 2272 bool prot_numa = cp_flags & MM_CP_PROT_NUMA; 2273 bool uffd_wp = cp_flags & MM_CP_UFFD_WP; 2274 bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; 2275 int ret = 1; 2276 2277 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2278 2279 if (prot_numa && !thp_migration_supported()) 2280 return 1; 2281 2282 ptl = __pmd_trans_huge_lock(pmd, vma); 2283 if (!ptl) 2284 return 0; 2285 2286 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 2287 if (is_swap_pmd(*pmd)) { 2288 swp_entry_t entry = pmd_to_swp_entry(*pmd); 2289 struct folio *folio = pfn_swap_entry_folio(entry); 2290 pmd_t newpmd; 2291 2292 VM_BUG_ON(!is_pmd_migration_entry(*pmd)); 2293 if (is_writable_migration_entry(entry)) { 2294 /* 2295 * A protection check is difficult so 2296 * just be safe and disable write 2297 */ 2298 if (folio_test_anon(folio)) 2299 entry = make_readable_exclusive_migration_entry(swp_offset(entry)); 2300 else 2301 entry = make_readable_migration_entry(swp_offset(entry)); 2302 newpmd = swp_entry_to_pmd(entry); 2303 if (pmd_swp_soft_dirty(*pmd)) 2304 newpmd = pmd_swp_mksoft_dirty(newpmd); 2305 } else { 2306 newpmd = *pmd; 2307 } 2308 2309 if (uffd_wp) 2310 newpmd = pmd_swp_mkuffd_wp(newpmd); 2311 else if (uffd_wp_resolve) 2312 newpmd = pmd_swp_clear_uffd_wp(newpmd); 2313 if (!pmd_same(*pmd, newpmd)) 2314 set_pmd_at(mm, addr, pmd, newpmd); 2315 goto unlock; 2316 } 2317 #endif 2318 2319 if (prot_numa) { 2320 struct folio *folio; 2321 bool toptier; 2322 /* 2323 * Avoid trapping faults against the zero page. The read-only 2324 * data is likely to be read-cached on the local CPU and 2325 * local/remote hits to the zero page are not interesting. 2326 */ 2327 if (is_huge_zero_pmd(*pmd)) 2328 goto unlock; 2329 2330 if (pmd_protnone(*pmd)) 2331 goto unlock; 2332 2333 folio = pmd_folio(*pmd); 2334 toptier = node_is_toptier(folio_nid(folio)); 2335 /* 2336 * Skip scanning top tier node if normal numa 2337 * balancing is disabled 2338 */ 2339 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) && 2340 toptier) 2341 goto unlock; 2342 2343 if (folio_use_access_time(folio)) 2344 folio_xchg_access_time(folio, 2345 jiffies_to_msecs(jiffies)); 2346 } 2347 /* 2348 * In case prot_numa, we are under mmap_read_lock(mm). It's critical 2349 * to not clear pmd intermittently to avoid race with MADV_DONTNEED 2350 * which is also under mmap_read_lock(mm): 2351 * 2352 * CPU0: CPU1: 2353 * change_huge_pmd(prot_numa=1) 2354 * pmdp_huge_get_and_clear_notify() 2355 * madvise_dontneed() 2356 * zap_pmd_range() 2357 * pmd_trans_huge(*pmd) == 0 (without ptl) 2358 * // skip the pmd 2359 * set_pmd_at(); 2360 * // pmd is re-established 2361 * 2362 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it 2363 * which may break userspace. 2364 * 2365 * pmdp_invalidate_ad() is required to make sure we don't miss 2366 * dirty/young flags set by hardware. 2367 */ 2368 oldpmd = pmdp_invalidate_ad(vma, addr, pmd); 2369 2370 entry = pmd_modify(oldpmd, newprot); 2371 if (uffd_wp) 2372 entry = pmd_mkuffd_wp(entry); 2373 else if (uffd_wp_resolve) 2374 /* 2375 * Leave the write bit to be handled by PF interrupt 2376 * handler, then things like COW could be properly 2377 * handled. 2378 */ 2379 entry = pmd_clear_uffd_wp(entry); 2380 2381 /* See change_pte_range(). */ 2382 if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) && 2383 can_change_pmd_writable(vma, addr, entry)) 2384 entry = pmd_mkwrite(entry, vma); 2385 2386 ret = HPAGE_PMD_NR; 2387 set_pmd_at(mm, addr, pmd, entry); 2388 2389 if (huge_pmd_needs_flush(oldpmd, entry)) 2390 tlb_flush_pmd_range(tlb, addr, HPAGE_PMD_SIZE); 2391 unlock: 2392 spin_unlock(ptl); 2393 return ret; 2394 } 2395 2396 /* 2397 * Returns: 2398 * 2399 * - 0: if pud leaf changed from under us 2400 * - 1: if pud can be skipped 2401 * - HPAGE_PUD_NR: if pud was successfully processed 2402 */ 2403 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2404 int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 2405 pud_t *pudp, unsigned long addr, pgprot_t newprot, 2406 unsigned long cp_flags) 2407 { 2408 struct mm_struct *mm = vma->vm_mm; 2409 pud_t oldpud, entry; 2410 spinlock_t *ptl; 2411 2412 tlb_change_page_size(tlb, HPAGE_PUD_SIZE); 2413 2414 /* NUMA balancing doesn't apply to dax */ 2415 if (cp_flags & MM_CP_PROT_NUMA) 2416 return 1; 2417 2418 /* 2419 * Huge entries on userfault-wp only works with anonymous, while we 2420 * don't have anonymous PUDs yet. 2421 */ 2422 if (WARN_ON_ONCE(cp_flags & MM_CP_UFFD_WP_ALL)) 2423 return 1; 2424 2425 ptl = __pud_trans_huge_lock(pudp, vma); 2426 if (!ptl) 2427 return 0; 2428 2429 /* 2430 * Can't clear PUD or it can race with concurrent zapping. See 2431 * change_huge_pmd(). 2432 */ 2433 oldpud = pudp_invalidate(vma, addr, pudp); 2434 entry = pud_modify(oldpud, newprot); 2435 set_pud_at(mm, addr, pudp, entry); 2436 tlb_flush_pud_range(tlb, addr, HPAGE_PUD_SIZE); 2437 2438 spin_unlock(ptl); 2439 return HPAGE_PUD_NR; 2440 } 2441 #endif 2442 2443 #ifdef CONFIG_USERFAULTFD 2444 /* 2445 * The PT lock for src_pmd and dst_vma/src_vma (for reading) are locked by 2446 * the caller, but it must return after releasing the page_table_lock. 2447 * Just move the page from src_pmd to dst_pmd if possible. 2448 * Return zero if succeeded in moving the page, -EAGAIN if it needs to be 2449 * repeated by the caller, or other errors in case of failure. 2450 */ 2451 int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval, 2452 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, 2453 unsigned long dst_addr, unsigned long src_addr) 2454 { 2455 pmd_t _dst_pmd, src_pmdval; 2456 struct page *src_page; 2457 struct folio *src_folio; 2458 struct anon_vma *src_anon_vma; 2459 spinlock_t *src_ptl, *dst_ptl; 2460 pgtable_t src_pgtable; 2461 struct mmu_notifier_range range; 2462 int err = 0; 2463 2464 src_pmdval = *src_pmd; 2465 src_ptl = pmd_lockptr(mm, src_pmd); 2466 2467 lockdep_assert_held(src_ptl); 2468 vma_assert_locked(src_vma); 2469 vma_assert_locked(dst_vma); 2470 2471 /* Sanity checks before the operation */ 2472 if (WARN_ON_ONCE(!pmd_none(dst_pmdval)) || WARN_ON_ONCE(src_addr & ~HPAGE_PMD_MASK) || 2473 WARN_ON_ONCE(dst_addr & ~HPAGE_PMD_MASK)) { 2474 spin_unlock(src_ptl); 2475 return -EINVAL; 2476 } 2477 2478 if (!pmd_trans_huge(src_pmdval)) { 2479 spin_unlock(src_ptl); 2480 if (is_pmd_migration_entry(src_pmdval)) { 2481 pmd_migration_entry_wait(mm, &src_pmdval); 2482 return -EAGAIN; 2483 } 2484 return -ENOENT; 2485 } 2486 2487 src_page = pmd_page(src_pmdval); 2488 2489 if (!is_huge_zero_pmd(src_pmdval)) { 2490 if (unlikely(!PageAnonExclusive(src_page))) { 2491 spin_unlock(src_ptl); 2492 return -EBUSY; 2493 } 2494 2495 src_folio = page_folio(src_page); 2496 folio_get(src_folio); 2497 } else 2498 src_folio = NULL; 2499 2500 spin_unlock(src_ptl); 2501 2502 flush_cache_range(src_vma, src_addr, src_addr + HPAGE_PMD_SIZE); 2503 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, src_addr, 2504 src_addr + HPAGE_PMD_SIZE); 2505 mmu_notifier_invalidate_range_start(&range); 2506 2507 if (src_folio) { 2508 folio_lock(src_folio); 2509 2510 /* 2511 * split_huge_page walks the anon_vma chain without the page 2512 * lock. Serialize against it with the anon_vma lock, the page 2513 * lock is not enough. 2514 */ 2515 src_anon_vma = folio_get_anon_vma(src_folio); 2516 if (!src_anon_vma) { 2517 err = -EAGAIN; 2518 goto unlock_folio; 2519 } 2520 anon_vma_lock_write(src_anon_vma); 2521 } else 2522 src_anon_vma = NULL; 2523 2524 dst_ptl = pmd_lockptr(mm, dst_pmd); 2525 double_pt_lock(src_ptl, dst_ptl); 2526 if (unlikely(!pmd_same(*src_pmd, src_pmdval) || 2527 !pmd_same(*dst_pmd, dst_pmdval))) { 2528 err = -EAGAIN; 2529 goto unlock_ptls; 2530 } 2531 if (src_folio) { 2532 if (folio_maybe_dma_pinned(src_folio) || 2533 !PageAnonExclusive(&src_folio->page)) { 2534 err = -EBUSY; 2535 goto unlock_ptls; 2536 } 2537 2538 if (WARN_ON_ONCE(!folio_test_head(src_folio)) || 2539 WARN_ON_ONCE(!folio_test_anon(src_folio))) { 2540 err = -EBUSY; 2541 goto unlock_ptls; 2542 } 2543 2544 src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); 2545 /* Folio got pinned from under us. Put it back and fail the move. */ 2546 if (folio_maybe_dma_pinned(src_folio)) { 2547 set_pmd_at(mm, src_addr, src_pmd, src_pmdval); 2548 err = -EBUSY; 2549 goto unlock_ptls; 2550 } 2551 2552 folio_move_anon_rmap(src_folio, dst_vma); 2553 src_folio->index = linear_page_index(dst_vma, dst_addr); 2554 2555 _dst_pmd = mk_huge_pmd(&src_folio->page, dst_vma->vm_page_prot); 2556 /* Follow mremap() behavior and treat the entry dirty after the move */ 2557 _dst_pmd = pmd_mkwrite(pmd_mkdirty(_dst_pmd), dst_vma); 2558 } else { 2559 src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); 2560 _dst_pmd = mk_huge_pmd(src_page, dst_vma->vm_page_prot); 2561 } 2562 set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd); 2563 2564 src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd); 2565 pgtable_trans_huge_deposit(mm, dst_pmd, src_pgtable); 2566 unlock_ptls: 2567 double_pt_unlock(src_ptl, dst_ptl); 2568 if (src_anon_vma) { 2569 anon_vma_unlock_write(src_anon_vma); 2570 put_anon_vma(src_anon_vma); 2571 } 2572 unlock_folio: 2573 /* unblock rmap walks */ 2574 if (src_folio) 2575 folio_unlock(src_folio); 2576 mmu_notifier_invalidate_range_end(&range); 2577 if (src_folio) 2578 folio_put(src_folio); 2579 return err; 2580 } 2581 #endif /* CONFIG_USERFAULTFD */ 2582 2583 /* 2584 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise. 2585 * 2586 * Note that if it returns page table lock pointer, this routine returns without 2587 * unlocking page table lock. So callers must unlock it. 2588 */ 2589 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) 2590 { 2591 spinlock_t *ptl; 2592 ptl = pmd_lock(vma->vm_mm, pmd); 2593 if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || 2594 pmd_devmap(*pmd))) 2595 return ptl; 2596 spin_unlock(ptl); 2597 return NULL; 2598 } 2599 2600 /* 2601 * Returns page table lock pointer if a given pud maps a thp, NULL otherwise. 2602 * 2603 * Note that if it returns page table lock pointer, this routine returns without 2604 * unlocking page table lock. So callers must unlock it. 2605 */ 2606 spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma) 2607 { 2608 spinlock_t *ptl; 2609 2610 ptl = pud_lock(vma->vm_mm, pud); 2611 if (likely(pud_trans_huge(*pud) || pud_devmap(*pud))) 2612 return ptl; 2613 spin_unlock(ptl); 2614 return NULL; 2615 } 2616 2617 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2618 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 2619 pud_t *pud, unsigned long addr) 2620 { 2621 spinlock_t *ptl; 2622 pud_t orig_pud; 2623 2624 ptl = __pud_trans_huge_lock(pud, vma); 2625 if (!ptl) 2626 return 0; 2627 2628 orig_pud = pudp_huge_get_and_clear_full(vma, addr, pud, tlb->fullmm); 2629 arch_check_zapped_pud(vma, orig_pud); 2630 tlb_remove_pud_tlb_entry(tlb, pud, addr); 2631 if (vma_is_special_huge(vma)) { 2632 spin_unlock(ptl); 2633 /* No zero page support yet */ 2634 } else { 2635 /* No support for anonymous PUD pages yet */ 2636 BUG(); 2637 } 2638 return 1; 2639 } 2640 2641 static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud, 2642 unsigned long haddr) 2643 { 2644 VM_BUG_ON(haddr & ~HPAGE_PUD_MASK); 2645 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2646 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma); 2647 VM_BUG_ON(!pud_trans_huge(*pud) && !pud_devmap(*pud)); 2648 2649 count_vm_event(THP_SPLIT_PUD); 2650 2651 pudp_huge_clear_flush(vma, haddr, pud); 2652 } 2653 2654 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 2655 unsigned long address) 2656 { 2657 spinlock_t *ptl; 2658 struct mmu_notifier_range range; 2659 2660 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 2661 address & HPAGE_PUD_MASK, 2662 (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE); 2663 mmu_notifier_invalidate_range_start(&range); 2664 ptl = pud_lock(vma->vm_mm, pud); 2665 if (unlikely(!pud_trans_huge(*pud) && !pud_devmap(*pud))) 2666 goto out; 2667 __split_huge_pud_locked(vma, pud, range.start); 2668 2669 out: 2670 spin_unlock(ptl); 2671 mmu_notifier_invalidate_range_end(&range); 2672 } 2673 #else 2674 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 2675 unsigned long address) 2676 { 2677 } 2678 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 2679 2680 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 2681 unsigned long haddr, pmd_t *pmd) 2682 { 2683 struct mm_struct *mm = vma->vm_mm; 2684 pgtable_t pgtable; 2685 pmd_t _pmd, old_pmd; 2686 unsigned long addr; 2687 pte_t *pte; 2688 int i; 2689 2690 /* 2691 * Leave pmd empty until pte is filled note that it is fine to delay 2692 * notification until mmu_notifier_invalidate_range_end() as we are 2693 * replacing a zero pmd write protected page with a zero pte write 2694 * protected page. 2695 * 2696 * See Documentation/mm/mmu_notifier.rst 2697 */ 2698 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); 2699 2700 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2701 pmd_populate(mm, &_pmd, pgtable); 2702 2703 pte = pte_offset_map(&_pmd, haddr); 2704 VM_BUG_ON(!pte); 2705 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2706 pte_t entry; 2707 2708 entry = pfn_pte(my_zero_pfn(addr), vma->vm_page_prot); 2709 entry = pte_mkspecial(entry); 2710 if (pmd_uffd_wp(old_pmd)) 2711 entry = pte_mkuffd_wp(entry); 2712 VM_BUG_ON(!pte_none(ptep_get(pte))); 2713 set_pte_at(mm, addr, pte, entry); 2714 pte++; 2715 } 2716 pte_unmap(pte - 1); 2717 smp_wmb(); /* make pte visible before pmd */ 2718 pmd_populate(mm, pmd, pgtable); 2719 } 2720 2721 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 2722 unsigned long haddr, bool freeze) 2723 { 2724 struct mm_struct *mm = vma->vm_mm; 2725 struct folio *folio; 2726 struct page *page; 2727 pgtable_t pgtable; 2728 pmd_t old_pmd, _pmd; 2729 bool young, write, soft_dirty, pmd_migration = false, uffd_wp = false; 2730 bool anon_exclusive = false, dirty = false; 2731 unsigned long addr; 2732 pte_t *pte; 2733 int i; 2734 2735 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 2736 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2737 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 2738 VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd) 2739 && !pmd_devmap(*pmd)); 2740 2741 count_vm_event(THP_SPLIT_PMD); 2742 2743 if (!vma_is_anonymous(vma)) { 2744 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); 2745 /* 2746 * We are going to unmap this huge page. So 2747 * just go ahead and zap it 2748 */ 2749 if (arch_needs_pgtable_deposit()) 2750 zap_deposited_table(mm, pmd); 2751 if (vma_is_special_huge(vma)) 2752 return; 2753 if (unlikely(is_pmd_migration_entry(old_pmd))) { 2754 swp_entry_t entry; 2755 2756 entry = pmd_to_swp_entry(old_pmd); 2757 folio = pfn_swap_entry_folio(entry); 2758 } else { 2759 page = pmd_page(old_pmd); 2760 folio = page_folio(page); 2761 if (!folio_test_dirty(folio) && pmd_dirty(old_pmd)) 2762 folio_mark_dirty(folio); 2763 if (!folio_test_referenced(folio) && pmd_young(old_pmd)) 2764 folio_set_referenced(folio); 2765 folio_remove_rmap_pmd(folio, page, vma); 2766 folio_put(folio); 2767 } 2768 add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR); 2769 return; 2770 } 2771 2772 if (is_huge_zero_pmd(*pmd)) { 2773 /* 2774 * FIXME: Do we want to invalidate secondary mmu by calling 2775 * mmu_notifier_arch_invalidate_secondary_tlbs() see comments below 2776 * inside __split_huge_pmd() ? 2777 * 2778 * We are going from a zero huge page write protected to zero 2779 * small page also write protected so it does not seems useful 2780 * to invalidate secondary mmu at this time. 2781 */ 2782 return __split_huge_zero_page_pmd(vma, haddr, pmd); 2783 } 2784 2785 pmd_migration = is_pmd_migration_entry(*pmd); 2786 if (unlikely(pmd_migration)) { 2787 swp_entry_t entry; 2788 2789 old_pmd = *pmd; 2790 entry = pmd_to_swp_entry(old_pmd); 2791 page = pfn_swap_entry_to_page(entry); 2792 write = is_writable_migration_entry(entry); 2793 if (PageAnon(page)) 2794 anon_exclusive = is_readable_exclusive_migration_entry(entry); 2795 young = is_migration_entry_young(entry); 2796 dirty = is_migration_entry_dirty(entry); 2797 soft_dirty = pmd_swp_soft_dirty(old_pmd); 2798 uffd_wp = pmd_swp_uffd_wp(old_pmd); 2799 } else { 2800 /* 2801 * Up to this point the pmd is present and huge and userland has 2802 * the whole access to the hugepage during the split (which 2803 * happens in place). If we overwrite the pmd with the not-huge 2804 * version pointing to the pte here (which of course we could if 2805 * all CPUs were bug free), userland could trigger a small page 2806 * size TLB miss on the small sized TLB while the hugepage TLB 2807 * entry is still established in the huge TLB. Some CPU doesn't 2808 * like that. See 2809 * http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum 2810 * 383 on page 105. Intel should be safe but is also warns that 2811 * it's only safe if the permission and cache attributes of the 2812 * two entries loaded in the two TLB is identical (which should 2813 * be the case here). But it is generally safer to never allow 2814 * small and huge TLB entries for the same virtual address to be 2815 * loaded simultaneously. So instead of doing "pmd_populate(); 2816 * flush_pmd_tlb_range();" we first mark the current pmd 2817 * notpresent (atomically because here the pmd_trans_huge must 2818 * remain set at all times on the pmd until the split is 2819 * complete for this pmd), then we flush the SMP TLB and finally 2820 * we write the non-huge version of the pmd entry with 2821 * pmd_populate. 2822 */ 2823 old_pmd = pmdp_invalidate(vma, haddr, pmd); 2824 page = pmd_page(old_pmd); 2825 folio = page_folio(page); 2826 if (pmd_dirty(old_pmd)) { 2827 dirty = true; 2828 folio_set_dirty(folio); 2829 } 2830 write = pmd_write(old_pmd); 2831 young = pmd_young(old_pmd); 2832 soft_dirty = pmd_soft_dirty(old_pmd); 2833 uffd_wp = pmd_uffd_wp(old_pmd); 2834 2835 VM_WARN_ON_FOLIO(!folio_ref_count(folio), folio); 2836 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); 2837 2838 /* 2839 * Without "freeze", we'll simply split the PMD, propagating the 2840 * PageAnonExclusive() flag for each PTE by setting it for 2841 * each subpage -- no need to (temporarily) clear. 2842 * 2843 * With "freeze" we want to replace mapped pages by 2844 * migration entries right away. This is only possible if we 2845 * managed to clear PageAnonExclusive() -- see 2846 * set_pmd_migration_entry(). 2847 * 2848 * In case we cannot clear PageAnonExclusive(), split the PMD 2849 * only and let try_to_migrate_one() fail later. 2850 * 2851 * See folio_try_share_anon_rmap_pmd(): invalidate PMD first. 2852 */ 2853 anon_exclusive = PageAnonExclusive(page); 2854 if (freeze && anon_exclusive && 2855 folio_try_share_anon_rmap_pmd(folio, page)) 2856 freeze = false; 2857 if (!freeze) { 2858 rmap_t rmap_flags = RMAP_NONE; 2859 2860 folio_ref_add(folio, HPAGE_PMD_NR - 1); 2861 if (anon_exclusive) 2862 rmap_flags |= RMAP_EXCLUSIVE; 2863 folio_add_anon_rmap_ptes(folio, page, HPAGE_PMD_NR, 2864 vma, haddr, rmap_flags); 2865 } 2866 } 2867 2868 /* 2869 * Withdraw the table only after we mark the pmd entry invalid. 2870 * This's critical for some architectures (Power). 2871 */ 2872 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2873 pmd_populate(mm, &_pmd, pgtable); 2874 2875 pte = pte_offset_map(&_pmd, haddr); 2876 VM_BUG_ON(!pte); 2877 2878 /* 2879 * Note that NUMA hinting access restrictions are not transferred to 2880 * avoid any possibility of altering permissions across VMAs. 2881 */ 2882 if (freeze || pmd_migration) { 2883 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2884 pte_t entry; 2885 swp_entry_t swp_entry; 2886 2887 if (write) 2888 swp_entry = make_writable_migration_entry( 2889 page_to_pfn(page + i)); 2890 else if (anon_exclusive) 2891 swp_entry = make_readable_exclusive_migration_entry( 2892 page_to_pfn(page + i)); 2893 else 2894 swp_entry = make_readable_migration_entry( 2895 page_to_pfn(page + i)); 2896 if (young) 2897 swp_entry = make_migration_entry_young(swp_entry); 2898 if (dirty) 2899 swp_entry = make_migration_entry_dirty(swp_entry); 2900 entry = swp_entry_to_pte(swp_entry); 2901 if (soft_dirty) 2902 entry = pte_swp_mksoft_dirty(entry); 2903 if (uffd_wp) 2904 entry = pte_swp_mkuffd_wp(entry); 2905 2906 VM_WARN_ON(!pte_none(ptep_get(pte + i))); 2907 set_pte_at(mm, addr, pte + i, entry); 2908 } 2909 } else { 2910 pte_t entry; 2911 2912 entry = mk_pte(page, READ_ONCE(vma->vm_page_prot)); 2913 if (write) 2914 entry = pte_mkwrite(entry, vma); 2915 if (!young) 2916 entry = pte_mkold(entry); 2917 /* NOTE: this may set soft-dirty too on some archs */ 2918 if (dirty) 2919 entry = pte_mkdirty(entry); 2920 if (soft_dirty) 2921 entry = pte_mksoft_dirty(entry); 2922 if (uffd_wp) 2923 entry = pte_mkuffd_wp(entry); 2924 2925 for (i = 0; i < HPAGE_PMD_NR; i++) 2926 VM_WARN_ON(!pte_none(ptep_get(pte + i))); 2927 2928 set_ptes(mm, haddr, pte, entry, HPAGE_PMD_NR); 2929 } 2930 pte_unmap(pte); 2931 2932 if (!pmd_migration) 2933 folio_remove_rmap_pmd(folio, page, vma); 2934 if (freeze) 2935 put_page(page); 2936 2937 smp_wmb(); /* make pte visible before pmd */ 2938 pmd_populate(mm, pmd, pgtable); 2939 } 2940 2941 void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, 2942 pmd_t *pmd, bool freeze, struct folio *folio) 2943 { 2944 VM_WARN_ON_ONCE(folio && !folio_test_pmd_mappable(folio)); 2945 VM_WARN_ON_ONCE(!IS_ALIGNED(address, HPAGE_PMD_SIZE)); 2946 VM_WARN_ON_ONCE(folio && !folio_test_locked(folio)); 2947 VM_BUG_ON(freeze && !folio); 2948 2949 /* 2950 * When the caller requests to set up a migration entry, we 2951 * require a folio to check the PMD against. Otherwise, there 2952 * is a risk of replacing the wrong folio. 2953 */ 2954 if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd) || 2955 is_pmd_migration_entry(*pmd)) { 2956 if (folio && folio != pmd_folio(*pmd)) 2957 return; 2958 __split_huge_pmd_locked(vma, pmd, address, freeze); 2959 } 2960 } 2961 2962 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 2963 unsigned long address, bool freeze, struct folio *folio) 2964 { 2965 spinlock_t *ptl; 2966 struct mmu_notifier_range range; 2967 2968 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 2969 address & HPAGE_PMD_MASK, 2970 (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE); 2971 mmu_notifier_invalidate_range_start(&range); 2972 ptl = pmd_lock(vma->vm_mm, pmd); 2973 split_huge_pmd_locked(vma, range.start, pmd, freeze, folio); 2974 spin_unlock(ptl); 2975 mmu_notifier_invalidate_range_end(&range); 2976 } 2977 2978 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, 2979 bool freeze, struct folio *folio) 2980 { 2981 pmd_t *pmd = mm_find_pmd(vma->vm_mm, address); 2982 2983 if (!pmd) 2984 return; 2985 2986 __split_huge_pmd(vma, pmd, address, freeze, folio); 2987 } 2988 2989 static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address) 2990 { 2991 /* 2992 * If the new address isn't hpage aligned and it could previously 2993 * contain an hugepage: check if we need to split an huge pmd. 2994 */ 2995 if (!IS_ALIGNED(address, HPAGE_PMD_SIZE) && 2996 range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE), 2997 ALIGN(address, HPAGE_PMD_SIZE))) 2998 split_huge_pmd_address(vma, address, false, NULL); 2999 } 3000 3001 void vma_adjust_trans_huge(struct vm_area_struct *vma, 3002 unsigned long start, 3003 unsigned long end, 3004 long adjust_next) 3005 { 3006 /* Check if we need to split start first. */ 3007 split_huge_pmd_if_needed(vma, start); 3008 3009 /* Check if we need to split end next. */ 3010 split_huge_pmd_if_needed(vma, end); 3011 3012 /* 3013 * If we're also updating the next vma vm_start, 3014 * check if we need to split it. 3015 */ 3016 if (adjust_next > 0) { 3017 struct vm_area_struct *next = find_vma(vma->vm_mm, vma->vm_end); 3018 unsigned long nstart = next->vm_start; 3019 nstart += adjust_next; 3020 split_huge_pmd_if_needed(next, nstart); 3021 } 3022 } 3023 3024 static void unmap_folio(struct folio *folio) 3025 { 3026 enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SYNC | 3027 TTU_BATCH_FLUSH; 3028 3029 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 3030 3031 if (folio_test_pmd_mappable(folio)) 3032 ttu_flags |= TTU_SPLIT_HUGE_PMD; 3033 3034 /* 3035 * Anon pages need migration entries to preserve them, but file 3036 * pages can simply be left unmapped, then faulted back on demand. 3037 * If that is ever changed (perhaps for mlock), update remap_page(). 3038 */ 3039 if (folio_test_anon(folio)) 3040 try_to_migrate(folio, ttu_flags); 3041 else 3042 try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK); 3043 3044 try_to_unmap_flush(); 3045 } 3046 3047 static bool __discard_anon_folio_pmd_locked(struct vm_area_struct *vma, 3048 unsigned long addr, pmd_t *pmdp, 3049 struct folio *folio) 3050 { 3051 struct mm_struct *mm = vma->vm_mm; 3052 int ref_count, map_count; 3053 pmd_t orig_pmd = *pmdp; 3054 3055 if (folio_test_dirty(folio) || pmd_dirty(orig_pmd)) 3056 return false; 3057 3058 orig_pmd = pmdp_huge_clear_flush(vma, addr, pmdp); 3059 3060 /* 3061 * Syncing against concurrent GUP-fast: 3062 * - clear PMD; barrier; read refcount 3063 * - inc refcount; barrier; read PMD 3064 */ 3065 smp_mb(); 3066 3067 ref_count = folio_ref_count(folio); 3068 map_count = folio_mapcount(folio); 3069 3070 /* 3071 * Order reads for folio refcount and dirty flag 3072 * (see comments in __remove_mapping()). 3073 */ 3074 smp_rmb(); 3075 3076 /* 3077 * If the folio or its PMD is redirtied at this point, or if there 3078 * are unexpected references, we will give up to discard this folio 3079 * and remap it. 3080 * 3081 * The only folio refs must be one from isolation plus the rmap(s). 3082 */ 3083 if (folio_test_dirty(folio) || pmd_dirty(orig_pmd) || 3084 ref_count != map_count + 1) { 3085 set_pmd_at(mm, addr, pmdp, orig_pmd); 3086 return false; 3087 } 3088 3089 folio_remove_rmap_pmd(folio, pmd_page(orig_pmd), vma); 3090 zap_deposited_table(mm, pmdp); 3091 add_mm_counter(mm, MM_ANONPAGES, -HPAGE_PMD_NR); 3092 if (vma->vm_flags & VM_LOCKED) 3093 mlock_drain_local(); 3094 folio_put(folio); 3095 3096 return true; 3097 } 3098 3099 bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr, 3100 pmd_t *pmdp, struct folio *folio) 3101 { 3102 VM_WARN_ON_FOLIO(!folio_test_pmd_mappable(folio), folio); 3103 VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); 3104 VM_WARN_ON_ONCE(!IS_ALIGNED(addr, HPAGE_PMD_SIZE)); 3105 3106 if (folio_test_anon(folio) && !folio_test_swapbacked(folio)) 3107 return __discard_anon_folio_pmd_locked(vma, addr, pmdp, folio); 3108 3109 return false; 3110 } 3111 3112 static void remap_page(struct folio *folio, unsigned long nr, int flags) 3113 { 3114 int i = 0; 3115 3116 /* If unmap_folio() uses try_to_migrate() on file, remove this check */ 3117 if (!folio_test_anon(folio)) 3118 return; 3119 for (;;) { 3120 remove_migration_ptes(folio, folio, RMP_LOCKED | flags); 3121 i += folio_nr_pages(folio); 3122 if (i >= nr) 3123 break; 3124 folio = folio_next(folio); 3125 } 3126 } 3127 3128 static void lru_add_page_tail(struct folio *folio, struct page *tail, 3129 struct lruvec *lruvec, struct list_head *list) 3130 { 3131 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 3132 VM_BUG_ON_FOLIO(PageLRU(tail), folio); 3133 lockdep_assert_held(&lruvec->lru_lock); 3134 3135 if (list) { 3136 /* page reclaim is reclaiming a huge page */ 3137 VM_WARN_ON(folio_test_lru(folio)); 3138 get_page(tail); 3139 list_add_tail(&tail->lru, list); 3140 } else { 3141 /* head is still on lru (and we have it frozen) */ 3142 VM_WARN_ON(!folio_test_lru(folio)); 3143 if (folio_test_unevictable(folio)) 3144 tail->mlock_count = 0; 3145 else 3146 list_add_tail(&tail->lru, &folio->lru); 3147 SetPageLRU(tail); 3148 } 3149 } 3150 3151 static void __split_huge_page_tail(struct folio *folio, int tail, 3152 struct lruvec *lruvec, struct list_head *list, 3153 unsigned int new_order) 3154 { 3155 struct page *head = &folio->page; 3156 struct page *page_tail = head + tail; 3157 /* 3158 * Careful: new_folio is not a "real" folio before we cleared PageTail. 3159 * Don't pass it around before clear_compound_head(). 3160 */ 3161 struct folio *new_folio = (struct folio *)page_tail; 3162 3163 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail); 3164 3165 /* 3166 * Clone page flags before unfreezing refcount. 3167 * 3168 * After successful get_page_unless_zero() might follow flags change, 3169 * for example lock_page() which set PG_waiters. 3170 * 3171 * Note that for mapped sub-pages of an anonymous THP, 3172 * PG_anon_exclusive has been cleared in unmap_folio() and is stored in 3173 * the migration entry instead from where remap_page() will restore it. 3174 * We can still have PG_anon_exclusive set on effectively unmapped and 3175 * unreferenced sub-pages of an anonymous THP: we can simply drop 3176 * PG_anon_exclusive (-> PG_mappedtodisk) for these here. 3177 */ 3178 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 3179 page_tail->flags |= (head->flags & 3180 ((1L << PG_referenced) | 3181 (1L << PG_swapbacked) | 3182 (1L << PG_swapcache) | 3183 (1L << PG_mlocked) | 3184 (1L << PG_uptodate) | 3185 (1L << PG_active) | 3186 (1L << PG_workingset) | 3187 (1L << PG_locked) | 3188 (1L << PG_unevictable) | 3189 #ifdef CONFIG_ARCH_USES_PG_ARCH_2 3190 (1L << PG_arch_2) | 3191 #endif 3192 #ifdef CONFIG_ARCH_USES_PG_ARCH_3 3193 (1L << PG_arch_3) | 3194 #endif 3195 (1L << PG_dirty) | 3196 LRU_GEN_MASK | LRU_REFS_MASK)); 3197 3198 /* ->mapping in first and second tail page is replaced by other uses */ 3199 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, 3200 page_tail); 3201 new_folio->mapping = folio->mapping; 3202 new_folio->index = folio->index + tail; 3203 3204 /* 3205 * page->private should not be set in tail pages. Fix up and warn once 3206 * if private is unexpectedly set. 3207 */ 3208 if (unlikely(page_tail->private)) { 3209 VM_WARN_ON_ONCE_PAGE(true, page_tail); 3210 page_tail->private = 0; 3211 } 3212 if (folio_test_swapcache(folio)) 3213 new_folio->swap.val = folio->swap.val + tail; 3214 3215 /* Page flags must be visible before we make the page non-compound. */ 3216 smp_wmb(); 3217 3218 /* 3219 * Clear PageTail before unfreezing page refcount. 3220 * 3221 * After successful get_page_unless_zero() might follow put_page() 3222 * which needs correct compound_head(). 3223 */ 3224 clear_compound_head(page_tail); 3225 if (new_order) { 3226 prep_compound_page(page_tail, new_order); 3227 folio_set_large_rmappable(new_folio); 3228 } 3229 3230 /* Finally unfreeze refcount. Additional reference from page cache. */ 3231 page_ref_unfreeze(page_tail, 3232 1 + ((!folio_test_anon(folio) || folio_test_swapcache(folio)) ? 3233 folio_nr_pages(new_folio) : 0)); 3234 3235 if (folio_test_young(folio)) 3236 folio_set_young(new_folio); 3237 if (folio_test_idle(folio)) 3238 folio_set_idle(new_folio); 3239 3240 folio_xchg_last_cpupid(new_folio, folio_last_cpupid(folio)); 3241 3242 /* 3243 * always add to the tail because some iterators expect new 3244 * pages to show after the currently processed elements - e.g. 3245 * migrate_pages 3246 */ 3247 lru_add_page_tail(folio, page_tail, lruvec, list); 3248 } 3249 3250 static void __split_huge_page(struct page *page, struct list_head *list, 3251 pgoff_t end, unsigned int new_order) 3252 { 3253 struct folio *folio = page_folio(page); 3254 struct page *head = &folio->page; 3255 struct lruvec *lruvec; 3256 struct address_space *swap_cache = NULL; 3257 unsigned long offset = 0; 3258 int i, nr_dropped = 0; 3259 unsigned int new_nr = 1 << new_order; 3260 int order = folio_order(folio); 3261 unsigned int nr = 1 << order; 3262 3263 /* complete memcg works before add pages to LRU */ 3264 split_page_memcg(head, order, new_order); 3265 3266 if (folio_test_anon(folio) && folio_test_swapcache(folio)) { 3267 offset = swap_cache_index(folio->swap); 3268 swap_cache = swap_address_space(folio->swap); 3269 xa_lock(&swap_cache->i_pages); 3270 } 3271 3272 /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ 3273 lruvec = folio_lruvec_lock(folio); 3274 3275 ClearPageHasHWPoisoned(head); 3276 3277 for (i = nr - new_nr; i >= new_nr; i -= new_nr) { 3278 struct folio *tail; 3279 __split_huge_page_tail(folio, i, lruvec, list, new_order); 3280 tail = page_folio(head + i); 3281 /* Some pages can be beyond EOF: drop them from page cache */ 3282 if (tail->index >= end) { 3283 if (shmem_mapping(folio->mapping)) 3284 nr_dropped++; 3285 else if (folio_test_clear_dirty(tail)) 3286 folio_account_cleaned(tail, 3287 inode_to_wb(folio->mapping->host)); 3288 __filemap_remove_folio(tail, NULL); 3289 folio_put(tail); 3290 } else if (!folio_test_anon(folio)) { 3291 __xa_store(&folio->mapping->i_pages, tail->index, 3292 tail, 0); 3293 } else if (swap_cache) { 3294 __xa_store(&swap_cache->i_pages, offset + i, 3295 tail, 0); 3296 } 3297 } 3298 3299 if (!new_order) 3300 ClearPageCompound(head); 3301 else { 3302 struct folio *new_folio = (struct folio *)head; 3303 3304 folio_set_order(new_folio, new_order); 3305 } 3306 unlock_page_lruvec(lruvec); 3307 /* Caller disabled irqs, so they are still disabled here */ 3308 3309 split_page_owner(head, order, new_order); 3310 pgalloc_tag_split(folio, order, new_order); 3311 3312 /* See comment in __split_huge_page_tail() */ 3313 if (folio_test_anon(folio)) { 3314 /* Additional pin to swap cache */ 3315 if (folio_test_swapcache(folio)) { 3316 folio_ref_add(folio, 1 + new_nr); 3317 xa_unlock(&swap_cache->i_pages); 3318 } else { 3319 folio_ref_inc(folio); 3320 } 3321 } else { 3322 /* Additional pin to page cache */ 3323 folio_ref_add(folio, 1 + new_nr); 3324 xa_unlock(&folio->mapping->i_pages); 3325 } 3326 local_irq_enable(); 3327 3328 if (nr_dropped) 3329 shmem_uncharge(folio->mapping->host, nr_dropped); 3330 remap_page(folio, nr, PageAnon(head) ? RMP_USE_SHARED_ZEROPAGE : 0); 3331 3332 /* 3333 * set page to its compound_head when split to non order-0 pages, so 3334 * we can skip unlocking it below, since PG_locked is transferred to 3335 * the compound_head of the page and the caller will unlock it. 3336 */ 3337 if (new_order) 3338 page = compound_head(page); 3339 3340 for (i = 0; i < nr; i += new_nr) { 3341 struct page *subpage = head + i; 3342 struct folio *new_folio = page_folio(subpage); 3343 if (subpage == page) 3344 continue; 3345 folio_unlock(new_folio); 3346 3347 /* 3348 * Subpages may be freed if there wasn't any mapping 3349 * like if add_to_swap() is running on a lru page that 3350 * had its mapping zapped. And freeing these pages 3351 * requires taking the lru_lock so we do the put_page 3352 * of the tail pages after the split is complete. 3353 */ 3354 free_page_and_swap_cache(subpage); 3355 } 3356 } 3357 3358 /* Racy check whether the huge page can be split */ 3359 bool can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins) 3360 { 3361 int extra_pins; 3362 3363 /* Additional pins from page cache */ 3364 if (folio_test_anon(folio)) 3365 extra_pins = folio_test_swapcache(folio) ? 3366 folio_nr_pages(folio) : 0; 3367 else 3368 extra_pins = folio_nr_pages(folio); 3369 if (pextra_pins) 3370 *pextra_pins = extra_pins; 3371 return folio_mapcount(folio) == folio_ref_count(folio) - extra_pins - 3372 caller_pins; 3373 } 3374 3375 /* 3376 * This function splits a large folio into smaller folios of order @new_order. 3377 * @page can point to any page of the large folio to split. The split operation 3378 * does not change the position of @page. 3379 * 3380 * Prerequisites: 3381 * 3382 * 1) The caller must hold a reference on the @page's owning folio, also known 3383 * as the large folio. 3384 * 3385 * 2) The large folio must be locked. 3386 * 3387 * 3) The folio must not be pinned. Any unexpected folio references, including 3388 * GUP pins, will result in the folio not getting split; instead, the caller 3389 * will receive an -EAGAIN. 3390 * 3391 * 4) @new_order > 1, usually. Splitting to order-1 anonymous folios is not 3392 * supported for non-file-backed folios, because folio->_deferred_list, which 3393 * is used by partially mapped folios, is stored in subpage 2, but an order-1 3394 * folio only has subpages 0 and 1. File-backed order-1 folios are supported, 3395 * since they do not use _deferred_list. 3396 * 3397 * After splitting, the caller's folio reference will be transferred to @page, 3398 * resulting in a raised refcount of @page after this call. The other pages may 3399 * be freed if they are not mapped. 3400 * 3401 * If @list is null, tail pages will be added to LRU list, otherwise, to @list. 3402 * 3403 * Pages in @new_order will inherit the mapping, flags, and so on from the 3404 * huge page. 3405 * 3406 * Returns 0 if the huge page was split successfully. 3407 * 3408 * Returns -EAGAIN if the folio has unexpected reference (e.g., GUP) or if 3409 * the folio was concurrently removed from the page cache. 3410 * 3411 * Returns -EBUSY when trying to split the huge zeropage, if the folio is 3412 * under writeback, if fs-specific folio metadata cannot currently be 3413 * released, or if some unexpected race happened (e.g., anon VMA disappeared, 3414 * truncation). 3415 * 3416 * Callers should ensure that the order respects the address space mapping 3417 * min-order if one is set for non-anonymous folios. 3418 * 3419 * Returns -EINVAL when trying to split to an order that is incompatible 3420 * with the folio. Splitting to order 0 is compatible with all folios. 3421 */ 3422 int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, 3423 unsigned int new_order) 3424 { 3425 struct folio *folio = page_folio(page); 3426 struct deferred_split *ds_queue = get_deferred_split_queue(folio); 3427 /* reset xarray order to new order after split */ 3428 XA_STATE_ORDER(xas, &folio->mapping->i_pages, folio->index, new_order); 3429 bool is_anon = folio_test_anon(folio); 3430 struct address_space *mapping = NULL; 3431 struct anon_vma *anon_vma = NULL; 3432 int order = folio_order(folio); 3433 int extra_pins, ret; 3434 pgoff_t end; 3435 bool is_hzp; 3436 3437 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 3438 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 3439 3440 if (new_order >= folio_order(folio)) 3441 return -EINVAL; 3442 3443 if (is_anon) { 3444 /* order-1 is not supported for anonymous THP. */ 3445 if (new_order == 1) { 3446 VM_WARN_ONCE(1, "Cannot split to order-1 folio"); 3447 return -EINVAL; 3448 } 3449 } else if (new_order) { 3450 /* Split shmem folio to non-zero order not supported */ 3451 if (shmem_mapping(folio->mapping)) { 3452 VM_WARN_ONCE(1, 3453 "Cannot split shmem folio to non-0 order"); 3454 return -EINVAL; 3455 } 3456 /* 3457 * No split if the file system does not support large folio. 3458 * Note that we might still have THPs in such mappings due to 3459 * CONFIG_READ_ONLY_THP_FOR_FS. But in that case, the mapping 3460 * does not actually support large folios properly. 3461 */ 3462 if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && 3463 !mapping_large_folio_support(folio->mapping)) { 3464 VM_WARN_ONCE(1, 3465 "Cannot split file folio to non-0 order"); 3466 return -EINVAL; 3467 } 3468 } 3469 3470 /* Only swapping a whole PMD-mapped folio is supported */ 3471 if (folio_test_swapcache(folio) && new_order) 3472 return -EINVAL; 3473 3474 is_hzp = is_huge_zero_folio(folio); 3475 if (is_hzp) { 3476 pr_warn_ratelimited("Called split_huge_page for huge zero page\n"); 3477 return -EBUSY; 3478 } 3479 3480 if (folio_test_writeback(folio)) 3481 return -EBUSY; 3482 3483 if (is_anon) { 3484 /* 3485 * The caller does not necessarily hold an mmap_lock that would 3486 * prevent the anon_vma disappearing so we first we take a 3487 * reference to it and then lock the anon_vma for write. This 3488 * is similar to folio_lock_anon_vma_read except the write lock 3489 * is taken to serialise against parallel split or collapse 3490 * operations. 3491 */ 3492 anon_vma = folio_get_anon_vma(folio); 3493 if (!anon_vma) { 3494 ret = -EBUSY; 3495 goto out; 3496 } 3497 end = -1; 3498 mapping = NULL; 3499 anon_vma_lock_write(anon_vma); 3500 } else { 3501 unsigned int min_order; 3502 gfp_t gfp; 3503 3504 mapping = folio->mapping; 3505 3506 /* Truncated ? */ 3507 if (!mapping) { 3508 ret = -EBUSY; 3509 goto out; 3510 } 3511 3512 min_order = mapping_min_folio_order(folio->mapping); 3513 if (new_order < min_order) { 3514 VM_WARN_ONCE(1, "Cannot split mapped folio below min-order: %u", 3515 min_order); 3516 ret = -EINVAL; 3517 goto out; 3518 } 3519 3520 gfp = current_gfp_context(mapping_gfp_mask(mapping) & 3521 GFP_RECLAIM_MASK); 3522 3523 if (!filemap_release_folio(folio, gfp)) { 3524 ret = -EBUSY; 3525 goto out; 3526 } 3527 3528 xas_split_alloc(&xas, folio, folio_order(folio), gfp); 3529 if (xas_error(&xas)) { 3530 ret = xas_error(&xas); 3531 goto out; 3532 } 3533 3534 anon_vma = NULL; 3535 i_mmap_lock_read(mapping); 3536 3537 /* 3538 *__split_huge_page() may need to trim off pages beyond EOF: 3539 * but on 32-bit, i_size_read() takes an irq-unsafe seqlock, 3540 * which cannot be nested inside the page tree lock. So note 3541 * end now: i_size itself may be changed at any moment, but 3542 * folio lock is good enough to serialize the trimming. 3543 */ 3544 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); 3545 if (shmem_mapping(mapping)) 3546 end = shmem_fallocend(mapping->host, end); 3547 } 3548 3549 /* 3550 * Racy check if we can split the page, before unmap_folio() will 3551 * split PMDs 3552 */ 3553 if (!can_split_folio(folio, 1, &extra_pins)) { 3554 ret = -EAGAIN; 3555 goto out_unlock; 3556 } 3557 3558 unmap_folio(folio); 3559 3560 /* block interrupt reentry in xa_lock and spinlock */ 3561 local_irq_disable(); 3562 if (mapping) { 3563 /* 3564 * Check if the folio is present in page cache. 3565 * We assume all tail are present too, if folio is there. 3566 */ 3567 xas_lock(&xas); 3568 xas_reset(&xas); 3569 if (xas_load(&xas) != folio) 3570 goto fail; 3571 } 3572 3573 /* Prevent deferred_split_scan() touching ->_refcount */ 3574 spin_lock(&ds_queue->split_queue_lock); 3575 if (folio_ref_freeze(folio, 1 + extra_pins)) { 3576 if (folio_order(folio) > 1 && 3577 !list_empty(&folio->_deferred_list)) { 3578 ds_queue->split_queue_len--; 3579 if (folio_test_partially_mapped(folio)) { 3580 folio_clear_partially_mapped(folio); 3581 mod_mthp_stat(folio_order(folio), 3582 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 3583 } 3584 /* 3585 * Reinitialize page_deferred_list after removing the 3586 * page from the split_queue, otherwise a subsequent 3587 * split will see list corruption when checking the 3588 * page_deferred_list. 3589 */ 3590 list_del_init(&folio->_deferred_list); 3591 } 3592 spin_unlock(&ds_queue->split_queue_lock); 3593 if (mapping) { 3594 int nr = folio_nr_pages(folio); 3595 3596 xas_split(&xas, folio, folio_order(folio)); 3597 if (folio_test_pmd_mappable(folio) && 3598 new_order < HPAGE_PMD_ORDER) { 3599 if (folio_test_swapbacked(folio)) { 3600 __lruvec_stat_mod_folio(folio, 3601 NR_SHMEM_THPS, -nr); 3602 } else { 3603 __lruvec_stat_mod_folio(folio, 3604 NR_FILE_THPS, -nr); 3605 filemap_nr_thps_dec(mapping); 3606 } 3607 } 3608 } 3609 3610 if (is_anon) { 3611 mod_mthp_stat(order, MTHP_STAT_NR_ANON, -1); 3612 mod_mthp_stat(new_order, MTHP_STAT_NR_ANON, 1 << (order - new_order)); 3613 } 3614 __split_huge_page(page, list, end, new_order); 3615 ret = 0; 3616 } else { 3617 spin_unlock(&ds_queue->split_queue_lock); 3618 fail: 3619 if (mapping) 3620 xas_unlock(&xas); 3621 local_irq_enable(); 3622 remap_page(folio, folio_nr_pages(folio), 0); 3623 ret = -EAGAIN; 3624 } 3625 3626 out_unlock: 3627 if (anon_vma) { 3628 anon_vma_unlock_write(anon_vma); 3629 put_anon_vma(anon_vma); 3630 } 3631 if (mapping) 3632 i_mmap_unlock_read(mapping); 3633 out: 3634 xas_destroy(&xas); 3635 if (order == HPAGE_PMD_ORDER) 3636 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); 3637 count_mthp_stat(order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED); 3638 return ret; 3639 } 3640 3641 int min_order_for_split(struct folio *folio) 3642 { 3643 if (folio_test_anon(folio)) 3644 return 0; 3645 3646 if (!folio->mapping) { 3647 if (folio_test_pmd_mappable(folio)) 3648 count_vm_event(THP_SPLIT_PAGE_FAILED); 3649 return -EBUSY; 3650 } 3651 3652 return mapping_min_folio_order(folio->mapping); 3653 } 3654 3655 int split_folio_to_list(struct folio *folio, struct list_head *list) 3656 { 3657 int ret = min_order_for_split(folio); 3658 3659 if (ret < 0) 3660 return ret; 3661 3662 return split_huge_page_to_list_to_order(&folio->page, list, ret); 3663 } 3664 3665 /* 3666 * __folio_unqueue_deferred_split() is not to be called directly: 3667 * the folio_unqueue_deferred_split() inline wrapper in mm/internal.h 3668 * limits its calls to those folios which may have a _deferred_list for 3669 * queueing THP splits, and that list is (racily observed to be) non-empty. 3670 * 3671 * It is unsafe to call folio_unqueue_deferred_split() until folio refcount is 3672 * zero: because even when split_queue_lock is held, a non-empty _deferred_list 3673 * might be in use on deferred_split_scan()'s unlocked on-stack list. 3674 * 3675 * If memory cgroups are enabled, split_queue_lock is in the mem_cgroup: it is 3676 * therefore important to unqueue deferred split before changing folio memcg. 3677 */ 3678 bool __folio_unqueue_deferred_split(struct folio *folio) 3679 { 3680 struct deferred_split *ds_queue; 3681 unsigned long flags; 3682 bool unqueued = false; 3683 3684 WARN_ON_ONCE(folio_ref_count(folio)); 3685 WARN_ON_ONCE(!mem_cgroup_disabled() && !folio_memcg(folio)); 3686 3687 ds_queue = get_deferred_split_queue(folio); 3688 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 3689 if (!list_empty(&folio->_deferred_list)) { 3690 ds_queue->split_queue_len--; 3691 if (folio_test_partially_mapped(folio)) { 3692 folio_clear_partially_mapped(folio); 3693 mod_mthp_stat(folio_order(folio), 3694 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 3695 } 3696 list_del_init(&folio->_deferred_list); 3697 unqueued = true; 3698 } 3699 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 3700 3701 return unqueued; /* useful for debug warnings */ 3702 } 3703 3704 /* partially_mapped=false won't clear PG_partially_mapped folio flag */ 3705 void deferred_split_folio(struct folio *folio, bool partially_mapped) 3706 { 3707 struct deferred_split *ds_queue = get_deferred_split_queue(folio); 3708 #ifdef CONFIG_MEMCG 3709 struct mem_cgroup *memcg = folio_memcg(folio); 3710 #endif 3711 unsigned long flags; 3712 3713 /* 3714 * Order 1 folios have no space for a deferred list, but we also 3715 * won't waste much memory by not adding them to the deferred list. 3716 */ 3717 if (folio_order(folio) <= 1) 3718 return; 3719 3720 if (!partially_mapped && !split_underused_thp) 3721 return; 3722 3723 /* 3724 * Exclude swapcache: originally to avoid a corrupt deferred split 3725 * queue. Nowadays that is fully prevented by mem_cgroup_swapout(); 3726 * but if page reclaim is already handling the same folio, it is 3727 * unnecessary to handle it again in the shrinker, so excluding 3728 * swapcache here may still be a useful optimization. 3729 */ 3730 if (folio_test_swapcache(folio)) 3731 return; 3732 3733 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 3734 if (partially_mapped) { 3735 if (!folio_test_partially_mapped(folio)) { 3736 folio_set_partially_mapped(folio); 3737 if (folio_test_pmd_mappable(folio)) 3738 count_vm_event(THP_DEFERRED_SPLIT_PAGE); 3739 count_mthp_stat(folio_order(folio), MTHP_STAT_SPLIT_DEFERRED); 3740 mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, 1); 3741 3742 } 3743 } else { 3744 /* partially mapped folios cannot become non-partially mapped */ 3745 VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio); 3746 } 3747 if (list_empty(&folio->_deferred_list)) { 3748 list_add_tail(&folio->_deferred_list, &ds_queue->split_queue); 3749 ds_queue->split_queue_len++; 3750 #ifdef CONFIG_MEMCG 3751 if (memcg) 3752 set_shrinker_bit(memcg, folio_nid(folio), 3753 deferred_split_shrinker->id); 3754 #endif 3755 } 3756 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 3757 } 3758 3759 static unsigned long deferred_split_count(struct shrinker *shrink, 3760 struct shrink_control *sc) 3761 { 3762 struct pglist_data *pgdata = NODE_DATA(sc->nid); 3763 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 3764 3765 #ifdef CONFIG_MEMCG 3766 if (sc->memcg) 3767 ds_queue = &sc->memcg->deferred_split_queue; 3768 #endif 3769 return READ_ONCE(ds_queue->split_queue_len); 3770 } 3771 3772 static bool thp_underused(struct folio *folio) 3773 { 3774 int num_zero_pages = 0, num_filled_pages = 0; 3775 void *kaddr; 3776 int i; 3777 3778 if (khugepaged_max_ptes_none == HPAGE_PMD_NR - 1) 3779 return false; 3780 3781 for (i = 0; i < folio_nr_pages(folio); i++) { 3782 kaddr = kmap_local_folio(folio, i * PAGE_SIZE); 3783 if (!memchr_inv(kaddr, 0, PAGE_SIZE)) { 3784 num_zero_pages++; 3785 if (num_zero_pages > khugepaged_max_ptes_none) { 3786 kunmap_local(kaddr); 3787 return true; 3788 } 3789 } else { 3790 /* 3791 * Another path for early exit once the number 3792 * of non-zero filled pages exceeds threshold. 3793 */ 3794 num_filled_pages++; 3795 if (num_filled_pages >= HPAGE_PMD_NR - khugepaged_max_ptes_none) { 3796 kunmap_local(kaddr); 3797 return false; 3798 } 3799 } 3800 kunmap_local(kaddr); 3801 } 3802 return false; 3803 } 3804 3805 static unsigned long deferred_split_scan(struct shrinker *shrink, 3806 struct shrink_control *sc) 3807 { 3808 struct pglist_data *pgdata = NODE_DATA(sc->nid); 3809 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 3810 unsigned long flags; 3811 LIST_HEAD(list); 3812 struct folio *folio, *next, *prev = NULL; 3813 int split = 0, removed = 0; 3814 3815 #ifdef CONFIG_MEMCG 3816 if (sc->memcg) 3817 ds_queue = &sc->memcg->deferred_split_queue; 3818 #endif 3819 3820 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 3821 /* Take pin on all head pages to avoid freeing them under us */ 3822 list_for_each_entry_safe(folio, next, &ds_queue->split_queue, 3823 _deferred_list) { 3824 if (folio_try_get(folio)) { 3825 list_move(&folio->_deferred_list, &list); 3826 } else { 3827 /* We lost race with folio_put() */ 3828 if (folio_test_partially_mapped(folio)) { 3829 folio_clear_partially_mapped(folio); 3830 mod_mthp_stat(folio_order(folio), 3831 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 3832 } 3833 list_del_init(&folio->_deferred_list); 3834 ds_queue->split_queue_len--; 3835 } 3836 if (!--sc->nr_to_scan) 3837 break; 3838 } 3839 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 3840 3841 list_for_each_entry_safe(folio, next, &list, _deferred_list) { 3842 bool did_split = false; 3843 bool underused = false; 3844 3845 if (!folio_test_partially_mapped(folio)) { 3846 underused = thp_underused(folio); 3847 if (!underused) 3848 goto next; 3849 } 3850 if (!folio_trylock(folio)) 3851 goto next; 3852 if (!split_folio(folio)) { 3853 did_split = true; 3854 if (underused) 3855 count_vm_event(THP_UNDERUSED_SPLIT_PAGE); 3856 split++; 3857 } 3858 folio_unlock(folio); 3859 next: 3860 /* 3861 * split_folio() removes folio from list on success. 3862 * Only add back to the queue if folio is partially mapped. 3863 * If thp_underused returns false, or if split_folio fails 3864 * in the case it was underused, then consider it used and 3865 * don't add it back to split_queue. 3866 */ 3867 if (did_split) { 3868 ; /* folio already removed from list */ 3869 } else if (!folio_test_partially_mapped(folio)) { 3870 list_del_init(&folio->_deferred_list); 3871 removed++; 3872 } else { 3873 /* 3874 * That unlocked list_del_init() above would be unsafe, 3875 * unless its folio is separated from any earlier folios 3876 * left on the list (which may be concurrently unqueued) 3877 * by one safe folio with refcount still raised. 3878 */ 3879 swap(folio, prev); 3880 } 3881 if (folio) 3882 folio_put(folio); 3883 } 3884 3885 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 3886 list_splice_tail(&list, &ds_queue->split_queue); 3887 ds_queue->split_queue_len -= removed; 3888 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 3889 3890 if (prev) 3891 folio_put(prev); 3892 3893 /* 3894 * Stop shrinker if we didn't split any page, but the queue is empty. 3895 * This can happen if pages were freed under us. 3896 */ 3897 if (!split && list_empty(&ds_queue->split_queue)) 3898 return SHRINK_STOP; 3899 return split; 3900 } 3901 3902 #ifdef CONFIG_DEBUG_FS 3903 static void split_huge_pages_all(void) 3904 { 3905 struct zone *zone; 3906 struct page *page; 3907 struct folio *folio; 3908 unsigned long pfn, max_zone_pfn; 3909 unsigned long total = 0, split = 0; 3910 3911 pr_debug("Split all THPs\n"); 3912 for_each_zone(zone) { 3913 if (!managed_zone(zone)) 3914 continue; 3915 max_zone_pfn = zone_end_pfn(zone); 3916 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 3917 int nr_pages; 3918 3919 page = pfn_to_online_page(pfn); 3920 if (!page || PageTail(page)) 3921 continue; 3922 folio = page_folio(page); 3923 if (!folio_try_get(folio)) 3924 continue; 3925 3926 if (unlikely(page_folio(page) != folio)) 3927 goto next; 3928 3929 if (zone != folio_zone(folio)) 3930 goto next; 3931 3932 if (!folio_test_large(folio) 3933 || folio_test_hugetlb(folio) 3934 || !folio_test_lru(folio)) 3935 goto next; 3936 3937 total++; 3938 folio_lock(folio); 3939 nr_pages = folio_nr_pages(folio); 3940 if (!split_folio(folio)) 3941 split++; 3942 pfn += nr_pages - 1; 3943 folio_unlock(folio); 3944 next: 3945 folio_put(folio); 3946 cond_resched(); 3947 } 3948 } 3949 3950 pr_debug("%lu of %lu THP split\n", split, total); 3951 } 3952 3953 static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma) 3954 { 3955 return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) || 3956 is_vm_hugetlb_page(vma); 3957 } 3958 3959 static int split_huge_pages_pid(int pid, unsigned long vaddr_start, 3960 unsigned long vaddr_end, unsigned int new_order) 3961 { 3962 int ret = 0; 3963 struct task_struct *task; 3964 struct mm_struct *mm; 3965 unsigned long total = 0, split = 0; 3966 unsigned long addr; 3967 3968 vaddr_start &= PAGE_MASK; 3969 vaddr_end &= PAGE_MASK; 3970 3971 task = find_get_task_by_vpid(pid); 3972 if (!task) { 3973 ret = -ESRCH; 3974 goto out; 3975 } 3976 3977 /* Find the mm_struct */ 3978 mm = get_task_mm(task); 3979 put_task_struct(task); 3980 3981 if (!mm) { 3982 ret = -EINVAL; 3983 goto out; 3984 } 3985 3986 pr_debug("Split huge pages in pid: %d, vaddr: [0x%lx - 0x%lx]\n", 3987 pid, vaddr_start, vaddr_end); 3988 3989 mmap_read_lock(mm); 3990 /* 3991 * always increase addr by PAGE_SIZE, since we could have a PTE page 3992 * table filled with PTE-mapped THPs, each of which is distinct. 3993 */ 3994 for (addr = vaddr_start; addr < vaddr_end; addr += PAGE_SIZE) { 3995 struct vm_area_struct *vma = vma_lookup(mm, addr); 3996 struct folio_walk fw; 3997 struct folio *folio; 3998 struct address_space *mapping; 3999 unsigned int target_order = new_order; 4000 4001 if (!vma) 4002 break; 4003 4004 /* skip special VMA and hugetlb VMA */ 4005 if (vma_not_suitable_for_thp_split(vma)) { 4006 addr = vma->vm_end; 4007 continue; 4008 } 4009 4010 folio = folio_walk_start(&fw, vma, addr, 0); 4011 if (!folio) 4012 continue; 4013 4014 if (!is_transparent_hugepage(folio)) 4015 goto next; 4016 4017 if (!folio_test_anon(folio)) { 4018 mapping = folio->mapping; 4019 target_order = max(new_order, 4020 mapping_min_folio_order(mapping)); 4021 } 4022 4023 if (target_order >= folio_order(folio)) 4024 goto next; 4025 4026 total++; 4027 /* 4028 * For folios with private, split_huge_page_to_list_to_order() 4029 * will try to drop it before split and then check if the folio 4030 * can be split or not. So skip the check here. 4031 */ 4032 if (!folio_test_private(folio) && 4033 !can_split_folio(folio, 0, NULL)) 4034 goto next; 4035 4036 if (!folio_trylock(folio)) 4037 goto next; 4038 folio_get(folio); 4039 folio_walk_end(&fw, vma); 4040 4041 if (!folio_test_anon(folio) && folio->mapping != mapping) 4042 goto unlock; 4043 4044 if (!split_folio_to_order(folio, target_order)) 4045 split++; 4046 4047 unlock: 4048 4049 folio_unlock(folio); 4050 folio_put(folio); 4051 4052 cond_resched(); 4053 continue; 4054 next: 4055 folio_walk_end(&fw, vma); 4056 cond_resched(); 4057 } 4058 mmap_read_unlock(mm); 4059 mmput(mm); 4060 4061 pr_debug("%lu of %lu THP split\n", split, total); 4062 4063 out: 4064 return ret; 4065 } 4066 4067 static int split_huge_pages_in_file(const char *file_path, pgoff_t off_start, 4068 pgoff_t off_end, unsigned int new_order) 4069 { 4070 struct filename *file; 4071 struct file *candidate; 4072 struct address_space *mapping; 4073 int ret = -EINVAL; 4074 pgoff_t index; 4075 int nr_pages = 1; 4076 unsigned long total = 0, split = 0; 4077 unsigned int min_order; 4078 unsigned int target_order; 4079 4080 file = getname_kernel(file_path); 4081 if (IS_ERR(file)) 4082 return ret; 4083 4084 candidate = file_open_name(file, O_RDONLY, 0); 4085 if (IS_ERR(candidate)) 4086 goto out; 4087 4088 pr_debug("split file-backed THPs in file: %s, page offset: [0x%lx - 0x%lx]\n", 4089 file_path, off_start, off_end); 4090 4091 mapping = candidate->f_mapping; 4092 min_order = mapping_min_folio_order(mapping); 4093 target_order = max(new_order, min_order); 4094 4095 for (index = off_start; index < off_end; index += nr_pages) { 4096 struct folio *folio = filemap_get_folio(mapping, index); 4097 4098 nr_pages = 1; 4099 if (IS_ERR(folio)) 4100 continue; 4101 4102 if (!folio_test_large(folio)) 4103 goto next; 4104 4105 total++; 4106 nr_pages = folio_nr_pages(folio); 4107 4108 if (target_order >= folio_order(folio)) 4109 goto next; 4110 4111 if (!folio_trylock(folio)) 4112 goto next; 4113 4114 if (folio->mapping != mapping) 4115 goto unlock; 4116 4117 if (!split_folio_to_order(folio, target_order)) 4118 split++; 4119 4120 unlock: 4121 folio_unlock(folio); 4122 next: 4123 folio_put(folio); 4124 cond_resched(); 4125 } 4126 4127 filp_close(candidate, NULL); 4128 ret = 0; 4129 4130 pr_debug("%lu of %lu file-backed THP split\n", split, total); 4131 out: 4132 putname(file); 4133 return ret; 4134 } 4135 4136 #define MAX_INPUT_BUF_SZ 255 4137 4138 static ssize_t split_huge_pages_write(struct file *file, const char __user *buf, 4139 size_t count, loff_t *ppops) 4140 { 4141 static DEFINE_MUTEX(split_debug_mutex); 4142 ssize_t ret; 4143 /* 4144 * hold pid, start_vaddr, end_vaddr, new_order or 4145 * file_path, off_start, off_end, new_order 4146 */ 4147 char input_buf[MAX_INPUT_BUF_SZ]; 4148 int pid; 4149 unsigned long vaddr_start, vaddr_end; 4150 unsigned int new_order = 0; 4151 4152 ret = mutex_lock_interruptible(&split_debug_mutex); 4153 if (ret) 4154 return ret; 4155 4156 ret = -EFAULT; 4157 4158 memset(input_buf, 0, MAX_INPUT_BUF_SZ); 4159 if (copy_from_user(input_buf, buf, min_t(size_t, count, MAX_INPUT_BUF_SZ))) 4160 goto out; 4161 4162 input_buf[MAX_INPUT_BUF_SZ - 1] = '\0'; 4163 4164 if (input_buf[0] == '/') { 4165 char *tok; 4166 char *buf = input_buf; 4167 char file_path[MAX_INPUT_BUF_SZ]; 4168 pgoff_t off_start = 0, off_end = 0; 4169 size_t input_len = strlen(input_buf); 4170 4171 tok = strsep(&buf, ","); 4172 if (tok && buf) { 4173 strscpy(file_path, tok); 4174 } else { 4175 ret = -EINVAL; 4176 goto out; 4177 } 4178 4179 ret = sscanf(buf, "0x%lx,0x%lx,%d", &off_start, &off_end, &new_order); 4180 if (ret != 2 && ret != 3) { 4181 ret = -EINVAL; 4182 goto out; 4183 } 4184 ret = split_huge_pages_in_file(file_path, off_start, off_end, new_order); 4185 if (!ret) 4186 ret = input_len; 4187 4188 goto out; 4189 } 4190 4191 ret = sscanf(input_buf, "%d,0x%lx,0x%lx,%d", &pid, &vaddr_start, &vaddr_end, &new_order); 4192 if (ret == 1 && pid == 1) { 4193 split_huge_pages_all(); 4194 ret = strlen(input_buf); 4195 goto out; 4196 } else if (ret != 3 && ret != 4) { 4197 ret = -EINVAL; 4198 goto out; 4199 } 4200 4201 ret = split_huge_pages_pid(pid, vaddr_start, vaddr_end, new_order); 4202 if (!ret) 4203 ret = strlen(input_buf); 4204 out: 4205 mutex_unlock(&split_debug_mutex); 4206 return ret; 4207 4208 } 4209 4210 static const struct file_operations split_huge_pages_fops = { 4211 .owner = THIS_MODULE, 4212 .write = split_huge_pages_write, 4213 }; 4214 4215 static int __init split_huge_pages_debugfs(void) 4216 { 4217 debugfs_create_file("split_huge_pages", 0200, NULL, NULL, 4218 &split_huge_pages_fops); 4219 return 0; 4220 } 4221 late_initcall(split_huge_pages_debugfs); 4222 #endif 4223 4224 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 4225 int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 4226 struct page *page) 4227 { 4228 struct folio *folio = page_folio(page); 4229 struct vm_area_struct *vma = pvmw->vma; 4230 struct mm_struct *mm = vma->vm_mm; 4231 unsigned long address = pvmw->address; 4232 bool anon_exclusive; 4233 pmd_t pmdval; 4234 swp_entry_t entry; 4235 pmd_t pmdswp; 4236 4237 if (!(pvmw->pmd && !pvmw->pte)) 4238 return 0; 4239 4240 flush_cache_range(vma, address, address + HPAGE_PMD_SIZE); 4241 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); 4242 4243 /* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. */ 4244 anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(page); 4245 if (anon_exclusive && folio_try_share_anon_rmap_pmd(folio, page)) { 4246 set_pmd_at(mm, address, pvmw->pmd, pmdval); 4247 return -EBUSY; 4248 } 4249 4250 if (pmd_dirty(pmdval)) 4251 folio_mark_dirty(folio); 4252 if (pmd_write(pmdval)) 4253 entry = make_writable_migration_entry(page_to_pfn(page)); 4254 else if (anon_exclusive) 4255 entry = make_readable_exclusive_migration_entry(page_to_pfn(page)); 4256 else 4257 entry = make_readable_migration_entry(page_to_pfn(page)); 4258 if (pmd_young(pmdval)) 4259 entry = make_migration_entry_young(entry); 4260 if (pmd_dirty(pmdval)) 4261 entry = make_migration_entry_dirty(entry); 4262 pmdswp = swp_entry_to_pmd(entry); 4263 if (pmd_soft_dirty(pmdval)) 4264 pmdswp = pmd_swp_mksoft_dirty(pmdswp); 4265 if (pmd_uffd_wp(pmdval)) 4266 pmdswp = pmd_swp_mkuffd_wp(pmdswp); 4267 set_pmd_at(mm, address, pvmw->pmd, pmdswp); 4268 folio_remove_rmap_pmd(folio, page, vma); 4269 folio_put(folio); 4270 trace_set_migration_pmd(address, pmd_val(pmdswp)); 4271 4272 return 0; 4273 } 4274 4275 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) 4276 { 4277 struct folio *folio = page_folio(new); 4278 struct vm_area_struct *vma = pvmw->vma; 4279 struct mm_struct *mm = vma->vm_mm; 4280 unsigned long address = pvmw->address; 4281 unsigned long haddr = address & HPAGE_PMD_MASK; 4282 pmd_t pmde; 4283 swp_entry_t entry; 4284 4285 if (!(pvmw->pmd && !pvmw->pte)) 4286 return; 4287 4288 entry = pmd_to_swp_entry(*pvmw->pmd); 4289 folio_get(folio); 4290 pmde = mk_huge_pmd(new, READ_ONCE(vma->vm_page_prot)); 4291 if (pmd_swp_soft_dirty(*pvmw->pmd)) 4292 pmde = pmd_mksoft_dirty(pmde); 4293 if (is_writable_migration_entry(entry)) 4294 pmde = pmd_mkwrite(pmde, vma); 4295 if (pmd_swp_uffd_wp(*pvmw->pmd)) 4296 pmde = pmd_mkuffd_wp(pmde); 4297 if (!is_migration_entry_young(entry)) 4298 pmde = pmd_mkold(pmde); 4299 /* NOTE: this may contain setting soft-dirty on some archs */ 4300 if (folio_test_dirty(folio) && is_migration_entry_dirty(entry)) 4301 pmde = pmd_mkdirty(pmde); 4302 4303 if (folio_test_anon(folio)) { 4304 rmap_t rmap_flags = RMAP_NONE; 4305 4306 if (!is_readable_migration_entry(entry)) 4307 rmap_flags |= RMAP_EXCLUSIVE; 4308 4309 folio_add_anon_rmap_pmd(folio, new, vma, haddr, rmap_flags); 4310 } else { 4311 folio_add_file_rmap_pmd(folio, new, vma); 4312 } 4313 VM_BUG_ON(pmd_write(pmde) && folio_test_anon(folio) && !PageAnonExclusive(new)); 4314 set_pmd_at(mm, haddr, pvmw->pmd, pmde); 4315 4316 /* No need to invalidate - it was non-present before */ 4317 update_mmu_cache_pmd(vma, address, pvmw->pmd); 4318 trace_remove_migration_pmd(address, pmd_val(pmde)); 4319 } 4320 #endif 4321