1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2009 Red Hat, Inc. 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/mm.h> 9 #include <linux/sched.h> 10 #include <linux/sched/mm.h> 11 #include <linux/sched/coredump.h> 12 #include <linux/sched/numa_balancing.h> 13 #include <linux/highmem.h> 14 #include <linux/hugetlb.h> 15 #include <linux/mmu_notifier.h> 16 #include <linux/rmap.h> 17 #include <linux/swap.h> 18 #include <linux/shrinker.h> 19 #include <linux/mm_inline.h> 20 #include <linux/swapops.h> 21 #include <linux/backing-dev.h> 22 #include <linux/dax.h> 23 #include <linux/mm_types.h> 24 #include <linux/khugepaged.h> 25 #include <linux/freezer.h> 26 #include <linux/pfn_t.h> 27 #include <linux/mman.h> 28 #include <linux/memremap.h> 29 #include <linux/pagemap.h> 30 #include <linux/debugfs.h> 31 #include <linux/migrate.h> 32 #include <linux/hashtable.h> 33 #include <linux/userfaultfd_k.h> 34 #include <linux/page_idle.h> 35 #include <linux/shmem_fs.h> 36 #include <linux/oom.h> 37 #include <linux/numa.h> 38 #include <linux/page_owner.h> 39 #include <linux/sched/sysctl.h> 40 #include <linux/memory-tiers.h> 41 #include <linux/compat.h> 42 #include <linux/pgalloc_tag.h> 43 #include <linux/pagewalk.h> 44 45 #include <asm/tlb.h> 46 #include <asm/pgalloc.h> 47 #include "internal.h" 48 #include "swap.h" 49 50 #define CREATE_TRACE_POINTS 51 #include <trace/events/thp.h> 52 53 /* 54 * By default, transparent hugepage support is disabled in order to avoid 55 * risking an increased memory footprint for applications that are not 56 * guaranteed to benefit from it. When transparent hugepage support is 57 * enabled, it is for all mappings, and khugepaged scans all mappings. 58 * Defrag is invoked by khugepaged hugepage allocations and by page faults 59 * for all hugepage allocations. 60 */ 61 unsigned long transparent_hugepage_flags __read_mostly = 62 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS 63 (1<<TRANSPARENT_HUGEPAGE_FLAG)| 64 #endif 65 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE 66 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| 67 #endif 68 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)| 69 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| 70 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 71 72 static struct shrinker *deferred_split_shrinker; 73 static unsigned long deferred_split_count(struct shrinker *shrink, 74 struct shrink_control *sc); 75 static unsigned long deferred_split_scan(struct shrinker *shrink, 76 struct shrink_control *sc); 77 static bool split_underused_thp = true; 78 79 static atomic_t huge_zero_refcount; 80 struct folio *huge_zero_folio __read_mostly; 81 unsigned long huge_zero_pfn __read_mostly = ~0UL; 82 unsigned long huge_anon_orders_always __read_mostly; 83 unsigned long huge_anon_orders_madvise __read_mostly; 84 unsigned long huge_anon_orders_inherit __read_mostly; 85 static bool anon_orders_configured __initdata; 86 87 unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, 88 unsigned long vm_flags, 89 unsigned long tva_flags, 90 unsigned long orders) 91 { 92 bool smaps = tva_flags & TVA_SMAPS; 93 bool in_pf = tva_flags & TVA_IN_PF; 94 bool enforce_sysfs = tva_flags & TVA_ENFORCE_SYSFS; 95 unsigned long supported_orders; 96 97 /* Check the intersection of requested and supported orders. */ 98 if (vma_is_anonymous(vma)) 99 supported_orders = THP_ORDERS_ALL_ANON; 100 else if (vma_is_dax(vma)) 101 supported_orders = THP_ORDERS_ALL_FILE_DAX; 102 else 103 supported_orders = THP_ORDERS_ALL_FILE_DEFAULT; 104 105 orders &= supported_orders; 106 if (!orders) 107 return 0; 108 109 if (!vma->vm_mm) /* vdso */ 110 return 0; 111 112 /* 113 * Explicitly disabled through madvise or prctl, or some 114 * architectures may disable THP for some mappings, for 115 * example, s390 kvm. 116 * */ 117 if ((vm_flags & VM_NOHUGEPAGE) || 118 test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags)) 119 return 0; 120 /* 121 * If the hardware/firmware marked hugepage support disabled. 122 */ 123 if (transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED)) 124 return 0; 125 126 /* khugepaged doesn't collapse DAX vma, but page fault is fine. */ 127 if (vma_is_dax(vma)) 128 return in_pf ? orders : 0; 129 130 /* 131 * khugepaged special VMA and hugetlb VMA. 132 * Must be checked after dax since some dax mappings may have 133 * VM_MIXEDMAP set. 134 */ 135 if (!in_pf && !smaps && (vm_flags & VM_NO_KHUGEPAGED)) 136 return 0; 137 138 /* 139 * Check alignment for file vma and size for both file and anon vma by 140 * filtering out the unsuitable orders. 141 * 142 * Skip the check for page fault. Huge fault does the check in fault 143 * handlers. 144 */ 145 if (!in_pf) { 146 int order = highest_order(orders); 147 unsigned long addr; 148 149 while (orders) { 150 addr = vma->vm_end - (PAGE_SIZE << order); 151 if (thp_vma_suitable_order(vma, addr, order)) 152 break; 153 order = next_order(&orders, order); 154 } 155 156 if (!orders) 157 return 0; 158 } 159 160 /* 161 * Enabled via shmem mount options or sysfs settings. 162 * Must be done before hugepage flags check since shmem has its 163 * own flags. 164 */ 165 if (!in_pf && shmem_file(vma->vm_file)) 166 return shmem_allowable_huge_orders(file_inode(vma->vm_file), 167 vma, vma->vm_pgoff, 168 !enforce_sysfs); 169 170 if (!vma_is_anonymous(vma)) { 171 /* 172 * Enforce sysfs THP requirements as necessary. Anonymous vmas 173 * were already handled in thp_vma_allowable_orders(). 174 */ 175 if (enforce_sysfs && 176 (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) && 177 !hugepage_global_always()))) 178 return 0; 179 180 /* 181 * Trust that ->huge_fault() handlers know what they are doing 182 * in fault path. 183 */ 184 if (((in_pf || smaps)) && vma->vm_ops->huge_fault) 185 return orders; 186 /* Only regular file is valid in collapse path */ 187 if (((!in_pf || smaps)) && file_thp_enabled(vma)) 188 return orders; 189 return 0; 190 } 191 192 if (vma_is_temporary_stack(vma)) 193 return 0; 194 195 /* 196 * THPeligible bit of smaps should show 1 for proper VMAs even 197 * though anon_vma is not initialized yet. 198 * 199 * Allow page fault since anon_vma may be not initialized until 200 * the first page fault. 201 */ 202 if (!vma->anon_vma) 203 return (smaps || in_pf) ? orders : 0; 204 205 return orders; 206 } 207 208 static bool get_huge_zero_page(void) 209 { 210 struct folio *zero_folio; 211 retry: 212 if (likely(atomic_inc_not_zero(&huge_zero_refcount))) 213 return true; 214 215 zero_folio = folio_alloc((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE, 216 HPAGE_PMD_ORDER); 217 if (!zero_folio) { 218 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); 219 return false; 220 } 221 preempt_disable(); 222 if (cmpxchg(&huge_zero_folio, NULL, zero_folio)) { 223 preempt_enable(); 224 folio_put(zero_folio); 225 goto retry; 226 } 227 WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio)); 228 229 /* We take additional reference here. It will be put back by shrinker */ 230 atomic_set(&huge_zero_refcount, 2); 231 preempt_enable(); 232 count_vm_event(THP_ZERO_PAGE_ALLOC); 233 return true; 234 } 235 236 static void put_huge_zero_page(void) 237 { 238 /* 239 * Counter should never go to zero here. Only shrinker can put 240 * last reference. 241 */ 242 BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); 243 } 244 245 struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) 246 { 247 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 248 return READ_ONCE(huge_zero_folio); 249 250 if (!get_huge_zero_page()) 251 return NULL; 252 253 if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 254 put_huge_zero_page(); 255 256 return READ_ONCE(huge_zero_folio); 257 } 258 259 void mm_put_huge_zero_folio(struct mm_struct *mm) 260 { 261 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 262 put_huge_zero_page(); 263 } 264 265 static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink, 266 struct shrink_control *sc) 267 { 268 /* we can free zero page only if last reference remains */ 269 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; 270 } 271 272 static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink, 273 struct shrink_control *sc) 274 { 275 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { 276 struct folio *zero_folio = xchg(&huge_zero_folio, NULL); 277 BUG_ON(zero_folio == NULL); 278 WRITE_ONCE(huge_zero_pfn, ~0UL); 279 folio_put(zero_folio); 280 return HPAGE_PMD_NR; 281 } 282 283 return 0; 284 } 285 286 static struct shrinker *huge_zero_page_shrinker; 287 288 #ifdef CONFIG_SYSFS 289 static ssize_t enabled_show(struct kobject *kobj, 290 struct kobj_attribute *attr, char *buf) 291 { 292 const char *output; 293 294 if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags)) 295 output = "[always] madvise never"; 296 else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 297 &transparent_hugepage_flags)) 298 output = "always [madvise] never"; 299 else 300 output = "always madvise [never]"; 301 302 return sysfs_emit(buf, "%s\n", output); 303 } 304 305 static ssize_t enabled_store(struct kobject *kobj, 306 struct kobj_attribute *attr, 307 const char *buf, size_t count) 308 { 309 ssize_t ret = count; 310 311 if (sysfs_streq(buf, "always")) { 312 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 313 set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 314 } else if (sysfs_streq(buf, "madvise")) { 315 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 316 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 317 } else if (sysfs_streq(buf, "never")) { 318 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 319 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 320 } else 321 ret = -EINVAL; 322 323 if (ret > 0) { 324 int err = start_stop_khugepaged(); 325 if (err) 326 ret = err; 327 } 328 return ret; 329 } 330 331 static struct kobj_attribute enabled_attr = __ATTR_RW(enabled); 332 333 ssize_t single_hugepage_flag_show(struct kobject *kobj, 334 struct kobj_attribute *attr, char *buf, 335 enum transparent_hugepage_flag flag) 336 { 337 return sysfs_emit(buf, "%d\n", 338 !!test_bit(flag, &transparent_hugepage_flags)); 339 } 340 341 ssize_t single_hugepage_flag_store(struct kobject *kobj, 342 struct kobj_attribute *attr, 343 const char *buf, size_t count, 344 enum transparent_hugepage_flag flag) 345 { 346 unsigned long value; 347 int ret; 348 349 ret = kstrtoul(buf, 10, &value); 350 if (ret < 0) 351 return ret; 352 if (value > 1) 353 return -EINVAL; 354 355 if (value) 356 set_bit(flag, &transparent_hugepage_flags); 357 else 358 clear_bit(flag, &transparent_hugepage_flags); 359 360 return count; 361 } 362 363 static ssize_t defrag_show(struct kobject *kobj, 364 struct kobj_attribute *attr, char *buf) 365 { 366 const char *output; 367 368 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, 369 &transparent_hugepage_flags)) 370 output = "[always] defer defer+madvise madvise never"; 371 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, 372 &transparent_hugepage_flags)) 373 output = "always [defer] defer+madvise madvise never"; 374 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, 375 &transparent_hugepage_flags)) 376 output = "always defer [defer+madvise] madvise never"; 377 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, 378 &transparent_hugepage_flags)) 379 output = "always defer defer+madvise [madvise] never"; 380 else 381 output = "always defer defer+madvise madvise [never]"; 382 383 return sysfs_emit(buf, "%s\n", output); 384 } 385 386 static ssize_t defrag_store(struct kobject *kobj, 387 struct kobj_attribute *attr, 388 const char *buf, size_t count) 389 { 390 if (sysfs_streq(buf, "always")) { 391 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 392 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 393 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 394 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 395 } else if (sysfs_streq(buf, "defer+madvise")) { 396 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 397 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 398 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 399 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 400 } else if (sysfs_streq(buf, "defer")) { 401 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 402 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 403 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 404 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 405 } else if (sysfs_streq(buf, "madvise")) { 406 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 407 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 408 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 409 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 410 } else if (sysfs_streq(buf, "never")) { 411 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 412 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 413 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 414 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 415 } else 416 return -EINVAL; 417 418 return count; 419 } 420 static struct kobj_attribute defrag_attr = __ATTR_RW(defrag); 421 422 static ssize_t use_zero_page_show(struct kobject *kobj, 423 struct kobj_attribute *attr, char *buf) 424 { 425 return single_hugepage_flag_show(kobj, attr, buf, 426 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 427 } 428 static ssize_t use_zero_page_store(struct kobject *kobj, 429 struct kobj_attribute *attr, const char *buf, size_t count) 430 { 431 return single_hugepage_flag_store(kobj, attr, buf, count, 432 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 433 } 434 static struct kobj_attribute use_zero_page_attr = __ATTR_RW(use_zero_page); 435 436 static ssize_t hpage_pmd_size_show(struct kobject *kobj, 437 struct kobj_attribute *attr, char *buf) 438 { 439 return sysfs_emit(buf, "%lu\n", HPAGE_PMD_SIZE); 440 } 441 static struct kobj_attribute hpage_pmd_size_attr = 442 __ATTR_RO(hpage_pmd_size); 443 444 static ssize_t split_underused_thp_show(struct kobject *kobj, 445 struct kobj_attribute *attr, char *buf) 446 { 447 return sysfs_emit(buf, "%d\n", split_underused_thp); 448 } 449 450 static ssize_t split_underused_thp_store(struct kobject *kobj, 451 struct kobj_attribute *attr, 452 const char *buf, size_t count) 453 { 454 int err = kstrtobool(buf, &split_underused_thp); 455 456 if (err < 0) 457 return err; 458 459 return count; 460 } 461 462 static struct kobj_attribute split_underused_thp_attr = __ATTR( 463 shrink_underused, 0644, split_underused_thp_show, split_underused_thp_store); 464 465 static struct attribute *hugepage_attr[] = { 466 &enabled_attr.attr, 467 &defrag_attr.attr, 468 &use_zero_page_attr.attr, 469 &hpage_pmd_size_attr.attr, 470 #ifdef CONFIG_SHMEM 471 &shmem_enabled_attr.attr, 472 #endif 473 &split_underused_thp_attr.attr, 474 NULL, 475 }; 476 477 static const struct attribute_group hugepage_attr_group = { 478 .attrs = hugepage_attr, 479 }; 480 481 static void hugepage_exit_sysfs(struct kobject *hugepage_kobj); 482 static void thpsize_release(struct kobject *kobj); 483 static DEFINE_SPINLOCK(huge_anon_orders_lock); 484 static LIST_HEAD(thpsize_list); 485 486 static ssize_t anon_enabled_show(struct kobject *kobj, 487 struct kobj_attribute *attr, char *buf) 488 { 489 int order = to_thpsize(kobj)->order; 490 const char *output; 491 492 if (test_bit(order, &huge_anon_orders_always)) 493 output = "[always] inherit madvise never"; 494 else if (test_bit(order, &huge_anon_orders_inherit)) 495 output = "always [inherit] madvise never"; 496 else if (test_bit(order, &huge_anon_orders_madvise)) 497 output = "always inherit [madvise] never"; 498 else 499 output = "always inherit madvise [never]"; 500 501 return sysfs_emit(buf, "%s\n", output); 502 } 503 504 static ssize_t anon_enabled_store(struct kobject *kobj, 505 struct kobj_attribute *attr, 506 const char *buf, size_t count) 507 { 508 int order = to_thpsize(kobj)->order; 509 ssize_t ret = count; 510 511 if (sysfs_streq(buf, "always")) { 512 spin_lock(&huge_anon_orders_lock); 513 clear_bit(order, &huge_anon_orders_inherit); 514 clear_bit(order, &huge_anon_orders_madvise); 515 set_bit(order, &huge_anon_orders_always); 516 spin_unlock(&huge_anon_orders_lock); 517 } else if (sysfs_streq(buf, "inherit")) { 518 spin_lock(&huge_anon_orders_lock); 519 clear_bit(order, &huge_anon_orders_always); 520 clear_bit(order, &huge_anon_orders_madvise); 521 set_bit(order, &huge_anon_orders_inherit); 522 spin_unlock(&huge_anon_orders_lock); 523 } else if (sysfs_streq(buf, "madvise")) { 524 spin_lock(&huge_anon_orders_lock); 525 clear_bit(order, &huge_anon_orders_always); 526 clear_bit(order, &huge_anon_orders_inherit); 527 set_bit(order, &huge_anon_orders_madvise); 528 spin_unlock(&huge_anon_orders_lock); 529 } else if (sysfs_streq(buf, "never")) { 530 spin_lock(&huge_anon_orders_lock); 531 clear_bit(order, &huge_anon_orders_always); 532 clear_bit(order, &huge_anon_orders_inherit); 533 clear_bit(order, &huge_anon_orders_madvise); 534 spin_unlock(&huge_anon_orders_lock); 535 } else 536 ret = -EINVAL; 537 538 if (ret > 0) { 539 int err; 540 541 err = start_stop_khugepaged(); 542 if (err) 543 ret = err; 544 } 545 return ret; 546 } 547 548 static struct kobj_attribute anon_enabled_attr = 549 __ATTR(enabled, 0644, anon_enabled_show, anon_enabled_store); 550 551 static struct attribute *anon_ctrl_attrs[] = { 552 &anon_enabled_attr.attr, 553 NULL, 554 }; 555 556 static const struct attribute_group anon_ctrl_attr_grp = { 557 .attrs = anon_ctrl_attrs, 558 }; 559 560 static struct attribute *file_ctrl_attrs[] = { 561 #ifdef CONFIG_SHMEM 562 &thpsize_shmem_enabled_attr.attr, 563 #endif 564 NULL, 565 }; 566 567 static const struct attribute_group file_ctrl_attr_grp = { 568 .attrs = file_ctrl_attrs, 569 }; 570 571 static struct attribute *any_ctrl_attrs[] = { 572 NULL, 573 }; 574 575 static const struct attribute_group any_ctrl_attr_grp = { 576 .attrs = any_ctrl_attrs, 577 }; 578 579 static const struct kobj_type thpsize_ktype = { 580 .release = &thpsize_release, 581 .sysfs_ops = &kobj_sysfs_ops, 582 }; 583 584 DEFINE_PER_CPU(struct mthp_stat, mthp_stats) = {{{0}}}; 585 586 static unsigned long sum_mthp_stat(int order, enum mthp_stat_item item) 587 { 588 unsigned long sum = 0; 589 int cpu; 590 591 for_each_possible_cpu(cpu) { 592 struct mthp_stat *this = &per_cpu(mthp_stats, cpu); 593 594 sum += this->stats[order][item]; 595 } 596 597 return sum; 598 } 599 600 #define DEFINE_MTHP_STAT_ATTR(_name, _index) \ 601 static ssize_t _name##_show(struct kobject *kobj, \ 602 struct kobj_attribute *attr, char *buf) \ 603 { \ 604 int order = to_thpsize(kobj)->order; \ 605 \ 606 return sysfs_emit(buf, "%lu\n", sum_mthp_stat(order, _index)); \ 607 } \ 608 static struct kobj_attribute _name##_attr = __ATTR_RO(_name) 609 610 DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC); 611 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK); 612 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); 613 DEFINE_MTHP_STAT_ATTR(swpout, MTHP_STAT_SWPOUT); 614 DEFINE_MTHP_STAT_ATTR(swpout_fallback, MTHP_STAT_SWPOUT_FALLBACK); 615 #ifdef CONFIG_SHMEM 616 DEFINE_MTHP_STAT_ATTR(shmem_alloc, MTHP_STAT_SHMEM_ALLOC); 617 DEFINE_MTHP_STAT_ATTR(shmem_fallback, MTHP_STAT_SHMEM_FALLBACK); 618 DEFINE_MTHP_STAT_ATTR(shmem_fallback_charge, MTHP_STAT_SHMEM_FALLBACK_CHARGE); 619 #endif 620 DEFINE_MTHP_STAT_ATTR(split, MTHP_STAT_SPLIT); 621 DEFINE_MTHP_STAT_ATTR(split_failed, MTHP_STAT_SPLIT_FAILED); 622 DEFINE_MTHP_STAT_ATTR(split_deferred, MTHP_STAT_SPLIT_DEFERRED); 623 DEFINE_MTHP_STAT_ATTR(nr_anon, MTHP_STAT_NR_ANON); 624 DEFINE_MTHP_STAT_ATTR(nr_anon_partially_mapped, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED); 625 626 static struct attribute *anon_stats_attrs[] = { 627 &anon_fault_alloc_attr.attr, 628 &anon_fault_fallback_attr.attr, 629 &anon_fault_fallback_charge_attr.attr, 630 #ifndef CONFIG_SHMEM 631 &swpout_attr.attr, 632 &swpout_fallback_attr.attr, 633 #endif 634 &split_deferred_attr.attr, 635 &nr_anon_attr.attr, 636 &nr_anon_partially_mapped_attr.attr, 637 NULL, 638 }; 639 640 static struct attribute_group anon_stats_attr_grp = { 641 .name = "stats", 642 .attrs = anon_stats_attrs, 643 }; 644 645 static struct attribute *file_stats_attrs[] = { 646 #ifdef CONFIG_SHMEM 647 &shmem_alloc_attr.attr, 648 &shmem_fallback_attr.attr, 649 &shmem_fallback_charge_attr.attr, 650 #endif 651 NULL, 652 }; 653 654 static struct attribute_group file_stats_attr_grp = { 655 .name = "stats", 656 .attrs = file_stats_attrs, 657 }; 658 659 static struct attribute *any_stats_attrs[] = { 660 #ifdef CONFIG_SHMEM 661 &swpout_attr.attr, 662 &swpout_fallback_attr.attr, 663 #endif 664 &split_attr.attr, 665 &split_failed_attr.attr, 666 NULL, 667 }; 668 669 static struct attribute_group any_stats_attr_grp = { 670 .name = "stats", 671 .attrs = any_stats_attrs, 672 }; 673 674 static int sysfs_add_group(struct kobject *kobj, 675 const struct attribute_group *grp) 676 { 677 int ret = -ENOENT; 678 679 /* 680 * If the group is named, try to merge first, assuming the subdirectory 681 * was already created. This avoids the warning emitted by 682 * sysfs_create_group() if the directory already exists. 683 */ 684 if (grp->name) 685 ret = sysfs_merge_group(kobj, grp); 686 if (ret) 687 ret = sysfs_create_group(kobj, grp); 688 689 return ret; 690 } 691 692 static struct thpsize *thpsize_create(int order, struct kobject *parent) 693 { 694 unsigned long size = (PAGE_SIZE << order) / SZ_1K; 695 struct thpsize *thpsize; 696 int ret = -ENOMEM; 697 698 thpsize = kzalloc(sizeof(*thpsize), GFP_KERNEL); 699 if (!thpsize) 700 goto err; 701 702 thpsize->order = order; 703 704 ret = kobject_init_and_add(&thpsize->kobj, &thpsize_ktype, parent, 705 "hugepages-%lukB", size); 706 if (ret) { 707 kfree(thpsize); 708 goto err; 709 } 710 711 712 ret = sysfs_add_group(&thpsize->kobj, &any_ctrl_attr_grp); 713 if (ret) 714 goto err_put; 715 716 ret = sysfs_add_group(&thpsize->kobj, &any_stats_attr_grp); 717 if (ret) 718 goto err_put; 719 720 if (BIT(order) & THP_ORDERS_ALL_ANON) { 721 ret = sysfs_add_group(&thpsize->kobj, &anon_ctrl_attr_grp); 722 if (ret) 723 goto err_put; 724 725 ret = sysfs_add_group(&thpsize->kobj, &anon_stats_attr_grp); 726 if (ret) 727 goto err_put; 728 } 729 730 if (BIT(order) & THP_ORDERS_ALL_FILE_DEFAULT) { 731 ret = sysfs_add_group(&thpsize->kobj, &file_ctrl_attr_grp); 732 if (ret) 733 goto err_put; 734 735 ret = sysfs_add_group(&thpsize->kobj, &file_stats_attr_grp); 736 if (ret) 737 goto err_put; 738 } 739 740 return thpsize; 741 err_put: 742 kobject_put(&thpsize->kobj); 743 err: 744 return ERR_PTR(ret); 745 } 746 747 static void thpsize_release(struct kobject *kobj) 748 { 749 kfree(to_thpsize(kobj)); 750 } 751 752 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj) 753 { 754 int err; 755 struct thpsize *thpsize; 756 unsigned long orders; 757 int order; 758 759 /* 760 * Default to setting PMD-sized THP to inherit the global setting and 761 * disable all other sizes. powerpc's PMD_ORDER isn't a compile-time 762 * constant so we have to do this here. 763 */ 764 if (!anon_orders_configured) 765 huge_anon_orders_inherit = BIT(PMD_ORDER); 766 767 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj); 768 if (unlikely(!*hugepage_kobj)) { 769 pr_err("failed to create transparent hugepage kobject\n"); 770 return -ENOMEM; 771 } 772 773 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group); 774 if (err) { 775 pr_err("failed to register transparent hugepage group\n"); 776 goto delete_obj; 777 } 778 779 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group); 780 if (err) { 781 pr_err("failed to register transparent hugepage group\n"); 782 goto remove_hp_group; 783 } 784 785 orders = THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_FILE_DEFAULT; 786 order = highest_order(orders); 787 while (orders) { 788 thpsize = thpsize_create(order, *hugepage_kobj); 789 if (IS_ERR(thpsize)) { 790 pr_err("failed to create thpsize for order %d\n", order); 791 err = PTR_ERR(thpsize); 792 goto remove_all; 793 } 794 list_add(&thpsize->node, &thpsize_list); 795 order = next_order(&orders, order); 796 } 797 798 return 0; 799 800 remove_all: 801 hugepage_exit_sysfs(*hugepage_kobj); 802 return err; 803 remove_hp_group: 804 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group); 805 delete_obj: 806 kobject_put(*hugepage_kobj); 807 return err; 808 } 809 810 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj) 811 { 812 struct thpsize *thpsize, *tmp; 813 814 list_for_each_entry_safe(thpsize, tmp, &thpsize_list, node) { 815 list_del(&thpsize->node); 816 kobject_put(&thpsize->kobj); 817 } 818 819 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group); 820 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group); 821 kobject_put(hugepage_kobj); 822 } 823 #else 824 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj) 825 { 826 return 0; 827 } 828 829 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) 830 { 831 } 832 #endif /* CONFIG_SYSFS */ 833 834 static int __init thp_shrinker_init(void) 835 { 836 huge_zero_page_shrinker = shrinker_alloc(0, "thp-zero"); 837 if (!huge_zero_page_shrinker) 838 return -ENOMEM; 839 840 deferred_split_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE | 841 SHRINKER_MEMCG_AWARE | 842 SHRINKER_NONSLAB, 843 "thp-deferred_split"); 844 if (!deferred_split_shrinker) { 845 shrinker_free(huge_zero_page_shrinker); 846 return -ENOMEM; 847 } 848 849 huge_zero_page_shrinker->count_objects = shrink_huge_zero_page_count; 850 huge_zero_page_shrinker->scan_objects = shrink_huge_zero_page_scan; 851 shrinker_register(huge_zero_page_shrinker); 852 853 deferred_split_shrinker->count_objects = deferred_split_count; 854 deferred_split_shrinker->scan_objects = deferred_split_scan; 855 shrinker_register(deferred_split_shrinker); 856 857 return 0; 858 } 859 860 static void __init thp_shrinker_exit(void) 861 { 862 shrinker_free(huge_zero_page_shrinker); 863 shrinker_free(deferred_split_shrinker); 864 } 865 866 static int __init hugepage_init(void) 867 { 868 int err; 869 struct kobject *hugepage_kobj; 870 871 if (!has_transparent_hugepage()) { 872 transparent_hugepage_flags = 1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED; 873 return -EINVAL; 874 } 875 876 /* 877 * hugepages can't be allocated by the buddy allocator 878 */ 879 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER > MAX_PAGE_ORDER); 880 881 err = hugepage_init_sysfs(&hugepage_kobj); 882 if (err) 883 goto err_sysfs; 884 885 err = khugepaged_init(); 886 if (err) 887 goto err_slab; 888 889 err = thp_shrinker_init(); 890 if (err) 891 goto err_shrinker; 892 893 /* 894 * By default disable transparent hugepages on smaller systems, 895 * where the extra memory used could hurt more than TLB overhead 896 * is likely to save. The admin can still enable it through /sys. 897 */ 898 if (totalram_pages() < (512 << (20 - PAGE_SHIFT))) { 899 transparent_hugepage_flags = 0; 900 return 0; 901 } 902 903 err = start_stop_khugepaged(); 904 if (err) 905 goto err_khugepaged; 906 907 return 0; 908 err_khugepaged: 909 thp_shrinker_exit(); 910 err_shrinker: 911 khugepaged_destroy(); 912 err_slab: 913 hugepage_exit_sysfs(hugepage_kobj); 914 err_sysfs: 915 return err; 916 } 917 subsys_initcall(hugepage_init); 918 919 static int __init setup_transparent_hugepage(char *str) 920 { 921 int ret = 0; 922 if (!str) 923 goto out; 924 if (!strcmp(str, "always")) { 925 set_bit(TRANSPARENT_HUGEPAGE_FLAG, 926 &transparent_hugepage_flags); 927 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 928 &transparent_hugepage_flags); 929 ret = 1; 930 } else if (!strcmp(str, "madvise")) { 931 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 932 &transparent_hugepage_flags); 933 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 934 &transparent_hugepage_flags); 935 ret = 1; 936 } else if (!strcmp(str, "never")) { 937 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 938 &transparent_hugepage_flags); 939 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 940 &transparent_hugepage_flags); 941 ret = 1; 942 } 943 out: 944 if (!ret) 945 pr_warn("transparent_hugepage= cannot parse, ignored\n"); 946 return ret; 947 } 948 __setup("transparent_hugepage=", setup_transparent_hugepage); 949 950 static inline int get_order_from_str(const char *size_str) 951 { 952 unsigned long size; 953 char *endptr; 954 int order; 955 956 size = memparse(size_str, &endptr); 957 958 if (!is_power_of_2(size)) 959 goto err; 960 order = get_order(size); 961 if (BIT(order) & ~THP_ORDERS_ALL_ANON) 962 goto err; 963 964 return order; 965 err: 966 pr_err("invalid size %s in thp_anon boot parameter\n", size_str); 967 return -EINVAL; 968 } 969 970 static char str_dup[PAGE_SIZE] __initdata; 971 static int __init setup_thp_anon(char *str) 972 { 973 char *token, *range, *policy, *subtoken; 974 unsigned long always, inherit, madvise; 975 char *start_size, *end_size; 976 int start, end, nr; 977 char *p; 978 979 if (!str || strlen(str) + 1 > PAGE_SIZE) 980 goto err; 981 strcpy(str_dup, str); 982 983 always = huge_anon_orders_always; 984 madvise = huge_anon_orders_madvise; 985 inherit = huge_anon_orders_inherit; 986 p = str_dup; 987 while ((token = strsep(&p, ";")) != NULL) { 988 range = strsep(&token, ":"); 989 policy = token; 990 991 if (!policy) 992 goto err; 993 994 while ((subtoken = strsep(&range, ",")) != NULL) { 995 if (strchr(subtoken, '-')) { 996 start_size = strsep(&subtoken, "-"); 997 end_size = subtoken; 998 999 start = get_order_from_str(start_size); 1000 end = get_order_from_str(end_size); 1001 } else { 1002 start = end = get_order_from_str(subtoken); 1003 } 1004 1005 if (start < 0 || end < 0 || start > end) 1006 goto err; 1007 1008 nr = end - start + 1; 1009 if (!strcmp(policy, "always")) { 1010 bitmap_set(&always, start, nr); 1011 bitmap_clear(&inherit, start, nr); 1012 bitmap_clear(&madvise, start, nr); 1013 } else if (!strcmp(policy, "madvise")) { 1014 bitmap_set(&madvise, start, nr); 1015 bitmap_clear(&inherit, start, nr); 1016 bitmap_clear(&always, start, nr); 1017 } else if (!strcmp(policy, "inherit")) { 1018 bitmap_set(&inherit, start, nr); 1019 bitmap_clear(&madvise, start, nr); 1020 bitmap_clear(&always, start, nr); 1021 } else if (!strcmp(policy, "never")) { 1022 bitmap_clear(&inherit, start, nr); 1023 bitmap_clear(&madvise, start, nr); 1024 bitmap_clear(&always, start, nr); 1025 } else { 1026 pr_err("invalid policy %s in thp_anon boot parameter\n", policy); 1027 goto err; 1028 } 1029 } 1030 } 1031 1032 huge_anon_orders_always = always; 1033 huge_anon_orders_madvise = madvise; 1034 huge_anon_orders_inherit = inherit; 1035 anon_orders_configured = true; 1036 return 1; 1037 1038 err: 1039 pr_warn("thp_anon=%s: error parsing string, ignoring setting\n", str); 1040 return 0; 1041 } 1042 __setup("thp_anon=", setup_thp_anon); 1043 1044 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 1045 { 1046 if (likely(vma->vm_flags & VM_WRITE)) 1047 pmd = pmd_mkwrite(pmd, vma); 1048 return pmd; 1049 } 1050 1051 #ifdef CONFIG_MEMCG 1052 static inline 1053 struct deferred_split *get_deferred_split_queue(struct folio *folio) 1054 { 1055 struct mem_cgroup *memcg = folio_memcg(folio); 1056 struct pglist_data *pgdat = NODE_DATA(folio_nid(folio)); 1057 1058 if (memcg) 1059 return &memcg->deferred_split_queue; 1060 else 1061 return &pgdat->deferred_split_queue; 1062 } 1063 #else 1064 static inline 1065 struct deferred_split *get_deferred_split_queue(struct folio *folio) 1066 { 1067 struct pglist_data *pgdat = NODE_DATA(folio_nid(folio)); 1068 1069 return &pgdat->deferred_split_queue; 1070 } 1071 #endif 1072 1073 static inline bool is_transparent_hugepage(const struct folio *folio) 1074 { 1075 if (!folio_test_large(folio)) 1076 return false; 1077 1078 return is_huge_zero_folio(folio) || 1079 folio_test_large_rmappable(folio); 1080 } 1081 1082 static unsigned long __thp_get_unmapped_area(struct file *filp, 1083 unsigned long addr, unsigned long len, 1084 loff_t off, unsigned long flags, unsigned long size, 1085 vm_flags_t vm_flags) 1086 { 1087 loff_t off_end = off + len; 1088 loff_t off_align = round_up(off, size); 1089 unsigned long len_pad, ret, off_sub; 1090 1091 if (!IS_ENABLED(CONFIG_64BIT) || in_compat_syscall()) 1092 return 0; 1093 1094 if (off_end <= off_align || (off_end - off_align) < size) 1095 return 0; 1096 1097 len_pad = len + size; 1098 if (len_pad < len || (off + len_pad) < off) 1099 return 0; 1100 1101 ret = mm_get_unmapped_area_vmflags(current->mm, filp, addr, len_pad, 1102 off >> PAGE_SHIFT, flags, vm_flags); 1103 1104 /* 1105 * The failure might be due to length padding. The caller will retry 1106 * without the padding. 1107 */ 1108 if (IS_ERR_VALUE(ret)) 1109 return 0; 1110 1111 /* 1112 * Do not try to align to THP boundary if allocation at the address 1113 * hint succeeds. 1114 */ 1115 if (ret == addr) 1116 return addr; 1117 1118 off_sub = (off - ret) & (size - 1); 1119 1120 if (test_bit(MMF_TOPDOWN, ¤t->mm->flags) && !off_sub) 1121 return ret + size; 1122 1123 ret += off_sub; 1124 return ret; 1125 } 1126 1127 unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, 1128 unsigned long len, unsigned long pgoff, unsigned long flags, 1129 vm_flags_t vm_flags) 1130 { 1131 unsigned long ret; 1132 loff_t off = (loff_t)pgoff << PAGE_SHIFT; 1133 1134 ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, vm_flags); 1135 if (ret) 1136 return ret; 1137 1138 return mm_get_unmapped_area_vmflags(current->mm, filp, addr, len, pgoff, flags, 1139 vm_flags); 1140 } 1141 1142 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, 1143 unsigned long len, unsigned long pgoff, unsigned long flags) 1144 { 1145 return thp_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0); 1146 } 1147 EXPORT_SYMBOL_GPL(thp_get_unmapped_area); 1148 1149 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, 1150 struct page *page, gfp_t gfp) 1151 { 1152 struct vm_area_struct *vma = vmf->vma; 1153 struct folio *folio = page_folio(page); 1154 pgtable_t pgtable; 1155 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1156 vm_fault_t ret = 0; 1157 1158 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 1159 1160 if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) { 1161 folio_put(folio); 1162 count_vm_event(THP_FAULT_FALLBACK); 1163 count_vm_event(THP_FAULT_FALLBACK_CHARGE); 1164 count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_FALLBACK); 1165 count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); 1166 return VM_FAULT_FALLBACK; 1167 } 1168 folio_throttle_swaprate(folio, gfp); 1169 1170 pgtable = pte_alloc_one(vma->vm_mm); 1171 if (unlikely(!pgtable)) { 1172 ret = VM_FAULT_OOM; 1173 goto release; 1174 } 1175 1176 folio_zero_user(folio, vmf->address); 1177 /* 1178 * The memory barrier inside __folio_mark_uptodate makes sure that 1179 * folio_zero_user writes become visible before the set_pmd_at() 1180 * write. 1181 */ 1182 __folio_mark_uptodate(folio); 1183 1184 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1185 if (unlikely(!pmd_none(*vmf->pmd))) { 1186 goto unlock_release; 1187 } else { 1188 pmd_t entry; 1189 1190 ret = check_stable_address_space(vma->vm_mm); 1191 if (ret) 1192 goto unlock_release; 1193 1194 /* Deliver the page fault to userland */ 1195 if (userfaultfd_missing(vma)) { 1196 spin_unlock(vmf->ptl); 1197 folio_put(folio); 1198 pte_free(vma->vm_mm, pgtable); 1199 ret = handle_userfault(vmf, VM_UFFD_MISSING); 1200 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 1201 return ret; 1202 } 1203 1204 entry = mk_huge_pmd(page, vma->vm_page_prot); 1205 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1206 folio_add_new_anon_rmap(folio, vma, haddr, RMAP_EXCLUSIVE); 1207 folio_add_lru_vma(folio, vma); 1208 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); 1209 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); 1210 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1211 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1212 mm_inc_nr_ptes(vma->vm_mm); 1213 deferred_split_folio(folio, false); 1214 spin_unlock(vmf->ptl); 1215 count_vm_event(THP_FAULT_ALLOC); 1216 count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_ALLOC); 1217 count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); 1218 } 1219 1220 return 0; 1221 unlock_release: 1222 spin_unlock(vmf->ptl); 1223 release: 1224 if (pgtable) 1225 pte_free(vma->vm_mm, pgtable); 1226 folio_put(folio); 1227 return ret; 1228 1229 } 1230 1231 /* 1232 * always: directly stall for all thp allocations 1233 * defer: wake kswapd and fail if not immediately available 1234 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise 1235 * fail if not immediately available 1236 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately 1237 * available 1238 * never: never stall for any thp allocation 1239 */ 1240 gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma) 1241 { 1242 const bool vma_madvised = vma && (vma->vm_flags & VM_HUGEPAGE); 1243 1244 /* Always do synchronous compaction */ 1245 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) 1246 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY); 1247 1248 /* Kick kcompactd and fail quickly */ 1249 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) 1250 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM; 1251 1252 /* Synchronous compaction if madvised, otherwise kick kcompactd */ 1253 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags)) 1254 return GFP_TRANSHUGE_LIGHT | 1255 (vma_madvised ? __GFP_DIRECT_RECLAIM : 1256 __GFP_KSWAPD_RECLAIM); 1257 1258 /* Only do synchronous compaction if madvised */ 1259 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags)) 1260 return GFP_TRANSHUGE_LIGHT | 1261 (vma_madvised ? __GFP_DIRECT_RECLAIM : 0); 1262 1263 return GFP_TRANSHUGE_LIGHT; 1264 } 1265 1266 /* Caller must hold page table lock. */ 1267 static void set_huge_zero_folio(pgtable_t pgtable, struct mm_struct *mm, 1268 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, 1269 struct folio *zero_folio) 1270 { 1271 pmd_t entry; 1272 if (!pmd_none(*pmd)) 1273 return; 1274 entry = mk_pmd(&zero_folio->page, vma->vm_page_prot); 1275 entry = pmd_mkhuge(entry); 1276 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1277 set_pmd_at(mm, haddr, pmd, entry); 1278 mm_inc_nr_ptes(mm); 1279 } 1280 1281 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) 1282 { 1283 struct vm_area_struct *vma = vmf->vma; 1284 gfp_t gfp; 1285 struct folio *folio; 1286 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1287 vm_fault_t ret; 1288 1289 if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER)) 1290 return VM_FAULT_FALLBACK; 1291 ret = vmf_anon_prepare(vmf); 1292 if (ret) 1293 return ret; 1294 khugepaged_enter_vma(vma, vma->vm_flags); 1295 1296 if (!(vmf->flags & FAULT_FLAG_WRITE) && 1297 !mm_forbids_zeropage(vma->vm_mm) && 1298 transparent_hugepage_use_zero_page()) { 1299 pgtable_t pgtable; 1300 struct folio *zero_folio; 1301 vm_fault_t ret; 1302 1303 pgtable = pte_alloc_one(vma->vm_mm); 1304 if (unlikely(!pgtable)) 1305 return VM_FAULT_OOM; 1306 zero_folio = mm_get_huge_zero_folio(vma->vm_mm); 1307 if (unlikely(!zero_folio)) { 1308 pte_free(vma->vm_mm, pgtable); 1309 count_vm_event(THP_FAULT_FALLBACK); 1310 return VM_FAULT_FALLBACK; 1311 } 1312 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1313 ret = 0; 1314 if (pmd_none(*vmf->pmd)) { 1315 ret = check_stable_address_space(vma->vm_mm); 1316 if (ret) { 1317 spin_unlock(vmf->ptl); 1318 pte_free(vma->vm_mm, pgtable); 1319 } else if (userfaultfd_missing(vma)) { 1320 spin_unlock(vmf->ptl); 1321 pte_free(vma->vm_mm, pgtable); 1322 ret = handle_userfault(vmf, VM_UFFD_MISSING); 1323 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 1324 } else { 1325 set_huge_zero_folio(pgtable, vma->vm_mm, vma, 1326 haddr, vmf->pmd, zero_folio); 1327 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1328 spin_unlock(vmf->ptl); 1329 } 1330 } else { 1331 spin_unlock(vmf->ptl); 1332 pte_free(vma->vm_mm, pgtable); 1333 } 1334 return ret; 1335 } 1336 gfp = vma_thp_gfp_mask(vma); 1337 folio = vma_alloc_folio(gfp, HPAGE_PMD_ORDER, vma, haddr, true); 1338 if (unlikely(!folio)) { 1339 count_vm_event(THP_FAULT_FALLBACK); 1340 count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_FALLBACK); 1341 return VM_FAULT_FALLBACK; 1342 } 1343 return __do_huge_pmd_anonymous_page(vmf, &folio->page, gfp); 1344 } 1345 1346 static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 1347 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write, 1348 pgtable_t pgtable) 1349 { 1350 struct mm_struct *mm = vma->vm_mm; 1351 pmd_t entry; 1352 spinlock_t *ptl; 1353 1354 ptl = pmd_lock(mm, pmd); 1355 if (!pmd_none(*pmd)) { 1356 if (write) { 1357 if (pmd_pfn(*pmd) != pfn_t_to_pfn(pfn)) { 1358 WARN_ON_ONCE(!is_huge_zero_pmd(*pmd)); 1359 goto out_unlock; 1360 } 1361 entry = pmd_mkyoung(*pmd); 1362 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1363 if (pmdp_set_access_flags(vma, addr, pmd, entry, 1)) 1364 update_mmu_cache_pmd(vma, addr, pmd); 1365 } 1366 1367 goto out_unlock; 1368 } 1369 1370 entry = pmd_mkhuge(pfn_t_pmd(pfn, prot)); 1371 if (pfn_t_devmap(pfn)) 1372 entry = pmd_mkdevmap(entry); 1373 if (write) { 1374 entry = pmd_mkyoung(pmd_mkdirty(entry)); 1375 entry = maybe_pmd_mkwrite(entry, vma); 1376 } 1377 1378 if (pgtable) { 1379 pgtable_trans_huge_deposit(mm, pmd, pgtable); 1380 mm_inc_nr_ptes(mm); 1381 pgtable = NULL; 1382 } 1383 1384 set_pmd_at(mm, addr, pmd, entry); 1385 update_mmu_cache_pmd(vma, addr, pmd); 1386 1387 out_unlock: 1388 spin_unlock(ptl); 1389 if (pgtable) 1390 pte_free(mm, pgtable); 1391 } 1392 1393 /** 1394 * vmf_insert_pfn_pmd - insert a pmd size pfn 1395 * @vmf: Structure describing the fault 1396 * @pfn: pfn to insert 1397 * @write: whether it's a write fault 1398 * 1399 * Insert a pmd size pfn. See vmf_insert_pfn() for additional info. 1400 * 1401 * Return: vm_fault_t value. 1402 */ 1403 vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, pfn_t pfn, bool write) 1404 { 1405 unsigned long addr = vmf->address & PMD_MASK; 1406 struct vm_area_struct *vma = vmf->vma; 1407 pgprot_t pgprot = vma->vm_page_prot; 1408 pgtable_t pgtable = NULL; 1409 1410 /* 1411 * If we had pmd_special, we could avoid all these restrictions, 1412 * but we need to be consistent with PTEs and architectures that 1413 * can't support a 'special' bit. 1414 */ 1415 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && 1416 !pfn_t_devmap(pfn)); 1417 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1418 (VM_PFNMAP|VM_MIXEDMAP)); 1419 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1420 1421 if (addr < vma->vm_start || addr >= vma->vm_end) 1422 return VM_FAULT_SIGBUS; 1423 1424 if (arch_needs_pgtable_deposit()) { 1425 pgtable = pte_alloc_one(vma->vm_mm); 1426 if (!pgtable) 1427 return VM_FAULT_OOM; 1428 } 1429 1430 track_pfn_insert(vma, &pgprot, pfn); 1431 1432 insert_pfn_pmd(vma, addr, vmf->pmd, pfn, pgprot, write, pgtable); 1433 return VM_FAULT_NOPAGE; 1434 } 1435 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd); 1436 1437 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1438 static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma) 1439 { 1440 if (likely(vma->vm_flags & VM_WRITE)) 1441 pud = pud_mkwrite(pud); 1442 return pud; 1443 } 1444 1445 static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr, 1446 pud_t *pud, pfn_t pfn, bool write) 1447 { 1448 struct mm_struct *mm = vma->vm_mm; 1449 pgprot_t prot = vma->vm_page_prot; 1450 pud_t entry; 1451 spinlock_t *ptl; 1452 1453 ptl = pud_lock(mm, pud); 1454 if (!pud_none(*pud)) { 1455 if (write) { 1456 if (pud_pfn(*pud) != pfn_t_to_pfn(pfn)) { 1457 WARN_ON_ONCE(!is_huge_zero_pud(*pud)); 1458 goto out_unlock; 1459 } 1460 entry = pud_mkyoung(*pud); 1461 entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma); 1462 if (pudp_set_access_flags(vma, addr, pud, entry, 1)) 1463 update_mmu_cache_pud(vma, addr, pud); 1464 } 1465 goto out_unlock; 1466 } 1467 1468 entry = pud_mkhuge(pfn_t_pud(pfn, prot)); 1469 if (pfn_t_devmap(pfn)) 1470 entry = pud_mkdevmap(entry); 1471 if (write) { 1472 entry = pud_mkyoung(pud_mkdirty(entry)); 1473 entry = maybe_pud_mkwrite(entry, vma); 1474 } 1475 set_pud_at(mm, addr, pud, entry); 1476 update_mmu_cache_pud(vma, addr, pud); 1477 1478 out_unlock: 1479 spin_unlock(ptl); 1480 } 1481 1482 /** 1483 * vmf_insert_pfn_pud - insert a pud size pfn 1484 * @vmf: Structure describing the fault 1485 * @pfn: pfn to insert 1486 * @write: whether it's a write fault 1487 * 1488 * Insert a pud size pfn. See vmf_insert_pfn() for additional info. 1489 * 1490 * Return: vm_fault_t value. 1491 */ 1492 vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, pfn_t pfn, bool write) 1493 { 1494 unsigned long addr = vmf->address & PUD_MASK; 1495 struct vm_area_struct *vma = vmf->vma; 1496 pgprot_t pgprot = vma->vm_page_prot; 1497 1498 /* 1499 * If we had pud_special, we could avoid all these restrictions, 1500 * but we need to be consistent with PTEs and architectures that 1501 * can't support a 'special' bit. 1502 */ 1503 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && 1504 !pfn_t_devmap(pfn)); 1505 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 1506 (VM_PFNMAP|VM_MIXEDMAP)); 1507 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 1508 1509 if (addr < vma->vm_start || addr >= vma->vm_end) 1510 return VM_FAULT_SIGBUS; 1511 1512 track_pfn_insert(vma, &pgprot, pfn); 1513 1514 insert_pfn_pud(vma, addr, vmf->pud, pfn, write); 1515 return VM_FAULT_NOPAGE; 1516 } 1517 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud); 1518 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1519 1520 void touch_pmd(struct vm_area_struct *vma, unsigned long addr, 1521 pmd_t *pmd, bool write) 1522 { 1523 pmd_t _pmd; 1524 1525 _pmd = pmd_mkyoung(*pmd); 1526 if (write) 1527 _pmd = pmd_mkdirty(_pmd); 1528 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, 1529 pmd, _pmd, write)) 1530 update_mmu_cache_pmd(vma, addr, pmd); 1531 } 1532 1533 struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, 1534 pmd_t *pmd, int flags, struct dev_pagemap **pgmap) 1535 { 1536 unsigned long pfn = pmd_pfn(*pmd); 1537 struct mm_struct *mm = vma->vm_mm; 1538 struct page *page; 1539 int ret; 1540 1541 assert_spin_locked(pmd_lockptr(mm, pmd)); 1542 1543 if (flags & FOLL_WRITE && !pmd_write(*pmd)) 1544 return NULL; 1545 1546 if (pmd_present(*pmd) && pmd_devmap(*pmd)) 1547 /* pass */; 1548 else 1549 return NULL; 1550 1551 if (flags & FOLL_TOUCH) 1552 touch_pmd(vma, addr, pmd, flags & FOLL_WRITE); 1553 1554 /* 1555 * device mapped pages can only be returned if the 1556 * caller will manage the page reference count. 1557 */ 1558 if (!(flags & (FOLL_GET | FOLL_PIN))) 1559 return ERR_PTR(-EEXIST); 1560 1561 pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT; 1562 *pgmap = get_dev_pagemap(pfn, *pgmap); 1563 if (!*pgmap) 1564 return ERR_PTR(-EFAULT); 1565 page = pfn_to_page(pfn); 1566 ret = try_grab_folio(page_folio(page), 1, flags); 1567 if (ret) 1568 page = ERR_PTR(ret); 1569 1570 return page; 1571 } 1572 1573 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1574 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 1575 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) 1576 { 1577 spinlock_t *dst_ptl, *src_ptl; 1578 struct page *src_page; 1579 struct folio *src_folio; 1580 pmd_t pmd; 1581 pgtable_t pgtable = NULL; 1582 int ret = -ENOMEM; 1583 1584 /* Skip if can be re-fill on fault */ 1585 if (!vma_is_anonymous(dst_vma)) 1586 return 0; 1587 1588 pgtable = pte_alloc_one(dst_mm); 1589 if (unlikely(!pgtable)) 1590 goto out; 1591 1592 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1593 src_ptl = pmd_lockptr(src_mm, src_pmd); 1594 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1595 1596 ret = -EAGAIN; 1597 pmd = *src_pmd; 1598 1599 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 1600 if (unlikely(is_swap_pmd(pmd))) { 1601 swp_entry_t entry = pmd_to_swp_entry(pmd); 1602 1603 VM_BUG_ON(!is_pmd_migration_entry(pmd)); 1604 if (!is_readable_migration_entry(entry)) { 1605 entry = make_readable_migration_entry( 1606 swp_offset(entry)); 1607 pmd = swp_entry_to_pmd(entry); 1608 if (pmd_swp_soft_dirty(*src_pmd)) 1609 pmd = pmd_swp_mksoft_dirty(pmd); 1610 if (pmd_swp_uffd_wp(*src_pmd)) 1611 pmd = pmd_swp_mkuffd_wp(pmd); 1612 set_pmd_at(src_mm, addr, src_pmd, pmd); 1613 } 1614 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1615 mm_inc_nr_ptes(dst_mm); 1616 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1617 if (!userfaultfd_wp(dst_vma)) 1618 pmd = pmd_swp_clear_uffd_wp(pmd); 1619 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1620 ret = 0; 1621 goto out_unlock; 1622 } 1623 #endif 1624 1625 if (unlikely(!pmd_trans_huge(pmd))) { 1626 pte_free(dst_mm, pgtable); 1627 goto out_unlock; 1628 } 1629 /* 1630 * When page table lock is held, the huge zero pmd should not be 1631 * under splitting since we don't split the page itself, only pmd to 1632 * a page table. 1633 */ 1634 if (is_huge_zero_pmd(pmd)) { 1635 /* 1636 * mm_get_huge_zero_folio() will never allocate a new 1637 * folio here, since we already have a zero page to 1638 * copy. It just takes a reference. 1639 */ 1640 mm_get_huge_zero_folio(dst_mm); 1641 goto out_zero_page; 1642 } 1643 1644 src_page = pmd_page(pmd); 1645 VM_BUG_ON_PAGE(!PageHead(src_page), src_page); 1646 src_folio = page_folio(src_page); 1647 1648 folio_get(src_folio); 1649 if (unlikely(folio_try_dup_anon_rmap_pmd(src_folio, src_page, src_vma))) { 1650 /* Page maybe pinned: split and retry the fault on PTEs. */ 1651 folio_put(src_folio); 1652 pte_free(dst_mm, pgtable); 1653 spin_unlock(src_ptl); 1654 spin_unlock(dst_ptl); 1655 __split_huge_pmd(src_vma, src_pmd, addr, false, NULL); 1656 return -EAGAIN; 1657 } 1658 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1659 out_zero_page: 1660 mm_inc_nr_ptes(dst_mm); 1661 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1662 pmdp_set_wrprotect(src_mm, addr, src_pmd); 1663 if (!userfaultfd_wp(dst_vma)) 1664 pmd = pmd_clear_uffd_wp(pmd); 1665 pmd = pmd_mkold(pmd_wrprotect(pmd)); 1666 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1667 1668 ret = 0; 1669 out_unlock: 1670 spin_unlock(src_ptl); 1671 spin_unlock(dst_ptl); 1672 out: 1673 return ret; 1674 } 1675 1676 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1677 void touch_pud(struct vm_area_struct *vma, unsigned long addr, 1678 pud_t *pud, bool write) 1679 { 1680 pud_t _pud; 1681 1682 _pud = pud_mkyoung(*pud); 1683 if (write) 1684 _pud = pud_mkdirty(_pud); 1685 if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK, 1686 pud, _pud, write)) 1687 update_mmu_cache_pud(vma, addr, pud); 1688 } 1689 1690 int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1691 pud_t *dst_pud, pud_t *src_pud, unsigned long addr, 1692 struct vm_area_struct *vma) 1693 { 1694 spinlock_t *dst_ptl, *src_ptl; 1695 pud_t pud; 1696 int ret; 1697 1698 dst_ptl = pud_lock(dst_mm, dst_pud); 1699 src_ptl = pud_lockptr(src_mm, src_pud); 1700 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1701 1702 ret = -EAGAIN; 1703 pud = *src_pud; 1704 if (unlikely(!pud_trans_huge(pud) && !pud_devmap(pud))) 1705 goto out_unlock; 1706 1707 /* 1708 * When page table lock is held, the huge zero pud should not be 1709 * under splitting since we don't split the page itself, only pud to 1710 * a page table. 1711 */ 1712 if (is_huge_zero_pud(pud)) { 1713 /* No huge zero pud yet */ 1714 } 1715 1716 /* 1717 * TODO: once we support anonymous pages, use 1718 * folio_try_dup_anon_rmap_*() and split if duplicating fails. 1719 */ 1720 pudp_set_wrprotect(src_mm, addr, src_pud); 1721 pud = pud_mkold(pud_wrprotect(pud)); 1722 set_pud_at(dst_mm, addr, dst_pud, pud); 1723 1724 ret = 0; 1725 out_unlock: 1726 spin_unlock(src_ptl); 1727 spin_unlock(dst_ptl); 1728 return ret; 1729 } 1730 1731 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud) 1732 { 1733 bool write = vmf->flags & FAULT_FLAG_WRITE; 1734 1735 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); 1736 if (unlikely(!pud_same(*vmf->pud, orig_pud))) 1737 goto unlock; 1738 1739 touch_pud(vmf->vma, vmf->address, vmf->pud, write); 1740 unlock: 1741 spin_unlock(vmf->ptl); 1742 } 1743 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1744 1745 void huge_pmd_set_accessed(struct vm_fault *vmf) 1746 { 1747 bool write = vmf->flags & FAULT_FLAG_WRITE; 1748 1749 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); 1750 if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd))) 1751 goto unlock; 1752 1753 touch_pmd(vmf->vma, vmf->address, vmf->pmd, write); 1754 1755 unlock: 1756 spin_unlock(vmf->ptl); 1757 } 1758 1759 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) 1760 { 1761 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; 1762 struct vm_area_struct *vma = vmf->vma; 1763 struct folio *folio; 1764 struct page *page; 1765 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1766 pmd_t orig_pmd = vmf->orig_pmd; 1767 1768 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); 1769 VM_BUG_ON_VMA(!vma->anon_vma, vma); 1770 1771 if (is_huge_zero_pmd(orig_pmd)) 1772 goto fallback; 1773 1774 spin_lock(vmf->ptl); 1775 1776 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1777 spin_unlock(vmf->ptl); 1778 return 0; 1779 } 1780 1781 page = pmd_page(orig_pmd); 1782 folio = page_folio(page); 1783 VM_BUG_ON_PAGE(!PageHead(page), page); 1784 1785 /* Early check when only holding the PT lock. */ 1786 if (PageAnonExclusive(page)) 1787 goto reuse; 1788 1789 if (!folio_trylock(folio)) { 1790 folio_get(folio); 1791 spin_unlock(vmf->ptl); 1792 folio_lock(folio); 1793 spin_lock(vmf->ptl); 1794 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1795 spin_unlock(vmf->ptl); 1796 folio_unlock(folio); 1797 folio_put(folio); 1798 return 0; 1799 } 1800 folio_put(folio); 1801 } 1802 1803 /* Recheck after temporarily dropping the PT lock. */ 1804 if (PageAnonExclusive(page)) { 1805 folio_unlock(folio); 1806 goto reuse; 1807 } 1808 1809 /* 1810 * See do_wp_page(): we can only reuse the folio exclusively if 1811 * there are no additional references. Note that we always drain 1812 * the LRU cache immediately after adding a THP. 1813 */ 1814 if (folio_ref_count(folio) > 1815 1 + folio_test_swapcache(folio) * folio_nr_pages(folio)) 1816 goto unlock_fallback; 1817 if (folio_test_swapcache(folio)) 1818 folio_free_swap(folio); 1819 if (folio_ref_count(folio) == 1) { 1820 pmd_t entry; 1821 1822 folio_move_anon_rmap(folio, vma); 1823 SetPageAnonExclusive(page); 1824 folio_unlock(folio); 1825 reuse: 1826 if (unlikely(unshare)) { 1827 spin_unlock(vmf->ptl); 1828 return 0; 1829 } 1830 entry = pmd_mkyoung(orig_pmd); 1831 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1832 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) 1833 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1834 spin_unlock(vmf->ptl); 1835 return 0; 1836 } 1837 1838 unlock_fallback: 1839 folio_unlock(folio); 1840 spin_unlock(vmf->ptl); 1841 fallback: 1842 __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL); 1843 return VM_FAULT_FALLBACK; 1844 } 1845 1846 static inline bool can_change_pmd_writable(struct vm_area_struct *vma, 1847 unsigned long addr, pmd_t pmd) 1848 { 1849 struct page *page; 1850 1851 if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) 1852 return false; 1853 1854 /* Don't touch entries that are not even readable (NUMA hinting). */ 1855 if (pmd_protnone(pmd)) 1856 return false; 1857 1858 /* Do we need write faults for softdirty tracking? */ 1859 if (pmd_needs_soft_dirty_wp(vma, pmd)) 1860 return false; 1861 1862 /* Do we need write faults for uffd-wp tracking? */ 1863 if (userfaultfd_huge_pmd_wp(vma, pmd)) 1864 return false; 1865 1866 if (!(vma->vm_flags & VM_SHARED)) { 1867 /* See can_change_pte_writable(). */ 1868 page = vm_normal_page_pmd(vma, addr, pmd); 1869 return page && PageAnon(page) && PageAnonExclusive(page); 1870 } 1871 1872 /* See can_change_pte_writable(). */ 1873 return pmd_dirty(pmd); 1874 } 1875 1876 /* NUMA hinting page fault entry point for trans huge pmds */ 1877 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) 1878 { 1879 struct vm_area_struct *vma = vmf->vma; 1880 struct folio *folio; 1881 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1882 int nid = NUMA_NO_NODE; 1883 int target_nid, last_cpupid; 1884 pmd_t pmd, old_pmd; 1885 bool writable = false; 1886 int flags = 0; 1887 1888 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1889 old_pmd = pmdp_get(vmf->pmd); 1890 1891 if (unlikely(!pmd_same(old_pmd, vmf->orig_pmd))) { 1892 spin_unlock(vmf->ptl); 1893 return 0; 1894 } 1895 1896 pmd = pmd_modify(old_pmd, vma->vm_page_prot); 1897 1898 /* 1899 * Detect now whether the PMD could be writable; this information 1900 * is only valid while holding the PT lock. 1901 */ 1902 writable = pmd_write(pmd); 1903 if (!writable && vma_wants_manual_pte_write_upgrade(vma) && 1904 can_change_pmd_writable(vma, vmf->address, pmd)) 1905 writable = true; 1906 1907 folio = vm_normal_folio_pmd(vma, haddr, pmd); 1908 if (!folio) 1909 goto out_map; 1910 1911 nid = folio_nid(folio); 1912 1913 target_nid = numa_migrate_check(folio, vmf, haddr, &flags, writable, 1914 &last_cpupid); 1915 if (target_nid == NUMA_NO_NODE) 1916 goto out_map; 1917 if (migrate_misplaced_folio_prepare(folio, vma, target_nid)) { 1918 flags |= TNF_MIGRATE_FAIL; 1919 goto out_map; 1920 } 1921 /* The folio is isolated and isolation code holds a folio reference. */ 1922 spin_unlock(vmf->ptl); 1923 writable = false; 1924 1925 if (!migrate_misplaced_folio(folio, vma, target_nid)) { 1926 flags |= TNF_MIGRATED; 1927 nid = target_nid; 1928 task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags); 1929 return 0; 1930 } 1931 1932 flags |= TNF_MIGRATE_FAIL; 1933 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1934 if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) { 1935 spin_unlock(vmf->ptl); 1936 return 0; 1937 } 1938 out_map: 1939 /* Restore the PMD */ 1940 pmd = pmd_modify(pmdp_get(vmf->pmd), vma->vm_page_prot); 1941 pmd = pmd_mkyoung(pmd); 1942 if (writable) 1943 pmd = pmd_mkwrite(pmd, vma); 1944 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); 1945 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1946 spin_unlock(vmf->ptl); 1947 1948 if (nid != NUMA_NO_NODE) 1949 task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags); 1950 return 0; 1951 } 1952 1953 /* 1954 * Return true if we do MADV_FREE successfully on entire pmd page. 1955 * Otherwise, return false. 1956 */ 1957 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1958 pmd_t *pmd, unsigned long addr, unsigned long next) 1959 { 1960 spinlock_t *ptl; 1961 pmd_t orig_pmd; 1962 struct folio *folio; 1963 struct mm_struct *mm = tlb->mm; 1964 bool ret = false; 1965 1966 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 1967 1968 ptl = pmd_trans_huge_lock(pmd, vma); 1969 if (!ptl) 1970 goto out_unlocked; 1971 1972 orig_pmd = *pmd; 1973 if (is_huge_zero_pmd(orig_pmd)) 1974 goto out; 1975 1976 if (unlikely(!pmd_present(orig_pmd))) { 1977 VM_BUG_ON(thp_migration_supported() && 1978 !is_pmd_migration_entry(orig_pmd)); 1979 goto out; 1980 } 1981 1982 folio = pmd_folio(orig_pmd); 1983 /* 1984 * If other processes are mapping this folio, we couldn't discard 1985 * the folio unless they all do MADV_FREE so let's skip the folio. 1986 */ 1987 if (folio_likely_mapped_shared(folio)) 1988 goto out; 1989 1990 if (!folio_trylock(folio)) 1991 goto out; 1992 1993 /* 1994 * If user want to discard part-pages of THP, split it so MADV_FREE 1995 * will deactivate only them. 1996 */ 1997 if (next - addr != HPAGE_PMD_SIZE) { 1998 folio_get(folio); 1999 spin_unlock(ptl); 2000 split_folio(folio); 2001 folio_unlock(folio); 2002 folio_put(folio); 2003 goto out_unlocked; 2004 } 2005 2006 if (folio_test_dirty(folio)) 2007 folio_clear_dirty(folio); 2008 folio_unlock(folio); 2009 2010 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) { 2011 pmdp_invalidate(vma, addr, pmd); 2012 orig_pmd = pmd_mkold(orig_pmd); 2013 orig_pmd = pmd_mkclean(orig_pmd); 2014 2015 set_pmd_at(mm, addr, pmd, orig_pmd); 2016 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 2017 } 2018 2019 folio_mark_lazyfree(folio); 2020 ret = true; 2021 out: 2022 spin_unlock(ptl); 2023 out_unlocked: 2024 return ret; 2025 } 2026 2027 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd) 2028 { 2029 pgtable_t pgtable; 2030 2031 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2032 pte_free(mm, pgtable); 2033 mm_dec_nr_ptes(mm); 2034 } 2035 2036 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2037 pmd_t *pmd, unsigned long addr) 2038 { 2039 pmd_t orig_pmd; 2040 spinlock_t *ptl; 2041 2042 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2043 2044 ptl = __pmd_trans_huge_lock(pmd, vma); 2045 if (!ptl) 2046 return 0; 2047 /* 2048 * For architectures like ppc64 we look at deposited pgtable 2049 * when calling pmdp_huge_get_and_clear. So do the 2050 * pgtable_trans_huge_withdraw after finishing pmdp related 2051 * operations. 2052 */ 2053 orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd, 2054 tlb->fullmm); 2055 arch_check_zapped_pmd(vma, orig_pmd); 2056 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 2057 if (vma_is_special_huge(vma)) { 2058 if (arch_needs_pgtable_deposit()) 2059 zap_deposited_table(tlb->mm, pmd); 2060 spin_unlock(ptl); 2061 } else if (is_huge_zero_pmd(orig_pmd)) { 2062 zap_deposited_table(tlb->mm, pmd); 2063 spin_unlock(ptl); 2064 } else { 2065 struct folio *folio = NULL; 2066 int flush_needed = 1; 2067 2068 if (pmd_present(orig_pmd)) { 2069 struct page *page = pmd_page(orig_pmd); 2070 2071 folio = page_folio(page); 2072 folio_remove_rmap_pmd(folio, page, vma); 2073 WARN_ON_ONCE(folio_mapcount(folio) < 0); 2074 VM_BUG_ON_PAGE(!PageHead(page), page); 2075 } else if (thp_migration_supported()) { 2076 swp_entry_t entry; 2077 2078 VM_BUG_ON(!is_pmd_migration_entry(orig_pmd)); 2079 entry = pmd_to_swp_entry(orig_pmd); 2080 folio = pfn_swap_entry_folio(entry); 2081 flush_needed = 0; 2082 } else 2083 WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!"); 2084 2085 if (folio_test_anon(folio)) { 2086 zap_deposited_table(tlb->mm, pmd); 2087 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); 2088 } else { 2089 if (arch_needs_pgtable_deposit()) 2090 zap_deposited_table(tlb->mm, pmd); 2091 add_mm_counter(tlb->mm, mm_counter_file(folio), 2092 -HPAGE_PMD_NR); 2093 } 2094 2095 spin_unlock(ptl); 2096 if (flush_needed) 2097 tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE); 2098 } 2099 return 1; 2100 } 2101 2102 #ifndef pmd_move_must_withdraw 2103 static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl, 2104 spinlock_t *old_pmd_ptl, 2105 struct vm_area_struct *vma) 2106 { 2107 /* 2108 * With split pmd lock we also need to move preallocated 2109 * PTE page table if new_pmd is on different PMD page table. 2110 * 2111 * We also don't deposit and withdraw tables for file pages. 2112 */ 2113 return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma); 2114 } 2115 #endif 2116 2117 static pmd_t move_soft_dirty_pmd(pmd_t pmd) 2118 { 2119 #ifdef CONFIG_MEM_SOFT_DIRTY 2120 if (unlikely(is_pmd_migration_entry(pmd))) 2121 pmd = pmd_swp_mksoft_dirty(pmd); 2122 else if (pmd_present(pmd)) 2123 pmd = pmd_mksoft_dirty(pmd); 2124 #endif 2125 return pmd; 2126 } 2127 2128 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, 2129 unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd) 2130 { 2131 spinlock_t *old_ptl, *new_ptl; 2132 pmd_t pmd; 2133 struct mm_struct *mm = vma->vm_mm; 2134 bool force_flush = false; 2135 2136 /* 2137 * The destination pmd shouldn't be established, free_pgtables() 2138 * should have released it; but move_page_tables() might have already 2139 * inserted a page table, if racing against shmem/file collapse. 2140 */ 2141 if (!pmd_none(*new_pmd)) { 2142 VM_BUG_ON(pmd_trans_huge(*new_pmd)); 2143 return false; 2144 } 2145 2146 /* 2147 * We don't have to worry about the ordering of src and dst 2148 * ptlocks because exclusive mmap_lock prevents deadlock. 2149 */ 2150 old_ptl = __pmd_trans_huge_lock(old_pmd, vma); 2151 if (old_ptl) { 2152 new_ptl = pmd_lockptr(mm, new_pmd); 2153 if (new_ptl != old_ptl) 2154 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 2155 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); 2156 if (pmd_present(pmd)) 2157 force_flush = true; 2158 VM_BUG_ON(!pmd_none(*new_pmd)); 2159 2160 if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) { 2161 pgtable_t pgtable; 2162 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); 2163 pgtable_trans_huge_deposit(mm, new_pmd, pgtable); 2164 } 2165 pmd = move_soft_dirty_pmd(pmd); 2166 set_pmd_at(mm, new_addr, new_pmd, pmd); 2167 if (force_flush) 2168 flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE); 2169 if (new_ptl != old_ptl) 2170 spin_unlock(new_ptl); 2171 spin_unlock(old_ptl); 2172 return true; 2173 } 2174 return false; 2175 } 2176 2177 /* 2178 * Returns 2179 * - 0 if PMD could not be locked 2180 * - 1 if PMD was locked but protections unchanged and TLB flush unnecessary 2181 * or if prot_numa but THP migration is not supported 2182 * - HPAGE_PMD_NR if protections changed and TLB flush necessary 2183 */ 2184 int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 2185 pmd_t *pmd, unsigned long addr, pgprot_t newprot, 2186 unsigned long cp_flags) 2187 { 2188 struct mm_struct *mm = vma->vm_mm; 2189 spinlock_t *ptl; 2190 pmd_t oldpmd, entry; 2191 bool prot_numa = cp_flags & MM_CP_PROT_NUMA; 2192 bool uffd_wp = cp_flags & MM_CP_UFFD_WP; 2193 bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; 2194 int ret = 1; 2195 2196 tlb_change_page_size(tlb, HPAGE_PMD_SIZE); 2197 2198 if (prot_numa && !thp_migration_supported()) 2199 return 1; 2200 2201 ptl = __pmd_trans_huge_lock(pmd, vma); 2202 if (!ptl) 2203 return 0; 2204 2205 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 2206 if (is_swap_pmd(*pmd)) { 2207 swp_entry_t entry = pmd_to_swp_entry(*pmd); 2208 struct folio *folio = pfn_swap_entry_folio(entry); 2209 pmd_t newpmd; 2210 2211 VM_BUG_ON(!is_pmd_migration_entry(*pmd)); 2212 if (is_writable_migration_entry(entry)) { 2213 /* 2214 * A protection check is difficult so 2215 * just be safe and disable write 2216 */ 2217 if (folio_test_anon(folio)) 2218 entry = make_readable_exclusive_migration_entry(swp_offset(entry)); 2219 else 2220 entry = make_readable_migration_entry(swp_offset(entry)); 2221 newpmd = swp_entry_to_pmd(entry); 2222 if (pmd_swp_soft_dirty(*pmd)) 2223 newpmd = pmd_swp_mksoft_dirty(newpmd); 2224 } else { 2225 newpmd = *pmd; 2226 } 2227 2228 if (uffd_wp) 2229 newpmd = pmd_swp_mkuffd_wp(newpmd); 2230 else if (uffd_wp_resolve) 2231 newpmd = pmd_swp_clear_uffd_wp(newpmd); 2232 if (!pmd_same(*pmd, newpmd)) 2233 set_pmd_at(mm, addr, pmd, newpmd); 2234 goto unlock; 2235 } 2236 #endif 2237 2238 if (prot_numa) { 2239 struct folio *folio; 2240 bool toptier; 2241 /* 2242 * Avoid trapping faults against the zero page. The read-only 2243 * data is likely to be read-cached on the local CPU and 2244 * local/remote hits to the zero page are not interesting. 2245 */ 2246 if (is_huge_zero_pmd(*pmd)) 2247 goto unlock; 2248 2249 if (pmd_protnone(*pmd)) 2250 goto unlock; 2251 2252 folio = pmd_folio(*pmd); 2253 toptier = node_is_toptier(folio_nid(folio)); 2254 /* 2255 * Skip scanning top tier node if normal numa 2256 * balancing is disabled 2257 */ 2258 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) && 2259 toptier) 2260 goto unlock; 2261 2262 if (folio_use_access_time(folio)) 2263 folio_xchg_access_time(folio, 2264 jiffies_to_msecs(jiffies)); 2265 } 2266 /* 2267 * In case prot_numa, we are under mmap_read_lock(mm). It's critical 2268 * to not clear pmd intermittently to avoid race with MADV_DONTNEED 2269 * which is also under mmap_read_lock(mm): 2270 * 2271 * CPU0: CPU1: 2272 * change_huge_pmd(prot_numa=1) 2273 * pmdp_huge_get_and_clear_notify() 2274 * madvise_dontneed() 2275 * zap_pmd_range() 2276 * pmd_trans_huge(*pmd) == 0 (without ptl) 2277 * // skip the pmd 2278 * set_pmd_at(); 2279 * // pmd is re-established 2280 * 2281 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it 2282 * which may break userspace. 2283 * 2284 * pmdp_invalidate_ad() is required to make sure we don't miss 2285 * dirty/young flags set by hardware. 2286 */ 2287 oldpmd = pmdp_invalidate_ad(vma, addr, pmd); 2288 2289 entry = pmd_modify(oldpmd, newprot); 2290 if (uffd_wp) 2291 entry = pmd_mkuffd_wp(entry); 2292 else if (uffd_wp_resolve) 2293 /* 2294 * Leave the write bit to be handled by PF interrupt 2295 * handler, then things like COW could be properly 2296 * handled. 2297 */ 2298 entry = pmd_clear_uffd_wp(entry); 2299 2300 /* See change_pte_range(). */ 2301 if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) && 2302 can_change_pmd_writable(vma, addr, entry)) 2303 entry = pmd_mkwrite(entry, vma); 2304 2305 ret = HPAGE_PMD_NR; 2306 set_pmd_at(mm, addr, pmd, entry); 2307 2308 if (huge_pmd_needs_flush(oldpmd, entry)) 2309 tlb_flush_pmd_range(tlb, addr, HPAGE_PMD_SIZE); 2310 unlock: 2311 spin_unlock(ptl); 2312 return ret; 2313 } 2314 2315 /* 2316 * Returns: 2317 * 2318 * - 0: if pud leaf changed from under us 2319 * - 1: if pud can be skipped 2320 * - HPAGE_PUD_NR: if pud was successfully processed 2321 */ 2322 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2323 int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 2324 pud_t *pudp, unsigned long addr, pgprot_t newprot, 2325 unsigned long cp_flags) 2326 { 2327 struct mm_struct *mm = vma->vm_mm; 2328 pud_t oldpud, entry; 2329 spinlock_t *ptl; 2330 2331 tlb_change_page_size(tlb, HPAGE_PUD_SIZE); 2332 2333 /* NUMA balancing doesn't apply to dax */ 2334 if (cp_flags & MM_CP_PROT_NUMA) 2335 return 1; 2336 2337 /* 2338 * Huge entries on userfault-wp only works with anonymous, while we 2339 * don't have anonymous PUDs yet. 2340 */ 2341 if (WARN_ON_ONCE(cp_flags & MM_CP_UFFD_WP_ALL)) 2342 return 1; 2343 2344 ptl = __pud_trans_huge_lock(pudp, vma); 2345 if (!ptl) 2346 return 0; 2347 2348 /* 2349 * Can't clear PUD or it can race with concurrent zapping. See 2350 * change_huge_pmd(). 2351 */ 2352 oldpud = pudp_invalidate(vma, addr, pudp); 2353 entry = pud_modify(oldpud, newprot); 2354 set_pud_at(mm, addr, pudp, entry); 2355 tlb_flush_pud_range(tlb, addr, HPAGE_PUD_SIZE); 2356 2357 spin_unlock(ptl); 2358 return HPAGE_PUD_NR; 2359 } 2360 #endif 2361 2362 #ifdef CONFIG_USERFAULTFD 2363 /* 2364 * The PT lock for src_pmd and dst_vma/src_vma (for reading) are locked by 2365 * the caller, but it must return after releasing the page_table_lock. 2366 * Just move the page from src_pmd to dst_pmd if possible. 2367 * Return zero if succeeded in moving the page, -EAGAIN if it needs to be 2368 * repeated by the caller, or other errors in case of failure. 2369 */ 2370 int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval, 2371 struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, 2372 unsigned long dst_addr, unsigned long src_addr) 2373 { 2374 pmd_t _dst_pmd, src_pmdval; 2375 struct page *src_page; 2376 struct folio *src_folio; 2377 struct anon_vma *src_anon_vma; 2378 spinlock_t *src_ptl, *dst_ptl; 2379 pgtable_t src_pgtable; 2380 struct mmu_notifier_range range; 2381 int err = 0; 2382 2383 src_pmdval = *src_pmd; 2384 src_ptl = pmd_lockptr(mm, src_pmd); 2385 2386 lockdep_assert_held(src_ptl); 2387 vma_assert_locked(src_vma); 2388 vma_assert_locked(dst_vma); 2389 2390 /* Sanity checks before the operation */ 2391 if (WARN_ON_ONCE(!pmd_none(dst_pmdval)) || WARN_ON_ONCE(src_addr & ~HPAGE_PMD_MASK) || 2392 WARN_ON_ONCE(dst_addr & ~HPAGE_PMD_MASK)) { 2393 spin_unlock(src_ptl); 2394 return -EINVAL; 2395 } 2396 2397 if (!pmd_trans_huge(src_pmdval)) { 2398 spin_unlock(src_ptl); 2399 if (is_pmd_migration_entry(src_pmdval)) { 2400 pmd_migration_entry_wait(mm, &src_pmdval); 2401 return -EAGAIN; 2402 } 2403 return -ENOENT; 2404 } 2405 2406 src_page = pmd_page(src_pmdval); 2407 2408 if (!is_huge_zero_pmd(src_pmdval)) { 2409 if (unlikely(!PageAnonExclusive(src_page))) { 2410 spin_unlock(src_ptl); 2411 return -EBUSY; 2412 } 2413 2414 src_folio = page_folio(src_page); 2415 folio_get(src_folio); 2416 } else 2417 src_folio = NULL; 2418 2419 spin_unlock(src_ptl); 2420 2421 flush_cache_range(src_vma, src_addr, src_addr + HPAGE_PMD_SIZE); 2422 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, src_addr, 2423 src_addr + HPAGE_PMD_SIZE); 2424 mmu_notifier_invalidate_range_start(&range); 2425 2426 if (src_folio) { 2427 folio_lock(src_folio); 2428 2429 /* 2430 * split_huge_page walks the anon_vma chain without the page 2431 * lock. Serialize against it with the anon_vma lock, the page 2432 * lock is not enough. 2433 */ 2434 src_anon_vma = folio_get_anon_vma(src_folio); 2435 if (!src_anon_vma) { 2436 err = -EAGAIN; 2437 goto unlock_folio; 2438 } 2439 anon_vma_lock_write(src_anon_vma); 2440 } else 2441 src_anon_vma = NULL; 2442 2443 dst_ptl = pmd_lockptr(mm, dst_pmd); 2444 double_pt_lock(src_ptl, dst_ptl); 2445 if (unlikely(!pmd_same(*src_pmd, src_pmdval) || 2446 !pmd_same(*dst_pmd, dst_pmdval))) { 2447 err = -EAGAIN; 2448 goto unlock_ptls; 2449 } 2450 if (src_folio) { 2451 if (folio_maybe_dma_pinned(src_folio) || 2452 !PageAnonExclusive(&src_folio->page)) { 2453 err = -EBUSY; 2454 goto unlock_ptls; 2455 } 2456 2457 if (WARN_ON_ONCE(!folio_test_head(src_folio)) || 2458 WARN_ON_ONCE(!folio_test_anon(src_folio))) { 2459 err = -EBUSY; 2460 goto unlock_ptls; 2461 } 2462 2463 src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); 2464 /* Folio got pinned from under us. Put it back and fail the move. */ 2465 if (folio_maybe_dma_pinned(src_folio)) { 2466 set_pmd_at(mm, src_addr, src_pmd, src_pmdval); 2467 err = -EBUSY; 2468 goto unlock_ptls; 2469 } 2470 2471 folio_move_anon_rmap(src_folio, dst_vma); 2472 src_folio->index = linear_page_index(dst_vma, dst_addr); 2473 2474 _dst_pmd = mk_huge_pmd(&src_folio->page, dst_vma->vm_page_prot); 2475 /* Follow mremap() behavior and treat the entry dirty after the move */ 2476 _dst_pmd = pmd_mkwrite(pmd_mkdirty(_dst_pmd), dst_vma); 2477 } else { 2478 src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); 2479 _dst_pmd = mk_huge_pmd(src_page, dst_vma->vm_page_prot); 2480 } 2481 set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd); 2482 2483 src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd); 2484 pgtable_trans_huge_deposit(mm, dst_pmd, src_pgtable); 2485 unlock_ptls: 2486 double_pt_unlock(src_ptl, dst_ptl); 2487 if (src_anon_vma) { 2488 anon_vma_unlock_write(src_anon_vma); 2489 put_anon_vma(src_anon_vma); 2490 } 2491 unlock_folio: 2492 /* unblock rmap walks */ 2493 if (src_folio) 2494 folio_unlock(src_folio); 2495 mmu_notifier_invalidate_range_end(&range); 2496 if (src_folio) 2497 folio_put(src_folio); 2498 return err; 2499 } 2500 #endif /* CONFIG_USERFAULTFD */ 2501 2502 /* 2503 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise. 2504 * 2505 * Note that if it returns page table lock pointer, this routine returns without 2506 * unlocking page table lock. So callers must unlock it. 2507 */ 2508 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) 2509 { 2510 spinlock_t *ptl; 2511 ptl = pmd_lock(vma->vm_mm, pmd); 2512 if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || 2513 pmd_devmap(*pmd))) 2514 return ptl; 2515 spin_unlock(ptl); 2516 return NULL; 2517 } 2518 2519 /* 2520 * Returns page table lock pointer if a given pud maps a thp, NULL otherwise. 2521 * 2522 * Note that if it returns page table lock pointer, this routine returns without 2523 * unlocking page table lock. So callers must unlock it. 2524 */ 2525 spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma) 2526 { 2527 spinlock_t *ptl; 2528 2529 ptl = pud_lock(vma->vm_mm, pud); 2530 if (likely(pud_trans_huge(*pud) || pud_devmap(*pud))) 2531 return ptl; 2532 spin_unlock(ptl); 2533 return NULL; 2534 } 2535 2536 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 2537 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 2538 pud_t *pud, unsigned long addr) 2539 { 2540 spinlock_t *ptl; 2541 pud_t orig_pud; 2542 2543 ptl = __pud_trans_huge_lock(pud, vma); 2544 if (!ptl) 2545 return 0; 2546 2547 orig_pud = pudp_huge_get_and_clear_full(vma, addr, pud, tlb->fullmm); 2548 arch_check_zapped_pud(vma, orig_pud); 2549 tlb_remove_pud_tlb_entry(tlb, pud, addr); 2550 if (vma_is_special_huge(vma)) { 2551 spin_unlock(ptl); 2552 /* No zero page support yet */ 2553 } else { 2554 /* No support for anonymous PUD pages yet */ 2555 BUG(); 2556 } 2557 return 1; 2558 } 2559 2560 static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud, 2561 unsigned long haddr) 2562 { 2563 VM_BUG_ON(haddr & ~HPAGE_PUD_MASK); 2564 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2565 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma); 2566 VM_BUG_ON(!pud_trans_huge(*pud) && !pud_devmap(*pud)); 2567 2568 count_vm_event(THP_SPLIT_PUD); 2569 2570 pudp_huge_clear_flush(vma, haddr, pud); 2571 } 2572 2573 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 2574 unsigned long address) 2575 { 2576 spinlock_t *ptl; 2577 struct mmu_notifier_range range; 2578 2579 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 2580 address & HPAGE_PUD_MASK, 2581 (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE); 2582 mmu_notifier_invalidate_range_start(&range); 2583 ptl = pud_lock(vma->vm_mm, pud); 2584 if (unlikely(!pud_trans_huge(*pud) && !pud_devmap(*pud))) 2585 goto out; 2586 __split_huge_pud_locked(vma, pud, range.start); 2587 2588 out: 2589 spin_unlock(ptl); 2590 mmu_notifier_invalidate_range_end(&range); 2591 } 2592 #else 2593 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 2594 unsigned long address) 2595 { 2596 } 2597 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 2598 2599 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 2600 unsigned long haddr, pmd_t *pmd) 2601 { 2602 struct mm_struct *mm = vma->vm_mm; 2603 pgtable_t pgtable; 2604 pmd_t _pmd, old_pmd; 2605 unsigned long addr; 2606 pte_t *pte; 2607 int i; 2608 2609 /* 2610 * Leave pmd empty until pte is filled note that it is fine to delay 2611 * notification until mmu_notifier_invalidate_range_end() as we are 2612 * replacing a zero pmd write protected page with a zero pte write 2613 * protected page. 2614 * 2615 * See Documentation/mm/mmu_notifier.rst 2616 */ 2617 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); 2618 2619 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2620 pmd_populate(mm, &_pmd, pgtable); 2621 2622 pte = pte_offset_map(&_pmd, haddr); 2623 VM_BUG_ON(!pte); 2624 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2625 pte_t entry; 2626 2627 entry = pfn_pte(my_zero_pfn(addr), vma->vm_page_prot); 2628 entry = pte_mkspecial(entry); 2629 if (pmd_uffd_wp(old_pmd)) 2630 entry = pte_mkuffd_wp(entry); 2631 VM_BUG_ON(!pte_none(ptep_get(pte))); 2632 set_pte_at(mm, addr, pte, entry); 2633 pte++; 2634 } 2635 pte_unmap(pte - 1); 2636 smp_wmb(); /* make pte visible before pmd */ 2637 pmd_populate(mm, pmd, pgtable); 2638 } 2639 2640 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 2641 unsigned long haddr, bool freeze) 2642 { 2643 struct mm_struct *mm = vma->vm_mm; 2644 struct folio *folio; 2645 struct page *page; 2646 pgtable_t pgtable; 2647 pmd_t old_pmd, _pmd; 2648 bool young, write, soft_dirty, pmd_migration = false, uffd_wp = false; 2649 bool anon_exclusive = false, dirty = false; 2650 unsigned long addr; 2651 pte_t *pte; 2652 int i; 2653 2654 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 2655 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2656 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 2657 VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd) 2658 && !pmd_devmap(*pmd)); 2659 2660 count_vm_event(THP_SPLIT_PMD); 2661 2662 if (!vma_is_anonymous(vma)) { 2663 old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd); 2664 /* 2665 * We are going to unmap this huge page. So 2666 * just go ahead and zap it 2667 */ 2668 if (arch_needs_pgtable_deposit()) 2669 zap_deposited_table(mm, pmd); 2670 if (vma_is_special_huge(vma)) 2671 return; 2672 if (unlikely(is_pmd_migration_entry(old_pmd))) { 2673 swp_entry_t entry; 2674 2675 entry = pmd_to_swp_entry(old_pmd); 2676 folio = pfn_swap_entry_folio(entry); 2677 } else { 2678 page = pmd_page(old_pmd); 2679 folio = page_folio(page); 2680 if (!folio_test_dirty(folio) && pmd_dirty(old_pmd)) 2681 folio_mark_dirty(folio); 2682 if (!folio_test_referenced(folio) && pmd_young(old_pmd)) 2683 folio_set_referenced(folio); 2684 folio_remove_rmap_pmd(folio, page, vma); 2685 folio_put(folio); 2686 } 2687 add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR); 2688 return; 2689 } 2690 2691 if (is_huge_zero_pmd(*pmd)) { 2692 /* 2693 * FIXME: Do we want to invalidate secondary mmu by calling 2694 * mmu_notifier_arch_invalidate_secondary_tlbs() see comments below 2695 * inside __split_huge_pmd() ? 2696 * 2697 * We are going from a zero huge page write protected to zero 2698 * small page also write protected so it does not seems useful 2699 * to invalidate secondary mmu at this time. 2700 */ 2701 return __split_huge_zero_page_pmd(vma, haddr, pmd); 2702 } 2703 2704 pmd_migration = is_pmd_migration_entry(*pmd); 2705 if (unlikely(pmd_migration)) { 2706 swp_entry_t entry; 2707 2708 old_pmd = *pmd; 2709 entry = pmd_to_swp_entry(old_pmd); 2710 page = pfn_swap_entry_to_page(entry); 2711 write = is_writable_migration_entry(entry); 2712 if (PageAnon(page)) 2713 anon_exclusive = is_readable_exclusive_migration_entry(entry); 2714 young = is_migration_entry_young(entry); 2715 dirty = is_migration_entry_dirty(entry); 2716 soft_dirty = pmd_swp_soft_dirty(old_pmd); 2717 uffd_wp = pmd_swp_uffd_wp(old_pmd); 2718 } else { 2719 /* 2720 * Up to this point the pmd is present and huge and userland has 2721 * the whole access to the hugepage during the split (which 2722 * happens in place). If we overwrite the pmd with the not-huge 2723 * version pointing to the pte here (which of course we could if 2724 * all CPUs were bug free), userland could trigger a small page 2725 * size TLB miss on the small sized TLB while the hugepage TLB 2726 * entry is still established in the huge TLB. Some CPU doesn't 2727 * like that. See 2728 * http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum 2729 * 383 on page 105. Intel should be safe but is also warns that 2730 * it's only safe if the permission and cache attributes of the 2731 * two entries loaded in the two TLB is identical (which should 2732 * be the case here). But it is generally safer to never allow 2733 * small and huge TLB entries for the same virtual address to be 2734 * loaded simultaneously. So instead of doing "pmd_populate(); 2735 * flush_pmd_tlb_range();" we first mark the current pmd 2736 * notpresent (atomically because here the pmd_trans_huge must 2737 * remain set at all times on the pmd until the split is 2738 * complete for this pmd), then we flush the SMP TLB and finally 2739 * we write the non-huge version of the pmd entry with 2740 * pmd_populate. 2741 */ 2742 old_pmd = pmdp_invalidate(vma, haddr, pmd); 2743 page = pmd_page(old_pmd); 2744 folio = page_folio(page); 2745 if (pmd_dirty(old_pmd)) { 2746 dirty = true; 2747 folio_set_dirty(folio); 2748 } 2749 write = pmd_write(old_pmd); 2750 young = pmd_young(old_pmd); 2751 soft_dirty = pmd_soft_dirty(old_pmd); 2752 uffd_wp = pmd_uffd_wp(old_pmd); 2753 2754 VM_WARN_ON_FOLIO(!folio_ref_count(folio), folio); 2755 VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); 2756 2757 /* 2758 * Without "freeze", we'll simply split the PMD, propagating the 2759 * PageAnonExclusive() flag for each PTE by setting it for 2760 * each subpage -- no need to (temporarily) clear. 2761 * 2762 * With "freeze" we want to replace mapped pages by 2763 * migration entries right away. This is only possible if we 2764 * managed to clear PageAnonExclusive() -- see 2765 * set_pmd_migration_entry(). 2766 * 2767 * In case we cannot clear PageAnonExclusive(), split the PMD 2768 * only and let try_to_migrate_one() fail later. 2769 * 2770 * See folio_try_share_anon_rmap_pmd(): invalidate PMD first. 2771 */ 2772 anon_exclusive = PageAnonExclusive(page); 2773 if (freeze && anon_exclusive && 2774 folio_try_share_anon_rmap_pmd(folio, page)) 2775 freeze = false; 2776 if (!freeze) { 2777 rmap_t rmap_flags = RMAP_NONE; 2778 2779 folio_ref_add(folio, HPAGE_PMD_NR - 1); 2780 if (anon_exclusive) 2781 rmap_flags |= RMAP_EXCLUSIVE; 2782 folio_add_anon_rmap_ptes(folio, page, HPAGE_PMD_NR, 2783 vma, haddr, rmap_flags); 2784 } 2785 } 2786 2787 /* 2788 * Withdraw the table only after we mark the pmd entry invalid. 2789 * This's critical for some architectures (Power). 2790 */ 2791 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2792 pmd_populate(mm, &_pmd, pgtable); 2793 2794 pte = pte_offset_map(&_pmd, haddr); 2795 VM_BUG_ON(!pte); 2796 2797 /* 2798 * Note that NUMA hinting access restrictions are not transferred to 2799 * avoid any possibility of altering permissions across VMAs. 2800 */ 2801 if (freeze || pmd_migration) { 2802 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2803 pte_t entry; 2804 swp_entry_t swp_entry; 2805 2806 if (write) 2807 swp_entry = make_writable_migration_entry( 2808 page_to_pfn(page + i)); 2809 else if (anon_exclusive) 2810 swp_entry = make_readable_exclusive_migration_entry( 2811 page_to_pfn(page + i)); 2812 else 2813 swp_entry = make_readable_migration_entry( 2814 page_to_pfn(page + i)); 2815 if (young) 2816 swp_entry = make_migration_entry_young(swp_entry); 2817 if (dirty) 2818 swp_entry = make_migration_entry_dirty(swp_entry); 2819 entry = swp_entry_to_pte(swp_entry); 2820 if (soft_dirty) 2821 entry = pte_swp_mksoft_dirty(entry); 2822 if (uffd_wp) 2823 entry = pte_swp_mkuffd_wp(entry); 2824 2825 VM_WARN_ON(!pte_none(ptep_get(pte + i))); 2826 set_pte_at(mm, addr, pte + i, entry); 2827 } 2828 } else { 2829 pte_t entry; 2830 2831 entry = mk_pte(page, READ_ONCE(vma->vm_page_prot)); 2832 if (write) 2833 entry = pte_mkwrite(entry, vma); 2834 if (!young) 2835 entry = pte_mkold(entry); 2836 /* NOTE: this may set soft-dirty too on some archs */ 2837 if (dirty) 2838 entry = pte_mkdirty(entry); 2839 if (soft_dirty) 2840 entry = pte_mksoft_dirty(entry); 2841 if (uffd_wp) 2842 entry = pte_mkuffd_wp(entry); 2843 2844 for (i = 0; i < HPAGE_PMD_NR; i++) 2845 VM_WARN_ON(!pte_none(ptep_get(pte + i))); 2846 2847 set_ptes(mm, haddr, pte, entry, HPAGE_PMD_NR); 2848 } 2849 pte_unmap(pte); 2850 2851 if (!pmd_migration) 2852 folio_remove_rmap_pmd(folio, page, vma); 2853 if (freeze) 2854 put_page(page); 2855 2856 smp_wmb(); /* make pte visible before pmd */ 2857 pmd_populate(mm, pmd, pgtable); 2858 } 2859 2860 void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, 2861 pmd_t *pmd, bool freeze, struct folio *folio) 2862 { 2863 VM_WARN_ON_ONCE(folio && !folio_test_pmd_mappable(folio)); 2864 VM_WARN_ON_ONCE(!IS_ALIGNED(address, HPAGE_PMD_SIZE)); 2865 VM_WARN_ON_ONCE(folio && !folio_test_locked(folio)); 2866 VM_BUG_ON(freeze && !folio); 2867 2868 /* 2869 * When the caller requests to set up a migration entry, we 2870 * require a folio to check the PMD against. Otherwise, there 2871 * is a risk of replacing the wrong folio. 2872 */ 2873 if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd) || 2874 is_pmd_migration_entry(*pmd)) { 2875 if (folio && folio != pmd_folio(*pmd)) 2876 return; 2877 __split_huge_pmd_locked(vma, pmd, address, freeze); 2878 } 2879 } 2880 2881 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 2882 unsigned long address, bool freeze, struct folio *folio) 2883 { 2884 spinlock_t *ptl; 2885 struct mmu_notifier_range range; 2886 2887 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, 2888 address & HPAGE_PMD_MASK, 2889 (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE); 2890 mmu_notifier_invalidate_range_start(&range); 2891 ptl = pmd_lock(vma->vm_mm, pmd); 2892 split_huge_pmd_locked(vma, range.start, pmd, freeze, folio); 2893 spin_unlock(ptl); 2894 mmu_notifier_invalidate_range_end(&range); 2895 } 2896 2897 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, 2898 bool freeze, struct folio *folio) 2899 { 2900 pmd_t *pmd = mm_find_pmd(vma->vm_mm, address); 2901 2902 if (!pmd) 2903 return; 2904 2905 __split_huge_pmd(vma, pmd, address, freeze, folio); 2906 } 2907 2908 static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address) 2909 { 2910 /* 2911 * If the new address isn't hpage aligned and it could previously 2912 * contain an hugepage: check if we need to split an huge pmd. 2913 */ 2914 if (!IS_ALIGNED(address, HPAGE_PMD_SIZE) && 2915 range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE), 2916 ALIGN(address, HPAGE_PMD_SIZE))) 2917 split_huge_pmd_address(vma, address, false, NULL); 2918 } 2919 2920 void vma_adjust_trans_huge(struct vm_area_struct *vma, 2921 unsigned long start, 2922 unsigned long end, 2923 long adjust_next) 2924 { 2925 /* Check if we need to split start first. */ 2926 split_huge_pmd_if_needed(vma, start); 2927 2928 /* Check if we need to split end next. */ 2929 split_huge_pmd_if_needed(vma, end); 2930 2931 /* 2932 * If we're also updating the next vma vm_start, 2933 * check if we need to split it. 2934 */ 2935 if (adjust_next > 0) { 2936 struct vm_area_struct *next = find_vma(vma->vm_mm, vma->vm_end); 2937 unsigned long nstart = next->vm_start; 2938 nstart += adjust_next; 2939 split_huge_pmd_if_needed(next, nstart); 2940 } 2941 } 2942 2943 static void unmap_folio(struct folio *folio) 2944 { 2945 enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SYNC | 2946 TTU_BATCH_FLUSH; 2947 2948 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 2949 2950 if (folio_test_pmd_mappable(folio)) 2951 ttu_flags |= TTU_SPLIT_HUGE_PMD; 2952 2953 /* 2954 * Anon pages need migration entries to preserve them, but file 2955 * pages can simply be left unmapped, then faulted back on demand. 2956 * If that is ever changed (perhaps for mlock), update remap_page(). 2957 */ 2958 if (folio_test_anon(folio)) 2959 try_to_migrate(folio, ttu_flags); 2960 else 2961 try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK); 2962 2963 try_to_unmap_flush(); 2964 } 2965 2966 static bool __discard_anon_folio_pmd_locked(struct vm_area_struct *vma, 2967 unsigned long addr, pmd_t *pmdp, 2968 struct folio *folio) 2969 { 2970 struct mm_struct *mm = vma->vm_mm; 2971 int ref_count, map_count; 2972 pmd_t orig_pmd = *pmdp; 2973 2974 if (folio_test_dirty(folio) || pmd_dirty(orig_pmd)) 2975 return false; 2976 2977 orig_pmd = pmdp_huge_clear_flush(vma, addr, pmdp); 2978 2979 /* 2980 * Syncing against concurrent GUP-fast: 2981 * - clear PMD; barrier; read refcount 2982 * - inc refcount; barrier; read PMD 2983 */ 2984 smp_mb(); 2985 2986 ref_count = folio_ref_count(folio); 2987 map_count = folio_mapcount(folio); 2988 2989 /* 2990 * Order reads for folio refcount and dirty flag 2991 * (see comments in __remove_mapping()). 2992 */ 2993 smp_rmb(); 2994 2995 /* 2996 * If the folio or its PMD is redirtied at this point, or if there 2997 * are unexpected references, we will give up to discard this folio 2998 * and remap it. 2999 * 3000 * The only folio refs must be one from isolation plus the rmap(s). 3001 */ 3002 if (folio_test_dirty(folio) || pmd_dirty(orig_pmd) || 3003 ref_count != map_count + 1) { 3004 set_pmd_at(mm, addr, pmdp, orig_pmd); 3005 return false; 3006 } 3007 3008 folio_remove_rmap_pmd(folio, pmd_page(orig_pmd), vma); 3009 zap_deposited_table(mm, pmdp); 3010 add_mm_counter(mm, MM_ANONPAGES, -HPAGE_PMD_NR); 3011 if (vma->vm_flags & VM_LOCKED) 3012 mlock_drain_local(); 3013 folio_put(folio); 3014 3015 return true; 3016 } 3017 3018 bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr, 3019 pmd_t *pmdp, struct folio *folio) 3020 { 3021 VM_WARN_ON_FOLIO(!folio_test_pmd_mappable(folio), folio); 3022 VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); 3023 VM_WARN_ON_ONCE(!IS_ALIGNED(addr, HPAGE_PMD_SIZE)); 3024 3025 if (folio_test_anon(folio) && !folio_test_swapbacked(folio)) 3026 return __discard_anon_folio_pmd_locked(vma, addr, pmdp, folio); 3027 3028 return false; 3029 } 3030 3031 static void remap_page(struct folio *folio, unsigned long nr, int flags) 3032 { 3033 int i = 0; 3034 3035 /* If unmap_folio() uses try_to_migrate() on file, remove this check */ 3036 if (!folio_test_anon(folio)) 3037 return; 3038 for (;;) { 3039 remove_migration_ptes(folio, folio, RMP_LOCKED | flags); 3040 i += folio_nr_pages(folio); 3041 if (i >= nr) 3042 break; 3043 folio = folio_next(folio); 3044 } 3045 } 3046 3047 static void lru_add_page_tail(struct folio *folio, struct page *tail, 3048 struct lruvec *lruvec, struct list_head *list) 3049 { 3050 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 3051 VM_BUG_ON_FOLIO(PageLRU(tail), folio); 3052 lockdep_assert_held(&lruvec->lru_lock); 3053 3054 if (list) { 3055 /* page reclaim is reclaiming a huge page */ 3056 VM_WARN_ON(folio_test_lru(folio)); 3057 get_page(tail); 3058 list_add_tail(&tail->lru, list); 3059 } else { 3060 /* head is still on lru (and we have it frozen) */ 3061 VM_WARN_ON(!folio_test_lru(folio)); 3062 if (folio_test_unevictable(folio)) 3063 tail->mlock_count = 0; 3064 else 3065 list_add_tail(&tail->lru, &folio->lru); 3066 SetPageLRU(tail); 3067 } 3068 } 3069 3070 static void __split_huge_page_tail(struct folio *folio, int tail, 3071 struct lruvec *lruvec, struct list_head *list, 3072 unsigned int new_order) 3073 { 3074 struct page *head = &folio->page; 3075 struct page *page_tail = head + tail; 3076 /* 3077 * Careful: new_folio is not a "real" folio before we cleared PageTail. 3078 * Don't pass it around before clear_compound_head(). 3079 */ 3080 struct folio *new_folio = (struct folio *)page_tail; 3081 3082 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail); 3083 3084 /* 3085 * Clone page flags before unfreezing refcount. 3086 * 3087 * After successful get_page_unless_zero() might follow flags change, 3088 * for example lock_page() which set PG_waiters. 3089 * 3090 * Note that for mapped sub-pages of an anonymous THP, 3091 * PG_anon_exclusive has been cleared in unmap_folio() and is stored in 3092 * the migration entry instead from where remap_page() will restore it. 3093 * We can still have PG_anon_exclusive set on effectively unmapped and 3094 * unreferenced sub-pages of an anonymous THP: we can simply drop 3095 * PG_anon_exclusive (-> PG_mappedtodisk) for these here. 3096 */ 3097 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 3098 page_tail->flags |= (head->flags & 3099 ((1L << PG_referenced) | 3100 (1L << PG_swapbacked) | 3101 (1L << PG_swapcache) | 3102 (1L << PG_mlocked) | 3103 (1L << PG_uptodate) | 3104 (1L << PG_active) | 3105 (1L << PG_workingset) | 3106 (1L << PG_locked) | 3107 (1L << PG_unevictable) | 3108 #ifdef CONFIG_ARCH_USES_PG_ARCH_2 3109 (1L << PG_arch_2) | 3110 #endif 3111 #ifdef CONFIG_ARCH_USES_PG_ARCH_3 3112 (1L << PG_arch_3) | 3113 #endif 3114 (1L << PG_dirty) | 3115 LRU_GEN_MASK | LRU_REFS_MASK)); 3116 3117 /* ->mapping in first and second tail page is replaced by other uses */ 3118 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, 3119 page_tail); 3120 page_tail->mapping = head->mapping; 3121 page_tail->index = head->index + tail; 3122 3123 /* 3124 * page->private should not be set in tail pages. Fix up and warn once 3125 * if private is unexpectedly set. 3126 */ 3127 if (unlikely(page_tail->private)) { 3128 VM_WARN_ON_ONCE_PAGE(true, page_tail); 3129 page_tail->private = 0; 3130 } 3131 if (folio_test_swapcache(folio)) 3132 new_folio->swap.val = folio->swap.val + tail; 3133 3134 /* Page flags must be visible before we make the page non-compound. */ 3135 smp_wmb(); 3136 3137 /* 3138 * Clear PageTail before unfreezing page refcount. 3139 * 3140 * After successful get_page_unless_zero() might follow put_page() 3141 * which needs correct compound_head(). 3142 */ 3143 clear_compound_head(page_tail); 3144 if (new_order) { 3145 prep_compound_page(page_tail, new_order); 3146 folio_set_large_rmappable(new_folio); 3147 } 3148 3149 /* Finally unfreeze refcount. Additional reference from page cache. */ 3150 page_ref_unfreeze(page_tail, 3151 1 + ((!folio_test_anon(folio) || folio_test_swapcache(folio)) ? 3152 folio_nr_pages(new_folio) : 0)); 3153 3154 if (folio_test_young(folio)) 3155 folio_set_young(new_folio); 3156 if (folio_test_idle(folio)) 3157 folio_set_idle(new_folio); 3158 3159 folio_xchg_last_cpupid(new_folio, folio_last_cpupid(folio)); 3160 3161 /* 3162 * always add to the tail because some iterators expect new 3163 * pages to show after the currently processed elements - e.g. 3164 * migrate_pages 3165 */ 3166 lru_add_page_tail(folio, page_tail, lruvec, list); 3167 } 3168 3169 static void __split_huge_page(struct page *page, struct list_head *list, 3170 pgoff_t end, unsigned int new_order) 3171 { 3172 struct folio *folio = page_folio(page); 3173 struct page *head = &folio->page; 3174 struct lruvec *lruvec; 3175 struct address_space *swap_cache = NULL; 3176 unsigned long offset = 0; 3177 int i, nr_dropped = 0; 3178 unsigned int new_nr = 1 << new_order; 3179 int order = folio_order(folio); 3180 unsigned int nr = 1 << order; 3181 3182 /* complete memcg works before add pages to LRU */ 3183 split_page_memcg(head, order, new_order); 3184 3185 if (folio_test_anon(folio) && folio_test_swapcache(folio)) { 3186 offset = swap_cache_index(folio->swap); 3187 swap_cache = swap_address_space(folio->swap); 3188 xa_lock(&swap_cache->i_pages); 3189 } 3190 3191 /* lock lru list/PageCompound, ref frozen by page_ref_freeze */ 3192 lruvec = folio_lruvec_lock(folio); 3193 3194 ClearPageHasHWPoisoned(head); 3195 3196 for (i = nr - new_nr; i >= new_nr; i -= new_nr) { 3197 __split_huge_page_tail(folio, i, lruvec, list, new_order); 3198 /* Some pages can be beyond EOF: drop them from page cache */ 3199 if (head[i].index >= end) { 3200 struct folio *tail = page_folio(head + i); 3201 3202 if (shmem_mapping(folio->mapping)) 3203 nr_dropped++; 3204 else if (folio_test_clear_dirty(tail)) 3205 folio_account_cleaned(tail, 3206 inode_to_wb(folio->mapping->host)); 3207 __filemap_remove_folio(tail, NULL); 3208 folio_put(tail); 3209 } else if (!PageAnon(page)) { 3210 __xa_store(&folio->mapping->i_pages, head[i].index, 3211 head + i, 0); 3212 } else if (swap_cache) { 3213 __xa_store(&swap_cache->i_pages, offset + i, 3214 head + i, 0); 3215 } 3216 } 3217 3218 if (!new_order) 3219 ClearPageCompound(head); 3220 else { 3221 struct folio *new_folio = (struct folio *)head; 3222 3223 folio_set_order(new_folio, new_order); 3224 } 3225 unlock_page_lruvec(lruvec); 3226 /* Caller disabled irqs, so they are still disabled here */ 3227 3228 split_page_owner(head, order, new_order); 3229 pgalloc_tag_split(head, 1 << order); 3230 3231 /* See comment in __split_huge_page_tail() */ 3232 if (folio_test_anon(folio)) { 3233 /* Additional pin to swap cache */ 3234 if (folio_test_swapcache(folio)) { 3235 folio_ref_add(folio, 1 + new_nr); 3236 xa_unlock(&swap_cache->i_pages); 3237 } else { 3238 folio_ref_inc(folio); 3239 } 3240 } else { 3241 /* Additional pin to page cache */ 3242 folio_ref_add(folio, 1 + new_nr); 3243 xa_unlock(&folio->mapping->i_pages); 3244 } 3245 local_irq_enable(); 3246 3247 if (nr_dropped) 3248 shmem_uncharge(folio->mapping->host, nr_dropped); 3249 remap_page(folio, nr, PageAnon(head) ? RMP_USE_SHARED_ZEROPAGE : 0); 3250 3251 /* 3252 * set page to its compound_head when split to non order-0 pages, so 3253 * we can skip unlocking it below, since PG_locked is transferred to 3254 * the compound_head of the page and the caller will unlock it. 3255 */ 3256 if (new_order) 3257 page = compound_head(page); 3258 3259 for (i = 0; i < nr; i += new_nr) { 3260 struct page *subpage = head + i; 3261 struct folio *new_folio = page_folio(subpage); 3262 if (subpage == page) 3263 continue; 3264 folio_unlock(new_folio); 3265 3266 /* 3267 * Subpages may be freed if there wasn't any mapping 3268 * like if add_to_swap() is running on a lru page that 3269 * had its mapping zapped. And freeing these pages 3270 * requires taking the lru_lock so we do the put_page 3271 * of the tail pages after the split is complete. 3272 */ 3273 free_page_and_swap_cache(subpage); 3274 } 3275 } 3276 3277 /* Racy check whether the huge page can be split */ 3278 bool can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins) 3279 { 3280 int extra_pins; 3281 3282 /* Additional pins from page cache */ 3283 if (folio_test_anon(folio)) 3284 extra_pins = folio_test_swapcache(folio) ? 3285 folio_nr_pages(folio) : 0; 3286 else 3287 extra_pins = folio_nr_pages(folio); 3288 if (pextra_pins) 3289 *pextra_pins = extra_pins; 3290 return folio_mapcount(folio) == folio_ref_count(folio) - extra_pins - 3291 caller_pins; 3292 } 3293 3294 /* 3295 * This function splits a large folio into smaller folios of order @new_order. 3296 * @page can point to any page of the large folio to split. The split operation 3297 * does not change the position of @page. 3298 * 3299 * Prerequisites: 3300 * 3301 * 1) The caller must hold a reference on the @page's owning folio, also known 3302 * as the large folio. 3303 * 3304 * 2) The large folio must be locked. 3305 * 3306 * 3) The folio must not be pinned. Any unexpected folio references, including 3307 * GUP pins, will result in the folio not getting split; instead, the caller 3308 * will receive an -EAGAIN. 3309 * 3310 * 4) @new_order > 1, usually. Splitting to order-1 anonymous folios is not 3311 * supported for non-file-backed folios, because folio->_deferred_list, which 3312 * is used by partially mapped folios, is stored in subpage 2, but an order-1 3313 * folio only has subpages 0 and 1. File-backed order-1 folios are supported, 3314 * since they do not use _deferred_list. 3315 * 3316 * After splitting, the caller's folio reference will be transferred to @page, 3317 * resulting in a raised refcount of @page after this call. The other pages may 3318 * be freed if they are not mapped. 3319 * 3320 * If @list is null, tail pages will be added to LRU list, otherwise, to @list. 3321 * 3322 * Pages in @new_order will inherit the mapping, flags, and so on from the 3323 * huge page. 3324 * 3325 * Returns 0 if the huge page was split successfully. 3326 * 3327 * Returns -EAGAIN if the folio has unexpected reference (e.g., GUP) or if 3328 * the folio was concurrently removed from the page cache. 3329 * 3330 * Returns -EBUSY when trying to split the huge zeropage, if the folio is 3331 * under writeback, if fs-specific folio metadata cannot currently be 3332 * released, or if some unexpected race happened (e.g., anon VMA disappeared, 3333 * truncation). 3334 * 3335 * Returns -EINVAL when trying to split to an order that is incompatible 3336 * with the folio. Splitting to order 0 is compatible with all folios. 3337 */ 3338 int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, 3339 unsigned int new_order) 3340 { 3341 struct folio *folio = page_folio(page); 3342 struct deferred_split *ds_queue = get_deferred_split_queue(folio); 3343 /* reset xarray order to new order after split */ 3344 XA_STATE_ORDER(xas, &folio->mapping->i_pages, folio->index, new_order); 3345 bool is_anon = folio_test_anon(folio); 3346 struct address_space *mapping = NULL; 3347 struct anon_vma *anon_vma = NULL; 3348 int order = folio_order(folio); 3349 int extra_pins, ret; 3350 pgoff_t end; 3351 bool is_hzp; 3352 3353 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 3354 VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); 3355 3356 if (new_order >= folio_order(folio)) 3357 return -EINVAL; 3358 3359 if (is_anon) { 3360 /* order-1 is not supported for anonymous THP. */ 3361 if (new_order == 1) { 3362 VM_WARN_ONCE(1, "Cannot split to order-1 folio"); 3363 return -EINVAL; 3364 } 3365 } else if (new_order) { 3366 /* Split shmem folio to non-zero order not supported */ 3367 if (shmem_mapping(folio->mapping)) { 3368 VM_WARN_ONCE(1, 3369 "Cannot split shmem folio to non-0 order"); 3370 return -EINVAL; 3371 } 3372 /* 3373 * No split if the file system does not support large folio. 3374 * Note that we might still have THPs in such mappings due to 3375 * CONFIG_READ_ONLY_THP_FOR_FS. But in that case, the mapping 3376 * does not actually support large folios properly. 3377 */ 3378 if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && 3379 !mapping_large_folio_support(folio->mapping)) { 3380 VM_WARN_ONCE(1, 3381 "Cannot split file folio to non-0 order"); 3382 return -EINVAL; 3383 } 3384 } 3385 3386 /* Only swapping a whole PMD-mapped folio is supported */ 3387 if (folio_test_swapcache(folio) && new_order) 3388 return -EINVAL; 3389 3390 is_hzp = is_huge_zero_folio(folio); 3391 if (is_hzp) { 3392 pr_warn_ratelimited("Called split_huge_page for huge zero page\n"); 3393 return -EBUSY; 3394 } 3395 3396 if (folio_test_writeback(folio)) 3397 return -EBUSY; 3398 3399 if (is_anon) { 3400 /* 3401 * The caller does not necessarily hold an mmap_lock that would 3402 * prevent the anon_vma disappearing so we first we take a 3403 * reference to it and then lock the anon_vma for write. This 3404 * is similar to folio_lock_anon_vma_read except the write lock 3405 * is taken to serialise against parallel split or collapse 3406 * operations. 3407 */ 3408 anon_vma = folio_get_anon_vma(folio); 3409 if (!anon_vma) { 3410 ret = -EBUSY; 3411 goto out; 3412 } 3413 end = -1; 3414 mapping = NULL; 3415 anon_vma_lock_write(anon_vma); 3416 } else { 3417 gfp_t gfp; 3418 3419 mapping = folio->mapping; 3420 3421 /* Truncated ? */ 3422 if (!mapping) { 3423 ret = -EBUSY; 3424 goto out; 3425 } 3426 3427 gfp = current_gfp_context(mapping_gfp_mask(mapping) & 3428 GFP_RECLAIM_MASK); 3429 3430 if (!filemap_release_folio(folio, gfp)) { 3431 ret = -EBUSY; 3432 goto out; 3433 } 3434 3435 xas_split_alloc(&xas, folio, folio_order(folio), gfp); 3436 if (xas_error(&xas)) { 3437 ret = xas_error(&xas); 3438 goto out; 3439 } 3440 3441 anon_vma = NULL; 3442 i_mmap_lock_read(mapping); 3443 3444 /* 3445 *__split_huge_page() may need to trim off pages beyond EOF: 3446 * but on 32-bit, i_size_read() takes an irq-unsafe seqlock, 3447 * which cannot be nested inside the page tree lock. So note 3448 * end now: i_size itself may be changed at any moment, but 3449 * folio lock is good enough to serialize the trimming. 3450 */ 3451 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); 3452 if (shmem_mapping(mapping)) 3453 end = shmem_fallocend(mapping->host, end); 3454 } 3455 3456 /* 3457 * Racy check if we can split the page, before unmap_folio() will 3458 * split PMDs 3459 */ 3460 if (!can_split_folio(folio, 1, &extra_pins)) { 3461 ret = -EAGAIN; 3462 goto out_unlock; 3463 } 3464 3465 unmap_folio(folio); 3466 3467 /* block interrupt reentry in xa_lock and spinlock */ 3468 local_irq_disable(); 3469 if (mapping) { 3470 /* 3471 * Check if the folio is present in page cache. 3472 * We assume all tail are present too, if folio is there. 3473 */ 3474 xas_lock(&xas); 3475 xas_reset(&xas); 3476 if (xas_load(&xas) != folio) 3477 goto fail; 3478 } 3479 3480 /* Prevent deferred_split_scan() touching ->_refcount */ 3481 spin_lock(&ds_queue->split_queue_lock); 3482 if (folio_ref_freeze(folio, 1 + extra_pins)) { 3483 if (folio_order(folio) > 1 && 3484 !list_empty(&folio->_deferred_list)) { 3485 ds_queue->split_queue_len--; 3486 if (folio_test_partially_mapped(folio)) { 3487 __folio_clear_partially_mapped(folio); 3488 mod_mthp_stat(folio_order(folio), 3489 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 3490 } 3491 /* 3492 * Reinitialize page_deferred_list after removing the 3493 * page from the split_queue, otherwise a subsequent 3494 * split will see list corruption when checking the 3495 * page_deferred_list. 3496 */ 3497 list_del_init(&folio->_deferred_list); 3498 } 3499 spin_unlock(&ds_queue->split_queue_lock); 3500 if (mapping) { 3501 int nr = folio_nr_pages(folio); 3502 3503 xas_split(&xas, folio, folio_order(folio)); 3504 if (folio_test_pmd_mappable(folio) && 3505 new_order < HPAGE_PMD_ORDER) { 3506 if (folio_test_swapbacked(folio)) { 3507 __lruvec_stat_mod_folio(folio, 3508 NR_SHMEM_THPS, -nr); 3509 } else { 3510 __lruvec_stat_mod_folio(folio, 3511 NR_FILE_THPS, -nr); 3512 filemap_nr_thps_dec(mapping); 3513 } 3514 } 3515 } 3516 3517 if (is_anon) { 3518 mod_mthp_stat(order, MTHP_STAT_NR_ANON, -1); 3519 mod_mthp_stat(new_order, MTHP_STAT_NR_ANON, 1 << (order - new_order)); 3520 } 3521 __split_huge_page(page, list, end, new_order); 3522 ret = 0; 3523 } else { 3524 spin_unlock(&ds_queue->split_queue_lock); 3525 fail: 3526 if (mapping) 3527 xas_unlock(&xas); 3528 local_irq_enable(); 3529 remap_page(folio, folio_nr_pages(folio), 0); 3530 ret = -EAGAIN; 3531 } 3532 3533 out_unlock: 3534 if (anon_vma) { 3535 anon_vma_unlock_write(anon_vma); 3536 put_anon_vma(anon_vma); 3537 } 3538 if (mapping) 3539 i_mmap_unlock_read(mapping); 3540 out: 3541 xas_destroy(&xas); 3542 if (order == HPAGE_PMD_ORDER) 3543 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); 3544 count_mthp_stat(order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED); 3545 return ret; 3546 } 3547 3548 void __folio_undo_large_rmappable(struct folio *folio) 3549 { 3550 struct deferred_split *ds_queue; 3551 unsigned long flags; 3552 3553 ds_queue = get_deferred_split_queue(folio); 3554 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 3555 if (!list_empty(&folio->_deferred_list)) { 3556 ds_queue->split_queue_len--; 3557 if (folio_test_partially_mapped(folio)) { 3558 __folio_clear_partially_mapped(folio); 3559 mod_mthp_stat(folio_order(folio), 3560 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 3561 } 3562 list_del_init(&folio->_deferred_list); 3563 } 3564 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 3565 } 3566 3567 /* partially_mapped=false won't clear PG_partially_mapped folio flag */ 3568 void deferred_split_folio(struct folio *folio, bool partially_mapped) 3569 { 3570 struct deferred_split *ds_queue = get_deferred_split_queue(folio); 3571 #ifdef CONFIG_MEMCG 3572 struct mem_cgroup *memcg = folio_memcg(folio); 3573 #endif 3574 unsigned long flags; 3575 3576 /* 3577 * Order 1 folios have no space for a deferred list, but we also 3578 * won't waste much memory by not adding them to the deferred list. 3579 */ 3580 if (folio_order(folio) <= 1) 3581 return; 3582 3583 if (!partially_mapped && !split_underused_thp) 3584 return; 3585 3586 /* 3587 * The try_to_unmap() in page reclaim path might reach here too, 3588 * this may cause a race condition to corrupt deferred split queue. 3589 * And, if page reclaim is already handling the same folio, it is 3590 * unnecessary to handle it again in shrinker. 3591 * 3592 * Check the swapcache flag to determine if the folio is being 3593 * handled by page reclaim since THP swap would add the folio into 3594 * swap cache before calling try_to_unmap(). 3595 */ 3596 if (folio_test_swapcache(folio)) 3597 return; 3598 3599 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 3600 if (partially_mapped) { 3601 if (!folio_test_partially_mapped(folio)) { 3602 __folio_set_partially_mapped(folio); 3603 if (folio_test_pmd_mappable(folio)) 3604 count_vm_event(THP_DEFERRED_SPLIT_PAGE); 3605 count_mthp_stat(folio_order(folio), MTHP_STAT_SPLIT_DEFERRED); 3606 mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, 1); 3607 3608 } 3609 } else { 3610 /* partially mapped folios cannot become non-partially mapped */ 3611 VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio); 3612 } 3613 if (list_empty(&folio->_deferred_list)) { 3614 list_add_tail(&folio->_deferred_list, &ds_queue->split_queue); 3615 ds_queue->split_queue_len++; 3616 #ifdef CONFIG_MEMCG 3617 if (memcg) 3618 set_shrinker_bit(memcg, folio_nid(folio), 3619 deferred_split_shrinker->id); 3620 #endif 3621 } 3622 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 3623 } 3624 3625 static unsigned long deferred_split_count(struct shrinker *shrink, 3626 struct shrink_control *sc) 3627 { 3628 struct pglist_data *pgdata = NODE_DATA(sc->nid); 3629 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 3630 3631 #ifdef CONFIG_MEMCG 3632 if (sc->memcg) 3633 ds_queue = &sc->memcg->deferred_split_queue; 3634 #endif 3635 return READ_ONCE(ds_queue->split_queue_len); 3636 } 3637 3638 static bool thp_underused(struct folio *folio) 3639 { 3640 int num_zero_pages = 0, num_filled_pages = 0; 3641 void *kaddr; 3642 int i; 3643 3644 if (khugepaged_max_ptes_none == HPAGE_PMD_NR - 1) 3645 return false; 3646 3647 for (i = 0; i < folio_nr_pages(folio); i++) { 3648 kaddr = kmap_local_folio(folio, i * PAGE_SIZE); 3649 if (!memchr_inv(kaddr, 0, PAGE_SIZE)) { 3650 num_zero_pages++; 3651 if (num_zero_pages > khugepaged_max_ptes_none) { 3652 kunmap_local(kaddr); 3653 return true; 3654 } 3655 } else { 3656 /* 3657 * Another path for early exit once the number 3658 * of non-zero filled pages exceeds threshold. 3659 */ 3660 num_filled_pages++; 3661 if (num_filled_pages >= HPAGE_PMD_NR - khugepaged_max_ptes_none) { 3662 kunmap_local(kaddr); 3663 return false; 3664 } 3665 } 3666 kunmap_local(kaddr); 3667 } 3668 return false; 3669 } 3670 3671 static unsigned long deferred_split_scan(struct shrinker *shrink, 3672 struct shrink_control *sc) 3673 { 3674 struct pglist_data *pgdata = NODE_DATA(sc->nid); 3675 struct deferred_split *ds_queue = &pgdata->deferred_split_queue; 3676 unsigned long flags; 3677 LIST_HEAD(list); 3678 struct folio *folio, *next; 3679 int split = 0; 3680 3681 #ifdef CONFIG_MEMCG 3682 if (sc->memcg) 3683 ds_queue = &sc->memcg->deferred_split_queue; 3684 #endif 3685 3686 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 3687 /* Take pin on all head pages to avoid freeing them under us */ 3688 list_for_each_entry_safe(folio, next, &ds_queue->split_queue, 3689 _deferred_list) { 3690 if (folio_try_get(folio)) { 3691 list_move(&folio->_deferred_list, &list); 3692 } else { 3693 /* We lost race with folio_put() */ 3694 if (folio_test_partially_mapped(folio)) { 3695 __folio_clear_partially_mapped(folio); 3696 mod_mthp_stat(folio_order(folio), 3697 MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); 3698 } 3699 list_del_init(&folio->_deferred_list); 3700 ds_queue->split_queue_len--; 3701 } 3702 if (!--sc->nr_to_scan) 3703 break; 3704 } 3705 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 3706 3707 list_for_each_entry_safe(folio, next, &list, _deferred_list) { 3708 bool did_split = false; 3709 bool underused = false; 3710 3711 if (!folio_test_partially_mapped(folio)) { 3712 underused = thp_underused(folio); 3713 if (!underused) 3714 goto next; 3715 } 3716 if (!folio_trylock(folio)) 3717 goto next; 3718 if (!split_folio(folio)) { 3719 did_split = true; 3720 if (underused) 3721 count_vm_event(THP_UNDERUSED_SPLIT_PAGE); 3722 split++; 3723 } 3724 folio_unlock(folio); 3725 next: 3726 /* 3727 * split_folio() removes folio from list on success. 3728 * Only add back to the queue if folio is partially mapped. 3729 * If thp_underused returns false, or if split_folio fails 3730 * in the case it was underused, then consider it used and 3731 * don't add it back to split_queue. 3732 */ 3733 if (!did_split && !folio_test_partially_mapped(folio)) { 3734 list_del_init(&folio->_deferred_list); 3735 ds_queue->split_queue_len--; 3736 } 3737 folio_put(folio); 3738 } 3739 3740 spin_lock_irqsave(&ds_queue->split_queue_lock, flags); 3741 list_splice_tail(&list, &ds_queue->split_queue); 3742 spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); 3743 3744 /* 3745 * Stop shrinker if we didn't split any page, but the queue is empty. 3746 * This can happen if pages were freed under us. 3747 */ 3748 if (!split && list_empty(&ds_queue->split_queue)) 3749 return SHRINK_STOP; 3750 return split; 3751 } 3752 3753 #ifdef CONFIG_DEBUG_FS 3754 static void split_huge_pages_all(void) 3755 { 3756 struct zone *zone; 3757 struct page *page; 3758 struct folio *folio; 3759 unsigned long pfn, max_zone_pfn; 3760 unsigned long total = 0, split = 0; 3761 3762 pr_debug("Split all THPs\n"); 3763 for_each_zone(zone) { 3764 if (!managed_zone(zone)) 3765 continue; 3766 max_zone_pfn = zone_end_pfn(zone); 3767 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 3768 int nr_pages; 3769 3770 page = pfn_to_online_page(pfn); 3771 if (!page || PageTail(page)) 3772 continue; 3773 folio = page_folio(page); 3774 if (!folio_try_get(folio)) 3775 continue; 3776 3777 if (unlikely(page_folio(page) != folio)) 3778 goto next; 3779 3780 if (zone != folio_zone(folio)) 3781 goto next; 3782 3783 if (!folio_test_large(folio) 3784 || folio_test_hugetlb(folio) 3785 || !folio_test_lru(folio)) 3786 goto next; 3787 3788 total++; 3789 folio_lock(folio); 3790 nr_pages = folio_nr_pages(folio); 3791 if (!split_folio(folio)) 3792 split++; 3793 pfn += nr_pages - 1; 3794 folio_unlock(folio); 3795 next: 3796 folio_put(folio); 3797 cond_resched(); 3798 } 3799 } 3800 3801 pr_debug("%lu of %lu THP split\n", split, total); 3802 } 3803 3804 static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma) 3805 { 3806 return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) || 3807 is_vm_hugetlb_page(vma); 3808 } 3809 3810 static int split_huge_pages_pid(int pid, unsigned long vaddr_start, 3811 unsigned long vaddr_end, unsigned int new_order) 3812 { 3813 int ret = 0; 3814 struct task_struct *task; 3815 struct mm_struct *mm; 3816 unsigned long total = 0, split = 0; 3817 unsigned long addr; 3818 3819 vaddr_start &= PAGE_MASK; 3820 vaddr_end &= PAGE_MASK; 3821 3822 /* Find the task_struct from pid */ 3823 rcu_read_lock(); 3824 task = find_task_by_vpid(pid); 3825 if (!task) { 3826 rcu_read_unlock(); 3827 ret = -ESRCH; 3828 goto out; 3829 } 3830 get_task_struct(task); 3831 rcu_read_unlock(); 3832 3833 /* Find the mm_struct */ 3834 mm = get_task_mm(task); 3835 put_task_struct(task); 3836 3837 if (!mm) { 3838 ret = -EINVAL; 3839 goto out; 3840 } 3841 3842 pr_debug("Split huge pages in pid: %d, vaddr: [0x%lx - 0x%lx]\n", 3843 pid, vaddr_start, vaddr_end); 3844 3845 mmap_read_lock(mm); 3846 /* 3847 * always increase addr by PAGE_SIZE, since we could have a PTE page 3848 * table filled with PTE-mapped THPs, each of which is distinct. 3849 */ 3850 for (addr = vaddr_start; addr < vaddr_end; addr += PAGE_SIZE) { 3851 struct vm_area_struct *vma = vma_lookup(mm, addr); 3852 struct folio_walk fw; 3853 struct folio *folio; 3854 3855 if (!vma) 3856 break; 3857 3858 /* skip special VMA and hugetlb VMA */ 3859 if (vma_not_suitable_for_thp_split(vma)) { 3860 addr = vma->vm_end; 3861 continue; 3862 } 3863 3864 folio = folio_walk_start(&fw, vma, addr, 0); 3865 if (!folio) 3866 continue; 3867 3868 if (!is_transparent_hugepage(folio)) 3869 goto next; 3870 3871 if (new_order >= folio_order(folio)) 3872 goto next; 3873 3874 total++; 3875 /* 3876 * For folios with private, split_huge_page_to_list_to_order() 3877 * will try to drop it before split and then check if the folio 3878 * can be split or not. So skip the check here. 3879 */ 3880 if (!folio_test_private(folio) && 3881 !can_split_folio(folio, 0, NULL)) 3882 goto next; 3883 3884 if (!folio_trylock(folio)) 3885 goto next; 3886 folio_get(folio); 3887 folio_walk_end(&fw, vma); 3888 3889 if (!split_folio_to_order(folio, new_order)) 3890 split++; 3891 3892 folio_unlock(folio); 3893 folio_put(folio); 3894 3895 cond_resched(); 3896 continue; 3897 next: 3898 folio_walk_end(&fw, vma); 3899 cond_resched(); 3900 } 3901 mmap_read_unlock(mm); 3902 mmput(mm); 3903 3904 pr_debug("%lu of %lu THP split\n", split, total); 3905 3906 out: 3907 return ret; 3908 } 3909 3910 static int split_huge_pages_in_file(const char *file_path, pgoff_t off_start, 3911 pgoff_t off_end, unsigned int new_order) 3912 { 3913 struct filename *file; 3914 struct file *candidate; 3915 struct address_space *mapping; 3916 int ret = -EINVAL; 3917 pgoff_t index; 3918 int nr_pages = 1; 3919 unsigned long total = 0, split = 0; 3920 3921 file = getname_kernel(file_path); 3922 if (IS_ERR(file)) 3923 return ret; 3924 3925 candidate = file_open_name(file, O_RDONLY, 0); 3926 if (IS_ERR(candidate)) 3927 goto out; 3928 3929 pr_debug("split file-backed THPs in file: %s, page offset: [0x%lx - 0x%lx]\n", 3930 file_path, off_start, off_end); 3931 3932 mapping = candidate->f_mapping; 3933 3934 for (index = off_start; index < off_end; index += nr_pages) { 3935 struct folio *folio = filemap_get_folio(mapping, index); 3936 3937 nr_pages = 1; 3938 if (IS_ERR(folio)) 3939 continue; 3940 3941 if (!folio_test_large(folio)) 3942 goto next; 3943 3944 total++; 3945 nr_pages = folio_nr_pages(folio); 3946 3947 if (new_order >= folio_order(folio)) 3948 goto next; 3949 3950 if (!folio_trylock(folio)) 3951 goto next; 3952 3953 if (!split_folio_to_order(folio, new_order)) 3954 split++; 3955 3956 folio_unlock(folio); 3957 next: 3958 folio_put(folio); 3959 cond_resched(); 3960 } 3961 3962 filp_close(candidate, NULL); 3963 ret = 0; 3964 3965 pr_debug("%lu of %lu file-backed THP split\n", split, total); 3966 out: 3967 putname(file); 3968 return ret; 3969 } 3970 3971 #define MAX_INPUT_BUF_SZ 255 3972 3973 static ssize_t split_huge_pages_write(struct file *file, const char __user *buf, 3974 size_t count, loff_t *ppops) 3975 { 3976 static DEFINE_MUTEX(split_debug_mutex); 3977 ssize_t ret; 3978 /* 3979 * hold pid, start_vaddr, end_vaddr, new_order or 3980 * file_path, off_start, off_end, new_order 3981 */ 3982 char input_buf[MAX_INPUT_BUF_SZ]; 3983 int pid; 3984 unsigned long vaddr_start, vaddr_end; 3985 unsigned int new_order = 0; 3986 3987 ret = mutex_lock_interruptible(&split_debug_mutex); 3988 if (ret) 3989 return ret; 3990 3991 ret = -EFAULT; 3992 3993 memset(input_buf, 0, MAX_INPUT_BUF_SZ); 3994 if (copy_from_user(input_buf, buf, min_t(size_t, count, MAX_INPUT_BUF_SZ))) 3995 goto out; 3996 3997 input_buf[MAX_INPUT_BUF_SZ - 1] = '\0'; 3998 3999 if (input_buf[0] == '/') { 4000 char *tok; 4001 char *buf = input_buf; 4002 char file_path[MAX_INPUT_BUF_SZ]; 4003 pgoff_t off_start = 0, off_end = 0; 4004 size_t input_len = strlen(input_buf); 4005 4006 tok = strsep(&buf, ","); 4007 if (tok) { 4008 strcpy(file_path, tok); 4009 } else { 4010 ret = -EINVAL; 4011 goto out; 4012 } 4013 4014 ret = sscanf(buf, "0x%lx,0x%lx,%d", &off_start, &off_end, &new_order); 4015 if (ret != 2 && ret != 3) { 4016 ret = -EINVAL; 4017 goto out; 4018 } 4019 ret = split_huge_pages_in_file(file_path, off_start, off_end, new_order); 4020 if (!ret) 4021 ret = input_len; 4022 4023 goto out; 4024 } 4025 4026 ret = sscanf(input_buf, "%d,0x%lx,0x%lx,%d", &pid, &vaddr_start, &vaddr_end, &new_order); 4027 if (ret == 1 && pid == 1) { 4028 split_huge_pages_all(); 4029 ret = strlen(input_buf); 4030 goto out; 4031 } else if (ret != 3 && ret != 4) { 4032 ret = -EINVAL; 4033 goto out; 4034 } 4035 4036 ret = split_huge_pages_pid(pid, vaddr_start, vaddr_end, new_order); 4037 if (!ret) 4038 ret = strlen(input_buf); 4039 out: 4040 mutex_unlock(&split_debug_mutex); 4041 return ret; 4042 4043 } 4044 4045 static const struct file_operations split_huge_pages_fops = { 4046 .owner = THIS_MODULE, 4047 .write = split_huge_pages_write, 4048 .llseek = no_llseek, 4049 }; 4050 4051 static int __init split_huge_pages_debugfs(void) 4052 { 4053 debugfs_create_file("split_huge_pages", 0200, NULL, NULL, 4054 &split_huge_pages_fops); 4055 return 0; 4056 } 4057 late_initcall(split_huge_pages_debugfs); 4058 #endif 4059 4060 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 4061 int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 4062 struct page *page) 4063 { 4064 struct folio *folio = page_folio(page); 4065 struct vm_area_struct *vma = pvmw->vma; 4066 struct mm_struct *mm = vma->vm_mm; 4067 unsigned long address = pvmw->address; 4068 bool anon_exclusive; 4069 pmd_t pmdval; 4070 swp_entry_t entry; 4071 pmd_t pmdswp; 4072 4073 if (!(pvmw->pmd && !pvmw->pte)) 4074 return 0; 4075 4076 flush_cache_range(vma, address, address + HPAGE_PMD_SIZE); 4077 pmdval = pmdp_invalidate(vma, address, pvmw->pmd); 4078 4079 /* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. */ 4080 anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(page); 4081 if (anon_exclusive && folio_try_share_anon_rmap_pmd(folio, page)) { 4082 set_pmd_at(mm, address, pvmw->pmd, pmdval); 4083 return -EBUSY; 4084 } 4085 4086 if (pmd_dirty(pmdval)) 4087 folio_mark_dirty(folio); 4088 if (pmd_write(pmdval)) 4089 entry = make_writable_migration_entry(page_to_pfn(page)); 4090 else if (anon_exclusive) 4091 entry = make_readable_exclusive_migration_entry(page_to_pfn(page)); 4092 else 4093 entry = make_readable_migration_entry(page_to_pfn(page)); 4094 if (pmd_young(pmdval)) 4095 entry = make_migration_entry_young(entry); 4096 if (pmd_dirty(pmdval)) 4097 entry = make_migration_entry_dirty(entry); 4098 pmdswp = swp_entry_to_pmd(entry); 4099 if (pmd_soft_dirty(pmdval)) 4100 pmdswp = pmd_swp_mksoft_dirty(pmdswp); 4101 if (pmd_uffd_wp(pmdval)) 4102 pmdswp = pmd_swp_mkuffd_wp(pmdswp); 4103 set_pmd_at(mm, address, pvmw->pmd, pmdswp); 4104 folio_remove_rmap_pmd(folio, page, vma); 4105 folio_put(folio); 4106 trace_set_migration_pmd(address, pmd_val(pmdswp)); 4107 4108 return 0; 4109 } 4110 4111 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) 4112 { 4113 struct folio *folio = page_folio(new); 4114 struct vm_area_struct *vma = pvmw->vma; 4115 struct mm_struct *mm = vma->vm_mm; 4116 unsigned long address = pvmw->address; 4117 unsigned long haddr = address & HPAGE_PMD_MASK; 4118 pmd_t pmde; 4119 swp_entry_t entry; 4120 4121 if (!(pvmw->pmd && !pvmw->pte)) 4122 return; 4123 4124 entry = pmd_to_swp_entry(*pvmw->pmd); 4125 folio_get(folio); 4126 pmde = mk_huge_pmd(new, READ_ONCE(vma->vm_page_prot)); 4127 if (pmd_swp_soft_dirty(*pvmw->pmd)) 4128 pmde = pmd_mksoft_dirty(pmde); 4129 if (is_writable_migration_entry(entry)) 4130 pmde = pmd_mkwrite(pmde, vma); 4131 if (pmd_swp_uffd_wp(*pvmw->pmd)) 4132 pmde = pmd_mkuffd_wp(pmde); 4133 if (!is_migration_entry_young(entry)) 4134 pmde = pmd_mkold(pmde); 4135 /* NOTE: this may contain setting soft-dirty on some archs */ 4136 if (folio_test_dirty(folio) && is_migration_entry_dirty(entry)) 4137 pmde = pmd_mkdirty(pmde); 4138 4139 if (folio_test_anon(folio)) { 4140 rmap_t rmap_flags = RMAP_NONE; 4141 4142 if (!is_readable_migration_entry(entry)) 4143 rmap_flags |= RMAP_EXCLUSIVE; 4144 4145 folio_add_anon_rmap_pmd(folio, new, vma, haddr, rmap_flags); 4146 } else { 4147 folio_add_file_rmap_pmd(folio, new, vma); 4148 } 4149 VM_BUG_ON(pmd_write(pmde) && folio_test_anon(folio) && !PageAnonExclusive(new)); 4150 set_pmd_at(mm, haddr, pvmw->pmd, pmde); 4151 4152 /* No need to invalidate - it was non-present before */ 4153 update_mmu_cache_pmd(vma, address, pvmw->pmd); 4154 trace_remove_migration_pmd(address, pmd_val(pmde)); 4155 } 4156 #endif 4157