1 /* 2 * Copyright (C) 2009 Red Hat, Inc. 3 * 4 * This work is licensed under the terms of the GNU GPL, version 2. See 5 * the COPYING file in the top-level directory. 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 10 #include <linux/mm.h> 11 #include <linux/sched.h> 12 #include <linux/highmem.h> 13 #include <linux/hugetlb.h> 14 #include <linux/mmu_notifier.h> 15 #include <linux/rmap.h> 16 #include <linux/swap.h> 17 #include <linux/shrinker.h> 18 #include <linux/mm_inline.h> 19 #include <linux/swapops.h> 20 #include <linux/dax.h> 21 #include <linux/kthread.h> 22 #include <linux/khugepaged.h> 23 #include <linux/freezer.h> 24 #include <linux/pfn_t.h> 25 #include <linux/mman.h> 26 #include <linux/memremap.h> 27 #include <linux/pagemap.h> 28 #include <linux/debugfs.h> 29 #include <linux/migrate.h> 30 #include <linux/hashtable.h> 31 #include <linux/userfaultfd_k.h> 32 #include <linux/page_idle.h> 33 34 #include <asm/tlb.h> 35 #include <asm/pgalloc.h> 36 #include "internal.h" 37 38 enum scan_result { 39 SCAN_FAIL, 40 SCAN_SUCCEED, 41 SCAN_PMD_NULL, 42 SCAN_EXCEED_NONE_PTE, 43 SCAN_PTE_NON_PRESENT, 44 SCAN_PAGE_RO, 45 SCAN_NO_REFERENCED_PAGE, 46 SCAN_PAGE_NULL, 47 SCAN_SCAN_ABORT, 48 SCAN_PAGE_COUNT, 49 SCAN_PAGE_LRU, 50 SCAN_PAGE_LOCK, 51 SCAN_PAGE_ANON, 52 SCAN_PAGE_COMPOUND, 53 SCAN_ANY_PROCESS, 54 SCAN_VMA_NULL, 55 SCAN_VMA_CHECK, 56 SCAN_ADDRESS_RANGE, 57 SCAN_SWAP_CACHE_PAGE, 58 SCAN_DEL_PAGE_LRU, 59 SCAN_ALLOC_HUGE_PAGE_FAIL, 60 SCAN_CGROUP_CHARGE_FAIL 61 }; 62 63 #define CREATE_TRACE_POINTS 64 #include <trace/events/huge_memory.h> 65 66 /* 67 * By default transparent hugepage support is disabled in order that avoid 68 * to risk increase the memory footprint of applications without a guaranteed 69 * benefit. When transparent hugepage support is enabled, is for all mappings, 70 * and khugepaged scans all mappings. 71 * Defrag is invoked by khugepaged hugepage allocations and by page faults 72 * for all hugepage allocations. 73 */ 74 unsigned long transparent_hugepage_flags __read_mostly = 75 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS 76 (1<<TRANSPARENT_HUGEPAGE_FLAG)| 77 #endif 78 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE 79 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| 80 #endif 81 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)| 82 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| 83 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 84 85 /* default scan 8*512 pte (or vmas) every 30 second */ 86 static unsigned int khugepaged_pages_to_scan __read_mostly = HPAGE_PMD_NR*8; 87 static unsigned int khugepaged_pages_collapsed; 88 static unsigned int khugepaged_full_scans; 89 static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000; 90 /* during fragmentation poll the hugepage allocator once every minute */ 91 static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000; 92 static struct task_struct *khugepaged_thread __read_mostly; 93 static DEFINE_MUTEX(khugepaged_mutex); 94 static DEFINE_SPINLOCK(khugepaged_mm_lock); 95 static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait); 96 /* 97 * default collapse hugepages if there is at least one pte mapped like 98 * it would have happened if the vma was large enough during page 99 * fault. 100 */ 101 static unsigned int khugepaged_max_ptes_none __read_mostly = HPAGE_PMD_NR-1; 102 103 static int khugepaged(void *none); 104 static int khugepaged_slab_init(void); 105 static void khugepaged_slab_exit(void); 106 107 #define MM_SLOTS_HASH_BITS 10 108 static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS); 109 110 static struct kmem_cache *mm_slot_cache __read_mostly; 111 112 /** 113 * struct mm_slot - hash lookup from mm to mm_slot 114 * @hash: hash collision list 115 * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head 116 * @mm: the mm that this information is valid for 117 */ 118 struct mm_slot { 119 struct hlist_node hash; 120 struct list_head mm_node; 121 struct mm_struct *mm; 122 }; 123 124 /** 125 * struct khugepaged_scan - cursor for scanning 126 * @mm_head: the head of the mm list to scan 127 * @mm_slot: the current mm_slot we are scanning 128 * @address: the next address inside that to be scanned 129 * 130 * There is only the one khugepaged_scan instance of this cursor structure. 131 */ 132 struct khugepaged_scan { 133 struct list_head mm_head; 134 struct mm_slot *mm_slot; 135 unsigned long address; 136 }; 137 static struct khugepaged_scan khugepaged_scan = { 138 .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head), 139 }; 140 141 static struct shrinker deferred_split_shrinker; 142 143 static void set_recommended_min_free_kbytes(void) 144 { 145 struct zone *zone; 146 int nr_zones = 0; 147 unsigned long recommended_min; 148 149 for_each_populated_zone(zone) 150 nr_zones++; 151 152 /* Ensure 2 pageblocks are free to assist fragmentation avoidance */ 153 recommended_min = pageblock_nr_pages * nr_zones * 2; 154 155 /* 156 * Make sure that on average at least two pageblocks are almost free 157 * of another type, one for a migratetype to fall back to and a 158 * second to avoid subsequent fallbacks of other types There are 3 159 * MIGRATE_TYPES we care about. 160 */ 161 recommended_min += pageblock_nr_pages * nr_zones * 162 MIGRATE_PCPTYPES * MIGRATE_PCPTYPES; 163 164 /* don't ever allow to reserve more than 5% of the lowmem */ 165 recommended_min = min(recommended_min, 166 (unsigned long) nr_free_buffer_pages() / 20); 167 recommended_min <<= (PAGE_SHIFT-10); 168 169 if (recommended_min > min_free_kbytes) { 170 if (user_min_free_kbytes >= 0) 171 pr_info("raising min_free_kbytes from %d to %lu " 172 "to help transparent hugepage allocations\n", 173 min_free_kbytes, recommended_min); 174 175 min_free_kbytes = recommended_min; 176 } 177 setup_per_zone_wmarks(); 178 } 179 180 static int start_stop_khugepaged(void) 181 { 182 int err = 0; 183 if (khugepaged_enabled()) { 184 if (!khugepaged_thread) 185 khugepaged_thread = kthread_run(khugepaged, NULL, 186 "khugepaged"); 187 if (IS_ERR(khugepaged_thread)) { 188 pr_err("khugepaged: kthread_run(khugepaged) failed\n"); 189 err = PTR_ERR(khugepaged_thread); 190 khugepaged_thread = NULL; 191 goto fail; 192 } 193 194 if (!list_empty(&khugepaged_scan.mm_head)) 195 wake_up_interruptible(&khugepaged_wait); 196 197 set_recommended_min_free_kbytes(); 198 } else if (khugepaged_thread) { 199 kthread_stop(khugepaged_thread); 200 khugepaged_thread = NULL; 201 } 202 fail: 203 return err; 204 } 205 206 static atomic_t huge_zero_refcount; 207 struct page *huge_zero_page __read_mostly; 208 209 struct page *get_huge_zero_page(void) 210 { 211 struct page *zero_page; 212 retry: 213 if (likely(atomic_inc_not_zero(&huge_zero_refcount))) 214 return READ_ONCE(huge_zero_page); 215 216 zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE, 217 HPAGE_PMD_ORDER); 218 if (!zero_page) { 219 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); 220 return NULL; 221 } 222 count_vm_event(THP_ZERO_PAGE_ALLOC); 223 preempt_disable(); 224 if (cmpxchg(&huge_zero_page, NULL, zero_page)) { 225 preempt_enable(); 226 __free_pages(zero_page, compound_order(zero_page)); 227 goto retry; 228 } 229 230 /* We take additional reference here. It will be put back by shrinker */ 231 atomic_set(&huge_zero_refcount, 2); 232 preempt_enable(); 233 return READ_ONCE(huge_zero_page); 234 } 235 236 static void put_huge_zero_page(void) 237 { 238 /* 239 * Counter should never go to zero here. Only shrinker can put 240 * last reference. 241 */ 242 BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); 243 } 244 245 static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink, 246 struct shrink_control *sc) 247 { 248 /* we can free zero page only if last reference remains */ 249 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; 250 } 251 252 static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink, 253 struct shrink_control *sc) 254 { 255 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { 256 struct page *zero_page = xchg(&huge_zero_page, NULL); 257 BUG_ON(zero_page == NULL); 258 __free_pages(zero_page, compound_order(zero_page)); 259 return HPAGE_PMD_NR; 260 } 261 262 return 0; 263 } 264 265 static struct shrinker huge_zero_page_shrinker = { 266 .count_objects = shrink_huge_zero_page_count, 267 .scan_objects = shrink_huge_zero_page_scan, 268 .seeks = DEFAULT_SEEKS, 269 }; 270 271 #ifdef CONFIG_SYSFS 272 273 static ssize_t double_flag_show(struct kobject *kobj, 274 struct kobj_attribute *attr, char *buf, 275 enum transparent_hugepage_flag enabled, 276 enum transparent_hugepage_flag req_madv) 277 { 278 if (test_bit(enabled, &transparent_hugepage_flags)) { 279 VM_BUG_ON(test_bit(req_madv, &transparent_hugepage_flags)); 280 return sprintf(buf, "[always] madvise never\n"); 281 } else if (test_bit(req_madv, &transparent_hugepage_flags)) 282 return sprintf(buf, "always [madvise] never\n"); 283 else 284 return sprintf(buf, "always madvise [never]\n"); 285 } 286 static ssize_t double_flag_store(struct kobject *kobj, 287 struct kobj_attribute *attr, 288 const char *buf, size_t count, 289 enum transparent_hugepage_flag enabled, 290 enum transparent_hugepage_flag req_madv) 291 { 292 if (!memcmp("always", buf, 293 min(sizeof("always")-1, count))) { 294 set_bit(enabled, &transparent_hugepage_flags); 295 clear_bit(req_madv, &transparent_hugepage_flags); 296 } else if (!memcmp("madvise", buf, 297 min(sizeof("madvise")-1, count))) { 298 clear_bit(enabled, &transparent_hugepage_flags); 299 set_bit(req_madv, &transparent_hugepage_flags); 300 } else if (!memcmp("never", buf, 301 min(sizeof("never")-1, count))) { 302 clear_bit(enabled, &transparent_hugepage_flags); 303 clear_bit(req_madv, &transparent_hugepage_flags); 304 } else 305 return -EINVAL; 306 307 return count; 308 } 309 310 static ssize_t enabled_show(struct kobject *kobj, 311 struct kobj_attribute *attr, char *buf) 312 { 313 return double_flag_show(kobj, attr, buf, 314 TRANSPARENT_HUGEPAGE_FLAG, 315 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG); 316 } 317 static ssize_t enabled_store(struct kobject *kobj, 318 struct kobj_attribute *attr, 319 const char *buf, size_t count) 320 { 321 ssize_t ret; 322 323 ret = double_flag_store(kobj, attr, buf, count, 324 TRANSPARENT_HUGEPAGE_FLAG, 325 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG); 326 327 if (ret > 0) { 328 int err; 329 330 mutex_lock(&khugepaged_mutex); 331 err = start_stop_khugepaged(); 332 mutex_unlock(&khugepaged_mutex); 333 334 if (err) 335 ret = err; 336 } 337 338 return ret; 339 } 340 static struct kobj_attribute enabled_attr = 341 __ATTR(enabled, 0644, enabled_show, enabled_store); 342 343 static ssize_t single_flag_show(struct kobject *kobj, 344 struct kobj_attribute *attr, char *buf, 345 enum transparent_hugepage_flag flag) 346 { 347 return sprintf(buf, "%d\n", 348 !!test_bit(flag, &transparent_hugepage_flags)); 349 } 350 351 static ssize_t single_flag_store(struct kobject *kobj, 352 struct kobj_attribute *attr, 353 const char *buf, size_t count, 354 enum transparent_hugepage_flag flag) 355 { 356 unsigned long value; 357 int ret; 358 359 ret = kstrtoul(buf, 10, &value); 360 if (ret < 0) 361 return ret; 362 if (value > 1) 363 return -EINVAL; 364 365 if (value) 366 set_bit(flag, &transparent_hugepage_flags); 367 else 368 clear_bit(flag, &transparent_hugepage_flags); 369 370 return count; 371 } 372 373 /* 374 * Currently defrag only disables __GFP_NOWAIT for allocation. A blind 375 * __GFP_REPEAT is too aggressive, it's never worth swapping tons of 376 * memory just to allocate one more hugepage. 377 */ 378 static ssize_t defrag_show(struct kobject *kobj, 379 struct kobj_attribute *attr, char *buf) 380 { 381 return double_flag_show(kobj, attr, buf, 382 TRANSPARENT_HUGEPAGE_DEFRAG_FLAG, 383 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG); 384 } 385 static ssize_t defrag_store(struct kobject *kobj, 386 struct kobj_attribute *attr, 387 const char *buf, size_t count) 388 { 389 return double_flag_store(kobj, attr, buf, count, 390 TRANSPARENT_HUGEPAGE_DEFRAG_FLAG, 391 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG); 392 } 393 static struct kobj_attribute defrag_attr = 394 __ATTR(defrag, 0644, defrag_show, defrag_store); 395 396 static ssize_t use_zero_page_show(struct kobject *kobj, 397 struct kobj_attribute *attr, char *buf) 398 { 399 return single_flag_show(kobj, attr, buf, 400 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 401 } 402 static ssize_t use_zero_page_store(struct kobject *kobj, 403 struct kobj_attribute *attr, const char *buf, size_t count) 404 { 405 return single_flag_store(kobj, attr, buf, count, 406 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 407 } 408 static struct kobj_attribute use_zero_page_attr = 409 __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store); 410 #ifdef CONFIG_DEBUG_VM 411 static ssize_t debug_cow_show(struct kobject *kobj, 412 struct kobj_attribute *attr, char *buf) 413 { 414 return single_flag_show(kobj, attr, buf, 415 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG); 416 } 417 static ssize_t debug_cow_store(struct kobject *kobj, 418 struct kobj_attribute *attr, 419 const char *buf, size_t count) 420 { 421 return single_flag_store(kobj, attr, buf, count, 422 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG); 423 } 424 static struct kobj_attribute debug_cow_attr = 425 __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store); 426 #endif /* CONFIG_DEBUG_VM */ 427 428 static struct attribute *hugepage_attr[] = { 429 &enabled_attr.attr, 430 &defrag_attr.attr, 431 &use_zero_page_attr.attr, 432 #ifdef CONFIG_DEBUG_VM 433 &debug_cow_attr.attr, 434 #endif 435 NULL, 436 }; 437 438 static struct attribute_group hugepage_attr_group = { 439 .attrs = hugepage_attr, 440 }; 441 442 static ssize_t scan_sleep_millisecs_show(struct kobject *kobj, 443 struct kobj_attribute *attr, 444 char *buf) 445 { 446 return sprintf(buf, "%u\n", khugepaged_scan_sleep_millisecs); 447 } 448 449 static ssize_t scan_sleep_millisecs_store(struct kobject *kobj, 450 struct kobj_attribute *attr, 451 const char *buf, size_t count) 452 { 453 unsigned long msecs; 454 int err; 455 456 err = kstrtoul(buf, 10, &msecs); 457 if (err || msecs > UINT_MAX) 458 return -EINVAL; 459 460 khugepaged_scan_sleep_millisecs = msecs; 461 wake_up_interruptible(&khugepaged_wait); 462 463 return count; 464 } 465 static struct kobj_attribute scan_sleep_millisecs_attr = 466 __ATTR(scan_sleep_millisecs, 0644, scan_sleep_millisecs_show, 467 scan_sleep_millisecs_store); 468 469 static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj, 470 struct kobj_attribute *attr, 471 char *buf) 472 { 473 return sprintf(buf, "%u\n", khugepaged_alloc_sleep_millisecs); 474 } 475 476 static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj, 477 struct kobj_attribute *attr, 478 const char *buf, size_t count) 479 { 480 unsigned long msecs; 481 int err; 482 483 err = kstrtoul(buf, 10, &msecs); 484 if (err || msecs > UINT_MAX) 485 return -EINVAL; 486 487 khugepaged_alloc_sleep_millisecs = msecs; 488 wake_up_interruptible(&khugepaged_wait); 489 490 return count; 491 } 492 static struct kobj_attribute alloc_sleep_millisecs_attr = 493 __ATTR(alloc_sleep_millisecs, 0644, alloc_sleep_millisecs_show, 494 alloc_sleep_millisecs_store); 495 496 static ssize_t pages_to_scan_show(struct kobject *kobj, 497 struct kobj_attribute *attr, 498 char *buf) 499 { 500 return sprintf(buf, "%u\n", khugepaged_pages_to_scan); 501 } 502 static ssize_t pages_to_scan_store(struct kobject *kobj, 503 struct kobj_attribute *attr, 504 const char *buf, size_t count) 505 { 506 int err; 507 unsigned long pages; 508 509 err = kstrtoul(buf, 10, &pages); 510 if (err || !pages || pages > UINT_MAX) 511 return -EINVAL; 512 513 khugepaged_pages_to_scan = pages; 514 515 return count; 516 } 517 static struct kobj_attribute pages_to_scan_attr = 518 __ATTR(pages_to_scan, 0644, pages_to_scan_show, 519 pages_to_scan_store); 520 521 static ssize_t pages_collapsed_show(struct kobject *kobj, 522 struct kobj_attribute *attr, 523 char *buf) 524 { 525 return sprintf(buf, "%u\n", khugepaged_pages_collapsed); 526 } 527 static struct kobj_attribute pages_collapsed_attr = 528 __ATTR_RO(pages_collapsed); 529 530 static ssize_t full_scans_show(struct kobject *kobj, 531 struct kobj_attribute *attr, 532 char *buf) 533 { 534 return sprintf(buf, "%u\n", khugepaged_full_scans); 535 } 536 static struct kobj_attribute full_scans_attr = 537 __ATTR_RO(full_scans); 538 539 static ssize_t khugepaged_defrag_show(struct kobject *kobj, 540 struct kobj_attribute *attr, char *buf) 541 { 542 return single_flag_show(kobj, attr, buf, 543 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 544 } 545 static ssize_t khugepaged_defrag_store(struct kobject *kobj, 546 struct kobj_attribute *attr, 547 const char *buf, size_t count) 548 { 549 return single_flag_store(kobj, attr, buf, count, 550 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); 551 } 552 static struct kobj_attribute khugepaged_defrag_attr = 553 __ATTR(defrag, 0644, khugepaged_defrag_show, 554 khugepaged_defrag_store); 555 556 /* 557 * max_ptes_none controls if khugepaged should collapse hugepages over 558 * any unmapped ptes in turn potentially increasing the memory 559 * footprint of the vmas. When max_ptes_none is 0 khugepaged will not 560 * reduce the available free memory in the system as it 561 * runs. Increasing max_ptes_none will instead potentially reduce the 562 * free memory in the system during the khugepaged scan. 563 */ 564 static ssize_t khugepaged_max_ptes_none_show(struct kobject *kobj, 565 struct kobj_attribute *attr, 566 char *buf) 567 { 568 return sprintf(buf, "%u\n", khugepaged_max_ptes_none); 569 } 570 static ssize_t khugepaged_max_ptes_none_store(struct kobject *kobj, 571 struct kobj_attribute *attr, 572 const char *buf, size_t count) 573 { 574 int err; 575 unsigned long max_ptes_none; 576 577 err = kstrtoul(buf, 10, &max_ptes_none); 578 if (err || max_ptes_none > HPAGE_PMD_NR-1) 579 return -EINVAL; 580 581 khugepaged_max_ptes_none = max_ptes_none; 582 583 return count; 584 } 585 static struct kobj_attribute khugepaged_max_ptes_none_attr = 586 __ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show, 587 khugepaged_max_ptes_none_store); 588 589 static struct attribute *khugepaged_attr[] = { 590 &khugepaged_defrag_attr.attr, 591 &khugepaged_max_ptes_none_attr.attr, 592 &pages_to_scan_attr.attr, 593 &pages_collapsed_attr.attr, 594 &full_scans_attr.attr, 595 &scan_sleep_millisecs_attr.attr, 596 &alloc_sleep_millisecs_attr.attr, 597 NULL, 598 }; 599 600 static struct attribute_group khugepaged_attr_group = { 601 .attrs = khugepaged_attr, 602 .name = "khugepaged", 603 }; 604 605 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj) 606 { 607 int err; 608 609 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj); 610 if (unlikely(!*hugepage_kobj)) { 611 pr_err("failed to create transparent hugepage kobject\n"); 612 return -ENOMEM; 613 } 614 615 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group); 616 if (err) { 617 pr_err("failed to register transparent hugepage group\n"); 618 goto delete_obj; 619 } 620 621 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group); 622 if (err) { 623 pr_err("failed to register transparent hugepage group\n"); 624 goto remove_hp_group; 625 } 626 627 return 0; 628 629 remove_hp_group: 630 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group); 631 delete_obj: 632 kobject_put(*hugepage_kobj); 633 return err; 634 } 635 636 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj) 637 { 638 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group); 639 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group); 640 kobject_put(hugepage_kobj); 641 } 642 #else 643 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj) 644 { 645 return 0; 646 } 647 648 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) 649 { 650 } 651 #endif /* CONFIG_SYSFS */ 652 653 static int __init hugepage_init(void) 654 { 655 int err; 656 struct kobject *hugepage_kobj; 657 658 if (!has_transparent_hugepage()) { 659 transparent_hugepage_flags = 0; 660 return -EINVAL; 661 } 662 663 err = hugepage_init_sysfs(&hugepage_kobj); 664 if (err) 665 goto err_sysfs; 666 667 err = khugepaged_slab_init(); 668 if (err) 669 goto err_slab; 670 671 err = register_shrinker(&huge_zero_page_shrinker); 672 if (err) 673 goto err_hzp_shrinker; 674 err = register_shrinker(&deferred_split_shrinker); 675 if (err) 676 goto err_split_shrinker; 677 678 /* 679 * By default disable transparent hugepages on smaller systems, 680 * where the extra memory used could hurt more than TLB overhead 681 * is likely to save. The admin can still enable it through /sys. 682 */ 683 if (totalram_pages < (512 << (20 - PAGE_SHIFT))) { 684 transparent_hugepage_flags = 0; 685 return 0; 686 } 687 688 err = start_stop_khugepaged(); 689 if (err) 690 goto err_khugepaged; 691 692 return 0; 693 err_khugepaged: 694 unregister_shrinker(&deferred_split_shrinker); 695 err_split_shrinker: 696 unregister_shrinker(&huge_zero_page_shrinker); 697 err_hzp_shrinker: 698 khugepaged_slab_exit(); 699 err_slab: 700 hugepage_exit_sysfs(hugepage_kobj); 701 err_sysfs: 702 return err; 703 } 704 subsys_initcall(hugepage_init); 705 706 static int __init setup_transparent_hugepage(char *str) 707 { 708 int ret = 0; 709 if (!str) 710 goto out; 711 if (!strcmp(str, "always")) { 712 set_bit(TRANSPARENT_HUGEPAGE_FLAG, 713 &transparent_hugepage_flags); 714 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 715 &transparent_hugepage_flags); 716 ret = 1; 717 } else if (!strcmp(str, "madvise")) { 718 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 719 &transparent_hugepage_flags); 720 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 721 &transparent_hugepage_flags); 722 ret = 1; 723 } else if (!strcmp(str, "never")) { 724 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 725 &transparent_hugepage_flags); 726 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 727 &transparent_hugepage_flags); 728 ret = 1; 729 } 730 out: 731 if (!ret) 732 pr_warn("transparent_hugepage= cannot parse, ignored\n"); 733 return ret; 734 } 735 __setup("transparent_hugepage=", setup_transparent_hugepage); 736 737 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 738 { 739 if (likely(vma->vm_flags & VM_WRITE)) 740 pmd = pmd_mkwrite(pmd); 741 return pmd; 742 } 743 744 static inline pmd_t mk_huge_pmd(struct page *page, pgprot_t prot) 745 { 746 pmd_t entry; 747 entry = mk_pmd(page, prot); 748 entry = pmd_mkhuge(entry); 749 return entry; 750 } 751 752 static inline struct list_head *page_deferred_list(struct page *page) 753 { 754 /* 755 * ->lru in the tail pages is occupied by compound_head. 756 * Let's use ->mapping + ->index in the second tail page as list_head. 757 */ 758 return (struct list_head *)&page[2].mapping; 759 } 760 761 void prep_transhuge_page(struct page *page) 762 { 763 /* 764 * we use page->mapping and page->indexlru in second tail page 765 * as list_head: assuming THP order >= 2 766 */ 767 BUILD_BUG_ON(HPAGE_PMD_ORDER < 2); 768 769 INIT_LIST_HEAD(page_deferred_list(page)); 770 set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR); 771 } 772 773 static int __do_huge_pmd_anonymous_page(struct mm_struct *mm, 774 struct vm_area_struct *vma, 775 unsigned long address, pmd_t *pmd, 776 struct page *page, gfp_t gfp, 777 unsigned int flags) 778 { 779 struct mem_cgroup *memcg; 780 pgtable_t pgtable; 781 spinlock_t *ptl; 782 unsigned long haddr = address & HPAGE_PMD_MASK; 783 784 VM_BUG_ON_PAGE(!PageCompound(page), page); 785 786 if (mem_cgroup_try_charge(page, mm, gfp, &memcg, true)) { 787 put_page(page); 788 count_vm_event(THP_FAULT_FALLBACK); 789 return VM_FAULT_FALLBACK; 790 } 791 792 pgtable = pte_alloc_one(mm, haddr); 793 if (unlikely(!pgtable)) { 794 mem_cgroup_cancel_charge(page, memcg, true); 795 put_page(page); 796 return VM_FAULT_OOM; 797 } 798 799 clear_huge_page(page, haddr, HPAGE_PMD_NR); 800 /* 801 * The memory barrier inside __SetPageUptodate makes sure that 802 * clear_huge_page writes become visible before the set_pmd_at() 803 * write. 804 */ 805 __SetPageUptodate(page); 806 807 ptl = pmd_lock(mm, pmd); 808 if (unlikely(!pmd_none(*pmd))) { 809 spin_unlock(ptl); 810 mem_cgroup_cancel_charge(page, memcg, true); 811 put_page(page); 812 pte_free(mm, pgtable); 813 } else { 814 pmd_t entry; 815 816 /* Deliver the page fault to userland */ 817 if (userfaultfd_missing(vma)) { 818 int ret; 819 820 spin_unlock(ptl); 821 mem_cgroup_cancel_charge(page, memcg, true); 822 put_page(page); 823 pte_free(mm, pgtable); 824 ret = handle_userfault(vma, address, flags, 825 VM_UFFD_MISSING); 826 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 827 return ret; 828 } 829 830 entry = mk_huge_pmd(page, vma->vm_page_prot); 831 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 832 page_add_new_anon_rmap(page, vma, haddr, true); 833 mem_cgroup_commit_charge(page, memcg, false, true); 834 lru_cache_add_active_or_unevictable(page, vma); 835 pgtable_trans_huge_deposit(mm, pmd, pgtable); 836 set_pmd_at(mm, haddr, pmd, entry); 837 add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR); 838 atomic_long_inc(&mm->nr_ptes); 839 spin_unlock(ptl); 840 count_vm_event(THP_FAULT_ALLOC); 841 } 842 843 return 0; 844 } 845 846 static inline gfp_t alloc_hugepage_gfpmask(int defrag, gfp_t extra_gfp) 847 { 848 return (GFP_TRANSHUGE & ~(defrag ? 0 : __GFP_RECLAIM)) | extra_gfp; 849 } 850 851 /* Caller must hold page table lock. */ 852 static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm, 853 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, 854 struct page *zero_page) 855 { 856 pmd_t entry; 857 if (!pmd_none(*pmd)) 858 return false; 859 entry = mk_pmd(zero_page, vma->vm_page_prot); 860 entry = pmd_mkhuge(entry); 861 if (pgtable) 862 pgtable_trans_huge_deposit(mm, pmd, pgtable); 863 set_pmd_at(mm, haddr, pmd, entry); 864 atomic_long_inc(&mm->nr_ptes); 865 return true; 866 } 867 868 int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, 869 unsigned long address, pmd_t *pmd, 870 unsigned int flags) 871 { 872 gfp_t gfp; 873 struct page *page; 874 unsigned long haddr = address & HPAGE_PMD_MASK; 875 876 if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end) 877 return VM_FAULT_FALLBACK; 878 if (unlikely(anon_vma_prepare(vma))) 879 return VM_FAULT_OOM; 880 if (unlikely(khugepaged_enter(vma, vma->vm_flags))) 881 return VM_FAULT_OOM; 882 if (!(flags & FAULT_FLAG_WRITE) && !mm_forbids_zeropage(mm) && 883 transparent_hugepage_use_zero_page()) { 884 spinlock_t *ptl; 885 pgtable_t pgtable; 886 struct page *zero_page; 887 bool set; 888 int ret; 889 pgtable = pte_alloc_one(mm, haddr); 890 if (unlikely(!pgtable)) 891 return VM_FAULT_OOM; 892 zero_page = get_huge_zero_page(); 893 if (unlikely(!zero_page)) { 894 pte_free(mm, pgtable); 895 count_vm_event(THP_FAULT_FALLBACK); 896 return VM_FAULT_FALLBACK; 897 } 898 ptl = pmd_lock(mm, pmd); 899 ret = 0; 900 set = false; 901 if (pmd_none(*pmd)) { 902 if (userfaultfd_missing(vma)) { 903 spin_unlock(ptl); 904 ret = handle_userfault(vma, address, flags, 905 VM_UFFD_MISSING); 906 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 907 } else { 908 set_huge_zero_page(pgtable, mm, vma, 909 haddr, pmd, 910 zero_page); 911 spin_unlock(ptl); 912 set = true; 913 } 914 } else 915 spin_unlock(ptl); 916 if (!set) { 917 pte_free(mm, pgtable); 918 put_huge_zero_page(); 919 } 920 return ret; 921 } 922 gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0); 923 page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER); 924 if (unlikely(!page)) { 925 count_vm_event(THP_FAULT_FALLBACK); 926 return VM_FAULT_FALLBACK; 927 } 928 prep_transhuge_page(page); 929 return __do_huge_pmd_anonymous_page(mm, vma, address, pmd, page, gfp, 930 flags); 931 } 932 933 static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 934 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write) 935 { 936 struct mm_struct *mm = vma->vm_mm; 937 pmd_t entry; 938 spinlock_t *ptl; 939 940 ptl = pmd_lock(mm, pmd); 941 entry = pmd_mkhuge(pfn_t_pmd(pfn, prot)); 942 if (pfn_t_devmap(pfn)) 943 entry = pmd_mkdevmap(entry); 944 if (write) { 945 entry = pmd_mkyoung(pmd_mkdirty(entry)); 946 entry = maybe_pmd_mkwrite(entry, vma); 947 } 948 set_pmd_at(mm, addr, pmd, entry); 949 update_mmu_cache_pmd(vma, addr, pmd); 950 spin_unlock(ptl); 951 } 952 953 int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 954 pmd_t *pmd, pfn_t pfn, bool write) 955 { 956 pgprot_t pgprot = vma->vm_page_prot; 957 /* 958 * If we had pmd_special, we could avoid all these restrictions, 959 * but we need to be consistent with PTEs and architectures that 960 * can't support a 'special' bit. 961 */ 962 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); 963 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 964 (VM_PFNMAP|VM_MIXEDMAP)); 965 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 966 BUG_ON(!pfn_t_devmap(pfn)); 967 968 if (addr < vma->vm_start || addr >= vma->vm_end) 969 return VM_FAULT_SIGBUS; 970 if (track_pfn_insert(vma, &pgprot, pfn)) 971 return VM_FAULT_SIGBUS; 972 insert_pfn_pmd(vma, addr, pmd, pfn, pgprot, write); 973 return VM_FAULT_NOPAGE; 974 } 975 976 static void touch_pmd(struct vm_area_struct *vma, unsigned long addr, 977 pmd_t *pmd) 978 { 979 pmd_t _pmd; 980 981 /* 982 * We should set the dirty bit only for FOLL_WRITE but for now 983 * the dirty bit in the pmd is meaningless. And if the dirty 984 * bit will become meaningful and we'll only set it with 985 * FOLL_WRITE, an atomic set_bit will be required on the pmd to 986 * set the young bit, instead of the current set_pmd_at. 987 */ 988 _pmd = pmd_mkyoung(pmd_mkdirty(*pmd)); 989 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, 990 pmd, _pmd, 1)) 991 update_mmu_cache_pmd(vma, addr, pmd); 992 } 993 994 struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, 995 pmd_t *pmd, int flags) 996 { 997 unsigned long pfn = pmd_pfn(*pmd); 998 struct mm_struct *mm = vma->vm_mm; 999 struct dev_pagemap *pgmap; 1000 struct page *page; 1001 1002 assert_spin_locked(pmd_lockptr(mm, pmd)); 1003 1004 if (flags & FOLL_WRITE && !pmd_write(*pmd)) 1005 return NULL; 1006 1007 if (pmd_present(*pmd) && pmd_devmap(*pmd)) 1008 /* pass */; 1009 else 1010 return NULL; 1011 1012 if (flags & FOLL_TOUCH) 1013 touch_pmd(vma, addr, pmd); 1014 1015 /* 1016 * device mapped pages can only be returned if the 1017 * caller will manage the page reference count. 1018 */ 1019 if (!(flags & FOLL_GET)) 1020 return ERR_PTR(-EEXIST); 1021 1022 pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT; 1023 pgmap = get_dev_pagemap(pfn, NULL); 1024 if (!pgmap) 1025 return ERR_PTR(-EFAULT); 1026 page = pfn_to_page(pfn); 1027 get_page(page); 1028 put_dev_pagemap(pgmap); 1029 1030 return page; 1031 } 1032 1033 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1034 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 1035 struct vm_area_struct *vma) 1036 { 1037 spinlock_t *dst_ptl, *src_ptl; 1038 struct page *src_page; 1039 pmd_t pmd; 1040 pgtable_t pgtable = NULL; 1041 int ret; 1042 1043 if (!vma_is_dax(vma)) { 1044 ret = -ENOMEM; 1045 pgtable = pte_alloc_one(dst_mm, addr); 1046 if (unlikely(!pgtable)) 1047 goto out; 1048 } 1049 1050 dst_ptl = pmd_lock(dst_mm, dst_pmd); 1051 src_ptl = pmd_lockptr(src_mm, src_pmd); 1052 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1053 1054 ret = -EAGAIN; 1055 pmd = *src_pmd; 1056 if (unlikely(!pmd_trans_huge(pmd) && !pmd_devmap(pmd))) { 1057 pte_free(dst_mm, pgtable); 1058 goto out_unlock; 1059 } 1060 /* 1061 * When page table lock is held, the huge zero pmd should not be 1062 * under splitting since we don't split the page itself, only pmd to 1063 * a page table. 1064 */ 1065 if (is_huge_zero_pmd(pmd)) { 1066 struct page *zero_page; 1067 /* 1068 * get_huge_zero_page() will never allocate a new page here, 1069 * since we already have a zero page to copy. It just takes a 1070 * reference. 1071 */ 1072 zero_page = get_huge_zero_page(); 1073 set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd, 1074 zero_page); 1075 ret = 0; 1076 goto out_unlock; 1077 } 1078 1079 if (!vma_is_dax(vma)) { 1080 /* thp accounting separate from pmd_devmap accounting */ 1081 src_page = pmd_page(pmd); 1082 VM_BUG_ON_PAGE(!PageHead(src_page), src_page); 1083 get_page(src_page); 1084 page_dup_rmap(src_page, true); 1085 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1086 atomic_long_inc(&dst_mm->nr_ptes); 1087 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 1088 } 1089 1090 pmdp_set_wrprotect(src_mm, addr, src_pmd); 1091 pmd = pmd_mkold(pmd_wrprotect(pmd)); 1092 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 1093 1094 ret = 0; 1095 out_unlock: 1096 spin_unlock(src_ptl); 1097 spin_unlock(dst_ptl); 1098 out: 1099 return ret; 1100 } 1101 1102 void huge_pmd_set_accessed(struct mm_struct *mm, 1103 struct vm_area_struct *vma, 1104 unsigned long address, 1105 pmd_t *pmd, pmd_t orig_pmd, 1106 int dirty) 1107 { 1108 spinlock_t *ptl; 1109 pmd_t entry; 1110 unsigned long haddr; 1111 1112 ptl = pmd_lock(mm, pmd); 1113 if (unlikely(!pmd_same(*pmd, orig_pmd))) 1114 goto unlock; 1115 1116 entry = pmd_mkyoung(orig_pmd); 1117 haddr = address & HPAGE_PMD_MASK; 1118 if (pmdp_set_access_flags(vma, haddr, pmd, entry, dirty)) 1119 update_mmu_cache_pmd(vma, address, pmd); 1120 1121 unlock: 1122 spin_unlock(ptl); 1123 } 1124 1125 static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm, 1126 struct vm_area_struct *vma, 1127 unsigned long address, 1128 pmd_t *pmd, pmd_t orig_pmd, 1129 struct page *page, 1130 unsigned long haddr) 1131 { 1132 struct mem_cgroup *memcg; 1133 spinlock_t *ptl; 1134 pgtable_t pgtable; 1135 pmd_t _pmd; 1136 int ret = 0, i; 1137 struct page **pages; 1138 unsigned long mmun_start; /* For mmu_notifiers */ 1139 unsigned long mmun_end; /* For mmu_notifiers */ 1140 1141 pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR, 1142 GFP_KERNEL); 1143 if (unlikely(!pages)) { 1144 ret |= VM_FAULT_OOM; 1145 goto out; 1146 } 1147 1148 for (i = 0; i < HPAGE_PMD_NR; i++) { 1149 pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE | 1150 __GFP_OTHER_NODE, 1151 vma, address, page_to_nid(page)); 1152 if (unlikely(!pages[i] || 1153 mem_cgroup_try_charge(pages[i], mm, GFP_KERNEL, 1154 &memcg, false))) { 1155 if (pages[i]) 1156 put_page(pages[i]); 1157 while (--i >= 0) { 1158 memcg = (void *)page_private(pages[i]); 1159 set_page_private(pages[i], 0); 1160 mem_cgroup_cancel_charge(pages[i], memcg, 1161 false); 1162 put_page(pages[i]); 1163 } 1164 kfree(pages); 1165 ret |= VM_FAULT_OOM; 1166 goto out; 1167 } 1168 set_page_private(pages[i], (unsigned long)memcg); 1169 } 1170 1171 for (i = 0; i < HPAGE_PMD_NR; i++) { 1172 copy_user_highpage(pages[i], page + i, 1173 haddr + PAGE_SIZE * i, vma); 1174 __SetPageUptodate(pages[i]); 1175 cond_resched(); 1176 } 1177 1178 mmun_start = haddr; 1179 mmun_end = haddr + HPAGE_PMD_SIZE; 1180 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 1181 1182 ptl = pmd_lock(mm, pmd); 1183 if (unlikely(!pmd_same(*pmd, orig_pmd))) 1184 goto out_free_pages; 1185 VM_BUG_ON_PAGE(!PageHead(page), page); 1186 1187 pmdp_huge_clear_flush_notify(vma, haddr, pmd); 1188 /* leave pmd empty until pte is filled */ 1189 1190 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 1191 pmd_populate(mm, &_pmd, pgtable); 1192 1193 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 1194 pte_t *pte, entry; 1195 entry = mk_pte(pages[i], vma->vm_page_prot); 1196 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 1197 memcg = (void *)page_private(pages[i]); 1198 set_page_private(pages[i], 0); 1199 page_add_new_anon_rmap(pages[i], vma, haddr, false); 1200 mem_cgroup_commit_charge(pages[i], memcg, false, false); 1201 lru_cache_add_active_or_unevictable(pages[i], vma); 1202 pte = pte_offset_map(&_pmd, haddr); 1203 VM_BUG_ON(!pte_none(*pte)); 1204 set_pte_at(mm, haddr, pte, entry); 1205 pte_unmap(pte); 1206 } 1207 kfree(pages); 1208 1209 smp_wmb(); /* make pte visible before pmd */ 1210 pmd_populate(mm, pmd, pgtable); 1211 page_remove_rmap(page, true); 1212 spin_unlock(ptl); 1213 1214 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 1215 1216 ret |= VM_FAULT_WRITE; 1217 put_page(page); 1218 1219 out: 1220 return ret; 1221 1222 out_free_pages: 1223 spin_unlock(ptl); 1224 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 1225 for (i = 0; i < HPAGE_PMD_NR; i++) { 1226 memcg = (void *)page_private(pages[i]); 1227 set_page_private(pages[i], 0); 1228 mem_cgroup_cancel_charge(pages[i], memcg, false); 1229 put_page(pages[i]); 1230 } 1231 kfree(pages); 1232 goto out; 1233 } 1234 1235 int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, 1236 unsigned long address, pmd_t *pmd, pmd_t orig_pmd) 1237 { 1238 spinlock_t *ptl; 1239 int ret = 0; 1240 struct page *page = NULL, *new_page; 1241 struct mem_cgroup *memcg; 1242 unsigned long haddr; 1243 unsigned long mmun_start; /* For mmu_notifiers */ 1244 unsigned long mmun_end; /* For mmu_notifiers */ 1245 gfp_t huge_gfp; /* for allocation and charge */ 1246 1247 ptl = pmd_lockptr(mm, pmd); 1248 VM_BUG_ON_VMA(!vma->anon_vma, vma); 1249 haddr = address & HPAGE_PMD_MASK; 1250 if (is_huge_zero_pmd(orig_pmd)) 1251 goto alloc; 1252 spin_lock(ptl); 1253 if (unlikely(!pmd_same(*pmd, orig_pmd))) 1254 goto out_unlock; 1255 1256 page = pmd_page(orig_pmd); 1257 VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page); 1258 /* 1259 * We can only reuse the page if nobody else maps the huge page or it's 1260 * part. We can do it by checking page_mapcount() on each sub-page, but 1261 * it's expensive. 1262 * The cheaper way is to check page_count() to be equal 1: every 1263 * mapcount takes page reference reference, so this way we can 1264 * guarantee, that the PMD is the only mapping. 1265 * This can give false negative if somebody pinned the page, but that's 1266 * fine. 1267 */ 1268 if (page_mapcount(page) == 1 && page_count(page) == 1) { 1269 pmd_t entry; 1270 entry = pmd_mkyoung(orig_pmd); 1271 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1272 if (pmdp_set_access_flags(vma, haddr, pmd, entry, 1)) 1273 update_mmu_cache_pmd(vma, address, pmd); 1274 ret |= VM_FAULT_WRITE; 1275 goto out_unlock; 1276 } 1277 get_page(page); 1278 spin_unlock(ptl); 1279 alloc: 1280 if (transparent_hugepage_enabled(vma) && 1281 !transparent_hugepage_debug_cow()) { 1282 huge_gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0); 1283 new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER); 1284 } else 1285 new_page = NULL; 1286 1287 if (likely(new_page)) { 1288 prep_transhuge_page(new_page); 1289 } else { 1290 if (!page) { 1291 split_huge_pmd(vma, pmd, address); 1292 ret |= VM_FAULT_FALLBACK; 1293 } else { 1294 ret = do_huge_pmd_wp_page_fallback(mm, vma, address, 1295 pmd, orig_pmd, page, haddr); 1296 if (ret & VM_FAULT_OOM) { 1297 split_huge_pmd(vma, pmd, address); 1298 ret |= VM_FAULT_FALLBACK; 1299 } 1300 put_page(page); 1301 } 1302 count_vm_event(THP_FAULT_FALLBACK); 1303 goto out; 1304 } 1305 1306 if (unlikely(mem_cgroup_try_charge(new_page, mm, huge_gfp, &memcg, 1307 true))) { 1308 put_page(new_page); 1309 if (page) { 1310 split_huge_pmd(vma, pmd, address); 1311 put_page(page); 1312 } else 1313 split_huge_pmd(vma, pmd, address); 1314 ret |= VM_FAULT_FALLBACK; 1315 count_vm_event(THP_FAULT_FALLBACK); 1316 goto out; 1317 } 1318 1319 count_vm_event(THP_FAULT_ALLOC); 1320 1321 if (!page) 1322 clear_huge_page(new_page, haddr, HPAGE_PMD_NR); 1323 else 1324 copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR); 1325 __SetPageUptodate(new_page); 1326 1327 mmun_start = haddr; 1328 mmun_end = haddr + HPAGE_PMD_SIZE; 1329 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 1330 1331 spin_lock(ptl); 1332 if (page) 1333 put_page(page); 1334 if (unlikely(!pmd_same(*pmd, orig_pmd))) { 1335 spin_unlock(ptl); 1336 mem_cgroup_cancel_charge(new_page, memcg, true); 1337 put_page(new_page); 1338 goto out_mn; 1339 } else { 1340 pmd_t entry; 1341 entry = mk_huge_pmd(new_page, vma->vm_page_prot); 1342 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1343 pmdp_huge_clear_flush_notify(vma, haddr, pmd); 1344 page_add_new_anon_rmap(new_page, vma, haddr, true); 1345 mem_cgroup_commit_charge(new_page, memcg, false, true); 1346 lru_cache_add_active_or_unevictable(new_page, vma); 1347 set_pmd_at(mm, haddr, pmd, entry); 1348 update_mmu_cache_pmd(vma, address, pmd); 1349 if (!page) { 1350 add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR); 1351 put_huge_zero_page(); 1352 } else { 1353 VM_BUG_ON_PAGE(!PageHead(page), page); 1354 page_remove_rmap(page, true); 1355 put_page(page); 1356 } 1357 ret |= VM_FAULT_WRITE; 1358 } 1359 spin_unlock(ptl); 1360 out_mn: 1361 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 1362 out: 1363 return ret; 1364 out_unlock: 1365 spin_unlock(ptl); 1366 return ret; 1367 } 1368 1369 struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, 1370 unsigned long addr, 1371 pmd_t *pmd, 1372 unsigned int flags) 1373 { 1374 struct mm_struct *mm = vma->vm_mm; 1375 struct page *page = NULL; 1376 1377 assert_spin_locked(pmd_lockptr(mm, pmd)); 1378 1379 if (flags & FOLL_WRITE && !pmd_write(*pmd)) 1380 goto out; 1381 1382 /* Avoid dumping huge zero page */ 1383 if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd)) 1384 return ERR_PTR(-EFAULT); 1385 1386 /* Full NUMA hinting faults to serialise migration in fault paths */ 1387 if ((flags & FOLL_NUMA) && pmd_protnone(*pmd)) 1388 goto out; 1389 1390 page = pmd_page(*pmd); 1391 VM_BUG_ON_PAGE(!PageHead(page), page); 1392 if (flags & FOLL_TOUCH) 1393 touch_pmd(vma, addr, pmd); 1394 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { 1395 /* 1396 * We don't mlock() pte-mapped THPs. This way we can avoid 1397 * leaking mlocked pages into non-VM_LOCKED VMAs. 1398 * 1399 * In most cases the pmd is the only mapping of the page as we 1400 * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for 1401 * writable private mappings in populate_vma_page_range(). 1402 * 1403 * The only scenario when we have the page shared here is if we 1404 * mlocking read-only mapping shared over fork(). We skip 1405 * mlocking such pages. 1406 */ 1407 if (compound_mapcount(page) == 1 && !PageDoubleMap(page) && 1408 page->mapping && trylock_page(page)) { 1409 lru_add_drain(); 1410 if (page->mapping) 1411 mlock_vma_page(page); 1412 unlock_page(page); 1413 } 1414 } 1415 page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT; 1416 VM_BUG_ON_PAGE(!PageCompound(page), page); 1417 if (flags & FOLL_GET) 1418 get_page(page); 1419 1420 out: 1421 return page; 1422 } 1423 1424 /* NUMA hinting page fault entry point for trans huge pmds */ 1425 int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, 1426 unsigned long addr, pmd_t pmd, pmd_t *pmdp) 1427 { 1428 spinlock_t *ptl; 1429 struct anon_vma *anon_vma = NULL; 1430 struct page *page; 1431 unsigned long haddr = addr & HPAGE_PMD_MASK; 1432 int page_nid = -1, this_nid = numa_node_id(); 1433 int target_nid, last_cpupid = -1; 1434 bool page_locked; 1435 bool migrated = false; 1436 bool was_writable; 1437 int flags = 0; 1438 1439 /* A PROT_NONE fault should not end up here */ 1440 BUG_ON(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))); 1441 1442 ptl = pmd_lock(mm, pmdp); 1443 if (unlikely(!pmd_same(pmd, *pmdp))) 1444 goto out_unlock; 1445 1446 /* 1447 * If there are potential migrations, wait for completion and retry 1448 * without disrupting NUMA hinting information. Do not relock and 1449 * check_same as the page may no longer be mapped. 1450 */ 1451 if (unlikely(pmd_trans_migrating(*pmdp))) { 1452 page = pmd_page(*pmdp); 1453 spin_unlock(ptl); 1454 wait_on_page_locked(page); 1455 goto out; 1456 } 1457 1458 page = pmd_page(pmd); 1459 BUG_ON(is_huge_zero_page(page)); 1460 page_nid = page_to_nid(page); 1461 last_cpupid = page_cpupid_last(page); 1462 count_vm_numa_event(NUMA_HINT_FAULTS); 1463 if (page_nid == this_nid) { 1464 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL); 1465 flags |= TNF_FAULT_LOCAL; 1466 } 1467 1468 /* See similar comment in do_numa_page for explanation */ 1469 if (!(vma->vm_flags & VM_WRITE)) 1470 flags |= TNF_NO_GROUP; 1471 1472 /* 1473 * Acquire the page lock to serialise THP migrations but avoid dropping 1474 * page_table_lock if at all possible 1475 */ 1476 page_locked = trylock_page(page); 1477 target_nid = mpol_misplaced(page, vma, haddr); 1478 if (target_nid == -1) { 1479 /* If the page was locked, there are no parallel migrations */ 1480 if (page_locked) 1481 goto clear_pmdnuma; 1482 } 1483 1484 /* Migration could have started since the pmd_trans_migrating check */ 1485 if (!page_locked) { 1486 spin_unlock(ptl); 1487 wait_on_page_locked(page); 1488 page_nid = -1; 1489 goto out; 1490 } 1491 1492 /* 1493 * Page is misplaced. Page lock serialises migrations. Acquire anon_vma 1494 * to serialises splits 1495 */ 1496 get_page(page); 1497 spin_unlock(ptl); 1498 anon_vma = page_lock_anon_vma_read(page); 1499 1500 /* Confirm the PMD did not change while page_table_lock was released */ 1501 spin_lock(ptl); 1502 if (unlikely(!pmd_same(pmd, *pmdp))) { 1503 unlock_page(page); 1504 put_page(page); 1505 page_nid = -1; 1506 goto out_unlock; 1507 } 1508 1509 /* Bail if we fail to protect against THP splits for any reason */ 1510 if (unlikely(!anon_vma)) { 1511 put_page(page); 1512 page_nid = -1; 1513 goto clear_pmdnuma; 1514 } 1515 1516 /* 1517 * Migrate the THP to the requested node, returns with page unlocked 1518 * and access rights restored. 1519 */ 1520 spin_unlock(ptl); 1521 migrated = migrate_misplaced_transhuge_page(mm, vma, 1522 pmdp, pmd, addr, page, target_nid); 1523 if (migrated) { 1524 flags |= TNF_MIGRATED; 1525 page_nid = target_nid; 1526 } else 1527 flags |= TNF_MIGRATE_FAIL; 1528 1529 goto out; 1530 clear_pmdnuma: 1531 BUG_ON(!PageLocked(page)); 1532 was_writable = pmd_write(pmd); 1533 pmd = pmd_modify(pmd, vma->vm_page_prot); 1534 pmd = pmd_mkyoung(pmd); 1535 if (was_writable) 1536 pmd = pmd_mkwrite(pmd); 1537 set_pmd_at(mm, haddr, pmdp, pmd); 1538 update_mmu_cache_pmd(vma, addr, pmdp); 1539 unlock_page(page); 1540 out_unlock: 1541 spin_unlock(ptl); 1542 1543 out: 1544 if (anon_vma) 1545 page_unlock_anon_vma_read(anon_vma); 1546 1547 if (page_nid != -1) 1548 task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, flags); 1549 1550 return 0; 1551 } 1552 1553 int madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1554 pmd_t *pmd, unsigned long addr, unsigned long next) 1555 1556 { 1557 spinlock_t *ptl; 1558 pmd_t orig_pmd; 1559 struct page *page; 1560 struct mm_struct *mm = tlb->mm; 1561 int ret = 0; 1562 1563 ptl = pmd_trans_huge_lock(pmd, vma); 1564 if (!ptl) 1565 goto out_unlocked; 1566 1567 orig_pmd = *pmd; 1568 if (is_huge_zero_pmd(orig_pmd)) { 1569 ret = 1; 1570 goto out; 1571 } 1572 1573 page = pmd_page(orig_pmd); 1574 /* 1575 * If other processes are mapping this page, we couldn't discard 1576 * the page unless they all do MADV_FREE so let's skip the page. 1577 */ 1578 if (page_mapcount(page) != 1) 1579 goto out; 1580 1581 if (!trylock_page(page)) 1582 goto out; 1583 1584 /* 1585 * If user want to discard part-pages of THP, split it so MADV_FREE 1586 * will deactivate only them. 1587 */ 1588 if (next - addr != HPAGE_PMD_SIZE) { 1589 get_page(page); 1590 spin_unlock(ptl); 1591 if (split_huge_page(page)) { 1592 put_page(page); 1593 unlock_page(page); 1594 goto out_unlocked; 1595 } 1596 put_page(page); 1597 unlock_page(page); 1598 ret = 1; 1599 goto out_unlocked; 1600 } 1601 1602 if (PageDirty(page)) 1603 ClearPageDirty(page); 1604 unlock_page(page); 1605 1606 if (PageActive(page)) 1607 deactivate_page(page); 1608 1609 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) { 1610 orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd, 1611 tlb->fullmm); 1612 orig_pmd = pmd_mkold(orig_pmd); 1613 orig_pmd = pmd_mkclean(orig_pmd); 1614 1615 set_pmd_at(mm, addr, pmd, orig_pmd); 1616 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 1617 } 1618 ret = 1; 1619 out: 1620 spin_unlock(ptl); 1621 out_unlocked: 1622 return ret; 1623 } 1624 1625 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1626 pmd_t *pmd, unsigned long addr) 1627 { 1628 pmd_t orig_pmd; 1629 spinlock_t *ptl; 1630 1631 ptl = __pmd_trans_huge_lock(pmd, vma); 1632 if (!ptl) 1633 return 0; 1634 /* 1635 * For architectures like ppc64 we look at deposited pgtable 1636 * when calling pmdp_huge_get_and_clear. So do the 1637 * pgtable_trans_huge_withdraw after finishing pmdp related 1638 * operations. 1639 */ 1640 orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd, 1641 tlb->fullmm); 1642 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 1643 if (vma_is_dax(vma)) { 1644 spin_unlock(ptl); 1645 if (is_huge_zero_pmd(orig_pmd)) 1646 put_huge_zero_page(); 1647 } else if (is_huge_zero_pmd(orig_pmd)) { 1648 pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd)); 1649 atomic_long_dec(&tlb->mm->nr_ptes); 1650 spin_unlock(ptl); 1651 put_huge_zero_page(); 1652 } else { 1653 struct page *page = pmd_page(orig_pmd); 1654 page_remove_rmap(page, true); 1655 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page); 1656 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); 1657 VM_BUG_ON_PAGE(!PageHead(page), page); 1658 pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd)); 1659 atomic_long_dec(&tlb->mm->nr_ptes); 1660 spin_unlock(ptl); 1661 tlb_remove_page(tlb, page); 1662 } 1663 return 1; 1664 } 1665 1666 bool move_huge_pmd(struct vm_area_struct *vma, struct vm_area_struct *new_vma, 1667 unsigned long old_addr, 1668 unsigned long new_addr, unsigned long old_end, 1669 pmd_t *old_pmd, pmd_t *new_pmd) 1670 { 1671 spinlock_t *old_ptl, *new_ptl; 1672 pmd_t pmd; 1673 1674 struct mm_struct *mm = vma->vm_mm; 1675 1676 if ((old_addr & ~HPAGE_PMD_MASK) || 1677 (new_addr & ~HPAGE_PMD_MASK) || 1678 old_end - old_addr < HPAGE_PMD_SIZE || 1679 (new_vma->vm_flags & VM_NOHUGEPAGE)) 1680 return false; 1681 1682 /* 1683 * The destination pmd shouldn't be established, free_pgtables() 1684 * should have release it. 1685 */ 1686 if (WARN_ON(!pmd_none(*new_pmd))) { 1687 VM_BUG_ON(pmd_trans_huge(*new_pmd)); 1688 return false; 1689 } 1690 1691 /* 1692 * We don't have to worry about the ordering of src and dst 1693 * ptlocks because exclusive mmap_sem prevents deadlock. 1694 */ 1695 old_ptl = __pmd_trans_huge_lock(old_pmd, vma); 1696 if (old_ptl) { 1697 new_ptl = pmd_lockptr(mm, new_pmd); 1698 if (new_ptl != old_ptl) 1699 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 1700 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); 1701 VM_BUG_ON(!pmd_none(*new_pmd)); 1702 1703 if (pmd_move_must_withdraw(new_ptl, old_ptl) && 1704 vma_is_anonymous(vma)) { 1705 pgtable_t pgtable; 1706 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); 1707 pgtable_trans_huge_deposit(mm, new_pmd, pgtable); 1708 } 1709 set_pmd_at(mm, new_addr, new_pmd, pmd_mksoft_dirty(pmd)); 1710 if (new_ptl != old_ptl) 1711 spin_unlock(new_ptl); 1712 spin_unlock(old_ptl); 1713 return true; 1714 } 1715 return false; 1716 } 1717 1718 /* 1719 * Returns 1720 * - 0 if PMD could not be locked 1721 * - 1 if PMD was locked but protections unchange and TLB flush unnecessary 1722 * - HPAGE_PMD_NR is protections changed and TLB flush necessary 1723 */ 1724 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 1725 unsigned long addr, pgprot_t newprot, int prot_numa) 1726 { 1727 struct mm_struct *mm = vma->vm_mm; 1728 spinlock_t *ptl; 1729 int ret = 0; 1730 1731 ptl = __pmd_trans_huge_lock(pmd, vma); 1732 if (ptl) { 1733 pmd_t entry; 1734 bool preserve_write = prot_numa && pmd_write(*pmd); 1735 ret = 1; 1736 1737 /* 1738 * Avoid trapping faults against the zero page. The read-only 1739 * data is likely to be read-cached on the local CPU and 1740 * local/remote hits to the zero page are not interesting. 1741 */ 1742 if (prot_numa && is_huge_zero_pmd(*pmd)) { 1743 spin_unlock(ptl); 1744 return ret; 1745 } 1746 1747 if (!prot_numa || !pmd_protnone(*pmd)) { 1748 entry = pmdp_huge_get_and_clear_notify(mm, addr, pmd); 1749 entry = pmd_modify(entry, newprot); 1750 if (preserve_write) 1751 entry = pmd_mkwrite(entry); 1752 ret = HPAGE_PMD_NR; 1753 set_pmd_at(mm, addr, pmd, entry); 1754 BUG_ON(!preserve_write && pmd_write(entry)); 1755 } 1756 spin_unlock(ptl); 1757 } 1758 1759 return ret; 1760 } 1761 1762 /* 1763 * Returns true if a given pmd maps a thp, false otherwise. 1764 * 1765 * Note that if it returns true, this routine returns without unlocking page 1766 * table lock. So callers must unlock it. 1767 */ 1768 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) 1769 { 1770 spinlock_t *ptl; 1771 ptl = pmd_lock(vma->vm_mm, pmd); 1772 if (likely(pmd_trans_huge(*pmd) || pmd_devmap(*pmd))) 1773 return ptl; 1774 spin_unlock(ptl); 1775 return NULL; 1776 } 1777 1778 #define VM_NO_THP (VM_SPECIAL | VM_HUGETLB | VM_SHARED | VM_MAYSHARE) 1779 1780 int hugepage_madvise(struct vm_area_struct *vma, 1781 unsigned long *vm_flags, int advice) 1782 { 1783 switch (advice) { 1784 case MADV_HUGEPAGE: 1785 #ifdef CONFIG_S390 1786 /* 1787 * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390 1788 * can't handle this properly after s390_enable_sie, so we simply 1789 * ignore the madvise to prevent qemu from causing a SIGSEGV. 1790 */ 1791 if (mm_has_pgste(vma->vm_mm)) 1792 return 0; 1793 #endif 1794 /* 1795 * Be somewhat over-protective like KSM for now! 1796 */ 1797 if (*vm_flags & VM_NO_THP) 1798 return -EINVAL; 1799 *vm_flags &= ~VM_NOHUGEPAGE; 1800 *vm_flags |= VM_HUGEPAGE; 1801 /* 1802 * If the vma become good for khugepaged to scan, 1803 * register it here without waiting a page fault that 1804 * may not happen any time soon. 1805 */ 1806 if (unlikely(khugepaged_enter_vma_merge(vma, *vm_flags))) 1807 return -ENOMEM; 1808 break; 1809 case MADV_NOHUGEPAGE: 1810 /* 1811 * Be somewhat over-protective like KSM for now! 1812 */ 1813 if (*vm_flags & VM_NO_THP) 1814 return -EINVAL; 1815 *vm_flags &= ~VM_HUGEPAGE; 1816 *vm_flags |= VM_NOHUGEPAGE; 1817 /* 1818 * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning 1819 * this vma even if we leave the mm registered in khugepaged if 1820 * it got registered before VM_NOHUGEPAGE was set. 1821 */ 1822 break; 1823 } 1824 1825 return 0; 1826 } 1827 1828 static int __init khugepaged_slab_init(void) 1829 { 1830 mm_slot_cache = kmem_cache_create("khugepaged_mm_slot", 1831 sizeof(struct mm_slot), 1832 __alignof__(struct mm_slot), 0, NULL); 1833 if (!mm_slot_cache) 1834 return -ENOMEM; 1835 1836 return 0; 1837 } 1838 1839 static void __init khugepaged_slab_exit(void) 1840 { 1841 kmem_cache_destroy(mm_slot_cache); 1842 } 1843 1844 static inline struct mm_slot *alloc_mm_slot(void) 1845 { 1846 if (!mm_slot_cache) /* initialization failed */ 1847 return NULL; 1848 return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL); 1849 } 1850 1851 static inline void free_mm_slot(struct mm_slot *mm_slot) 1852 { 1853 kmem_cache_free(mm_slot_cache, mm_slot); 1854 } 1855 1856 static struct mm_slot *get_mm_slot(struct mm_struct *mm) 1857 { 1858 struct mm_slot *mm_slot; 1859 1860 hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm) 1861 if (mm == mm_slot->mm) 1862 return mm_slot; 1863 1864 return NULL; 1865 } 1866 1867 static void insert_to_mm_slots_hash(struct mm_struct *mm, 1868 struct mm_slot *mm_slot) 1869 { 1870 mm_slot->mm = mm; 1871 hash_add(mm_slots_hash, &mm_slot->hash, (long)mm); 1872 } 1873 1874 static inline int khugepaged_test_exit(struct mm_struct *mm) 1875 { 1876 return atomic_read(&mm->mm_users) == 0; 1877 } 1878 1879 int __khugepaged_enter(struct mm_struct *mm) 1880 { 1881 struct mm_slot *mm_slot; 1882 int wakeup; 1883 1884 mm_slot = alloc_mm_slot(); 1885 if (!mm_slot) 1886 return -ENOMEM; 1887 1888 /* __khugepaged_exit() must not run from under us */ 1889 VM_BUG_ON_MM(khugepaged_test_exit(mm), mm); 1890 if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) { 1891 free_mm_slot(mm_slot); 1892 return 0; 1893 } 1894 1895 spin_lock(&khugepaged_mm_lock); 1896 insert_to_mm_slots_hash(mm, mm_slot); 1897 /* 1898 * Insert just behind the scanning cursor, to let the area settle 1899 * down a little. 1900 */ 1901 wakeup = list_empty(&khugepaged_scan.mm_head); 1902 list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head); 1903 spin_unlock(&khugepaged_mm_lock); 1904 1905 atomic_inc(&mm->mm_count); 1906 if (wakeup) 1907 wake_up_interruptible(&khugepaged_wait); 1908 1909 return 0; 1910 } 1911 1912 int khugepaged_enter_vma_merge(struct vm_area_struct *vma, 1913 unsigned long vm_flags) 1914 { 1915 unsigned long hstart, hend; 1916 if (!vma->anon_vma) 1917 /* 1918 * Not yet faulted in so we will register later in the 1919 * page fault if needed. 1920 */ 1921 return 0; 1922 if (vma->vm_ops) 1923 /* khugepaged not yet working on file or special mappings */ 1924 return 0; 1925 VM_BUG_ON_VMA(vm_flags & VM_NO_THP, vma); 1926 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 1927 hend = vma->vm_end & HPAGE_PMD_MASK; 1928 if (hstart < hend) 1929 return khugepaged_enter(vma, vm_flags); 1930 return 0; 1931 } 1932 1933 void __khugepaged_exit(struct mm_struct *mm) 1934 { 1935 struct mm_slot *mm_slot; 1936 int free = 0; 1937 1938 spin_lock(&khugepaged_mm_lock); 1939 mm_slot = get_mm_slot(mm); 1940 if (mm_slot && khugepaged_scan.mm_slot != mm_slot) { 1941 hash_del(&mm_slot->hash); 1942 list_del(&mm_slot->mm_node); 1943 free = 1; 1944 } 1945 spin_unlock(&khugepaged_mm_lock); 1946 1947 if (free) { 1948 clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 1949 free_mm_slot(mm_slot); 1950 mmdrop(mm); 1951 } else if (mm_slot) { 1952 /* 1953 * This is required to serialize against 1954 * khugepaged_test_exit() (which is guaranteed to run 1955 * under mmap sem read mode). Stop here (after we 1956 * return all pagetables will be destroyed) until 1957 * khugepaged has finished working on the pagetables 1958 * under the mmap_sem. 1959 */ 1960 down_write(&mm->mmap_sem); 1961 up_write(&mm->mmap_sem); 1962 } 1963 } 1964 1965 static void release_pte_page(struct page *page) 1966 { 1967 /* 0 stands for page_is_file_cache(page) == false */ 1968 dec_zone_page_state(page, NR_ISOLATED_ANON + 0); 1969 unlock_page(page); 1970 putback_lru_page(page); 1971 } 1972 1973 static void release_pte_pages(pte_t *pte, pte_t *_pte) 1974 { 1975 while (--_pte >= pte) { 1976 pte_t pteval = *_pte; 1977 if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval))) 1978 release_pte_page(pte_page(pteval)); 1979 } 1980 } 1981 1982 static int __collapse_huge_page_isolate(struct vm_area_struct *vma, 1983 unsigned long address, 1984 pte_t *pte) 1985 { 1986 struct page *page = NULL; 1987 pte_t *_pte; 1988 int none_or_zero = 0, result = 0; 1989 bool referenced = false, writable = false; 1990 1991 for (_pte = pte; _pte < pte+HPAGE_PMD_NR; 1992 _pte++, address += PAGE_SIZE) { 1993 pte_t pteval = *_pte; 1994 if (pte_none(pteval) || (pte_present(pteval) && 1995 is_zero_pfn(pte_pfn(pteval)))) { 1996 if (!userfaultfd_armed(vma) && 1997 ++none_or_zero <= khugepaged_max_ptes_none) { 1998 continue; 1999 } else { 2000 result = SCAN_EXCEED_NONE_PTE; 2001 goto out; 2002 } 2003 } 2004 if (!pte_present(pteval)) { 2005 result = SCAN_PTE_NON_PRESENT; 2006 goto out; 2007 } 2008 page = vm_normal_page(vma, address, pteval); 2009 if (unlikely(!page)) { 2010 result = SCAN_PAGE_NULL; 2011 goto out; 2012 } 2013 2014 VM_BUG_ON_PAGE(PageCompound(page), page); 2015 VM_BUG_ON_PAGE(!PageAnon(page), page); 2016 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 2017 2018 /* 2019 * We can do it before isolate_lru_page because the 2020 * page can't be freed from under us. NOTE: PG_lock 2021 * is needed to serialize against split_huge_page 2022 * when invoked from the VM. 2023 */ 2024 if (!trylock_page(page)) { 2025 result = SCAN_PAGE_LOCK; 2026 goto out; 2027 } 2028 2029 /* 2030 * cannot use mapcount: can't collapse if there's a gup pin. 2031 * The page must only be referenced by the scanned process 2032 * and page swap cache. 2033 */ 2034 if (page_count(page) != 1 + !!PageSwapCache(page)) { 2035 unlock_page(page); 2036 result = SCAN_PAGE_COUNT; 2037 goto out; 2038 } 2039 if (pte_write(pteval)) { 2040 writable = true; 2041 } else { 2042 if (PageSwapCache(page) && !reuse_swap_page(page)) { 2043 unlock_page(page); 2044 result = SCAN_SWAP_CACHE_PAGE; 2045 goto out; 2046 } 2047 /* 2048 * Page is not in the swap cache. It can be collapsed 2049 * into a THP. 2050 */ 2051 } 2052 2053 /* 2054 * Isolate the page to avoid collapsing an hugepage 2055 * currently in use by the VM. 2056 */ 2057 if (isolate_lru_page(page)) { 2058 unlock_page(page); 2059 result = SCAN_DEL_PAGE_LRU; 2060 goto out; 2061 } 2062 /* 0 stands for page_is_file_cache(page) == false */ 2063 inc_zone_page_state(page, NR_ISOLATED_ANON + 0); 2064 VM_BUG_ON_PAGE(!PageLocked(page), page); 2065 VM_BUG_ON_PAGE(PageLRU(page), page); 2066 2067 /* If there is no mapped pte young don't collapse the page */ 2068 if (pte_young(pteval) || 2069 page_is_young(page) || PageReferenced(page) || 2070 mmu_notifier_test_young(vma->vm_mm, address)) 2071 referenced = true; 2072 } 2073 if (likely(writable)) { 2074 if (likely(referenced)) { 2075 result = SCAN_SUCCEED; 2076 trace_mm_collapse_huge_page_isolate(page, none_or_zero, 2077 referenced, writable, result); 2078 return 1; 2079 } 2080 } else { 2081 result = SCAN_PAGE_RO; 2082 } 2083 2084 out: 2085 release_pte_pages(pte, _pte); 2086 trace_mm_collapse_huge_page_isolate(page, none_or_zero, 2087 referenced, writable, result); 2088 return 0; 2089 } 2090 2091 static void __collapse_huge_page_copy(pte_t *pte, struct page *page, 2092 struct vm_area_struct *vma, 2093 unsigned long address, 2094 spinlock_t *ptl) 2095 { 2096 pte_t *_pte; 2097 for (_pte = pte; _pte < pte+HPAGE_PMD_NR; _pte++) { 2098 pte_t pteval = *_pte; 2099 struct page *src_page; 2100 2101 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 2102 clear_user_highpage(page, address); 2103 add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1); 2104 if (is_zero_pfn(pte_pfn(pteval))) { 2105 /* 2106 * ptl mostly unnecessary. 2107 */ 2108 spin_lock(ptl); 2109 /* 2110 * paravirt calls inside pte_clear here are 2111 * superfluous. 2112 */ 2113 pte_clear(vma->vm_mm, address, _pte); 2114 spin_unlock(ptl); 2115 } 2116 } else { 2117 src_page = pte_page(pteval); 2118 copy_user_highpage(page, src_page, address, vma); 2119 VM_BUG_ON_PAGE(page_mapcount(src_page) != 1, src_page); 2120 release_pte_page(src_page); 2121 /* 2122 * ptl mostly unnecessary, but preempt has to 2123 * be disabled to update the per-cpu stats 2124 * inside page_remove_rmap(). 2125 */ 2126 spin_lock(ptl); 2127 /* 2128 * paravirt calls inside pte_clear here are 2129 * superfluous. 2130 */ 2131 pte_clear(vma->vm_mm, address, _pte); 2132 page_remove_rmap(src_page, false); 2133 spin_unlock(ptl); 2134 free_page_and_swap_cache(src_page); 2135 } 2136 2137 address += PAGE_SIZE; 2138 page++; 2139 } 2140 } 2141 2142 static void khugepaged_alloc_sleep(void) 2143 { 2144 DEFINE_WAIT(wait); 2145 2146 add_wait_queue(&khugepaged_wait, &wait); 2147 freezable_schedule_timeout_interruptible( 2148 msecs_to_jiffies(khugepaged_alloc_sleep_millisecs)); 2149 remove_wait_queue(&khugepaged_wait, &wait); 2150 } 2151 2152 static int khugepaged_node_load[MAX_NUMNODES]; 2153 2154 static bool khugepaged_scan_abort(int nid) 2155 { 2156 int i; 2157 2158 /* 2159 * If zone_reclaim_mode is disabled, then no extra effort is made to 2160 * allocate memory locally. 2161 */ 2162 if (!zone_reclaim_mode) 2163 return false; 2164 2165 /* If there is a count for this node already, it must be acceptable */ 2166 if (khugepaged_node_load[nid]) 2167 return false; 2168 2169 for (i = 0; i < MAX_NUMNODES; i++) { 2170 if (!khugepaged_node_load[i]) 2171 continue; 2172 if (node_distance(nid, i) > RECLAIM_DISTANCE) 2173 return true; 2174 } 2175 return false; 2176 } 2177 2178 #ifdef CONFIG_NUMA 2179 static int khugepaged_find_target_node(void) 2180 { 2181 static int last_khugepaged_target_node = NUMA_NO_NODE; 2182 int nid, target_node = 0, max_value = 0; 2183 2184 /* find first node with max normal pages hit */ 2185 for (nid = 0; nid < MAX_NUMNODES; nid++) 2186 if (khugepaged_node_load[nid] > max_value) { 2187 max_value = khugepaged_node_load[nid]; 2188 target_node = nid; 2189 } 2190 2191 /* do some balance if several nodes have the same hit record */ 2192 if (target_node <= last_khugepaged_target_node) 2193 for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES; 2194 nid++) 2195 if (max_value == khugepaged_node_load[nid]) { 2196 target_node = nid; 2197 break; 2198 } 2199 2200 last_khugepaged_target_node = target_node; 2201 return target_node; 2202 } 2203 2204 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) 2205 { 2206 if (IS_ERR(*hpage)) { 2207 if (!*wait) 2208 return false; 2209 2210 *wait = false; 2211 *hpage = NULL; 2212 khugepaged_alloc_sleep(); 2213 } else if (*hpage) { 2214 put_page(*hpage); 2215 *hpage = NULL; 2216 } 2217 2218 return true; 2219 } 2220 2221 static struct page * 2222 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm, 2223 unsigned long address, int node) 2224 { 2225 VM_BUG_ON_PAGE(*hpage, *hpage); 2226 2227 /* 2228 * Before allocating the hugepage, release the mmap_sem read lock. 2229 * The allocation can take potentially a long time if it involves 2230 * sync compaction, and we do not need to hold the mmap_sem during 2231 * that. We will recheck the vma after taking it again in write mode. 2232 */ 2233 up_read(&mm->mmap_sem); 2234 2235 *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER); 2236 if (unlikely(!*hpage)) { 2237 count_vm_event(THP_COLLAPSE_ALLOC_FAILED); 2238 *hpage = ERR_PTR(-ENOMEM); 2239 return NULL; 2240 } 2241 2242 prep_transhuge_page(*hpage); 2243 count_vm_event(THP_COLLAPSE_ALLOC); 2244 return *hpage; 2245 } 2246 #else 2247 static int khugepaged_find_target_node(void) 2248 { 2249 return 0; 2250 } 2251 2252 static inline struct page *alloc_hugepage(int defrag) 2253 { 2254 struct page *page; 2255 2256 page = alloc_pages(alloc_hugepage_gfpmask(defrag, 0), HPAGE_PMD_ORDER); 2257 if (page) 2258 prep_transhuge_page(page); 2259 return page; 2260 } 2261 2262 static struct page *khugepaged_alloc_hugepage(bool *wait) 2263 { 2264 struct page *hpage; 2265 2266 do { 2267 hpage = alloc_hugepage(khugepaged_defrag()); 2268 if (!hpage) { 2269 count_vm_event(THP_COLLAPSE_ALLOC_FAILED); 2270 if (!*wait) 2271 return NULL; 2272 2273 *wait = false; 2274 khugepaged_alloc_sleep(); 2275 } else 2276 count_vm_event(THP_COLLAPSE_ALLOC); 2277 } while (unlikely(!hpage) && likely(khugepaged_enabled())); 2278 2279 return hpage; 2280 } 2281 2282 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) 2283 { 2284 if (!*hpage) 2285 *hpage = khugepaged_alloc_hugepage(wait); 2286 2287 if (unlikely(!*hpage)) 2288 return false; 2289 2290 return true; 2291 } 2292 2293 static struct page * 2294 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm, 2295 unsigned long address, int node) 2296 { 2297 up_read(&mm->mmap_sem); 2298 VM_BUG_ON(!*hpage); 2299 2300 return *hpage; 2301 } 2302 #endif 2303 2304 static bool hugepage_vma_check(struct vm_area_struct *vma) 2305 { 2306 if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) || 2307 (vma->vm_flags & VM_NOHUGEPAGE)) 2308 return false; 2309 if (!vma->anon_vma || vma->vm_ops) 2310 return false; 2311 if (is_vma_temporary_stack(vma)) 2312 return false; 2313 VM_BUG_ON_VMA(vma->vm_flags & VM_NO_THP, vma); 2314 return true; 2315 } 2316 2317 static void collapse_huge_page(struct mm_struct *mm, 2318 unsigned long address, 2319 struct page **hpage, 2320 struct vm_area_struct *vma, 2321 int node) 2322 { 2323 pmd_t *pmd, _pmd; 2324 pte_t *pte; 2325 pgtable_t pgtable; 2326 struct page *new_page; 2327 spinlock_t *pmd_ptl, *pte_ptl; 2328 int isolated = 0, result = 0; 2329 unsigned long hstart, hend; 2330 struct mem_cgroup *memcg; 2331 unsigned long mmun_start; /* For mmu_notifiers */ 2332 unsigned long mmun_end; /* For mmu_notifiers */ 2333 gfp_t gfp; 2334 2335 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 2336 2337 /* Only allocate from the target node */ 2338 gfp = alloc_hugepage_gfpmask(khugepaged_defrag(), __GFP_OTHER_NODE) | 2339 __GFP_THISNODE; 2340 2341 /* release the mmap_sem read lock. */ 2342 new_page = khugepaged_alloc_page(hpage, gfp, mm, address, node); 2343 if (!new_page) { 2344 result = SCAN_ALLOC_HUGE_PAGE_FAIL; 2345 goto out_nolock; 2346 } 2347 2348 if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg, true))) { 2349 result = SCAN_CGROUP_CHARGE_FAIL; 2350 goto out_nolock; 2351 } 2352 2353 /* 2354 * Prevent all access to pagetables with the exception of 2355 * gup_fast later hanlded by the ptep_clear_flush and the VM 2356 * handled by the anon_vma lock + PG_lock. 2357 */ 2358 down_write(&mm->mmap_sem); 2359 if (unlikely(khugepaged_test_exit(mm))) { 2360 result = SCAN_ANY_PROCESS; 2361 goto out; 2362 } 2363 2364 vma = find_vma(mm, address); 2365 if (!vma) { 2366 result = SCAN_VMA_NULL; 2367 goto out; 2368 } 2369 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 2370 hend = vma->vm_end & HPAGE_PMD_MASK; 2371 if (address < hstart || address + HPAGE_PMD_SIZE > hend) { 2372 result = SCAN_ADDRESS_RANGE; 2373 goto out; 2374 } 2375 if (!hugepage_vma_check(vma)) { 2376 result = SCAN_VMA_CHECK; 2377 goto out; 2378 } 2379 pmd = mm_find_pmd(mm, address); 2380 if (!pmd) { 2381 result = SCAN_PMD_NULL; 2382 goto out; 2383 } 2384 2385 anon_vma_lock_write(vma->anon_vma); 2386 2387 pte = pte_offset_map(pmd, address); 2388 pte_ptl = pte_lockptr(mm, pmd); 2389 2390 mmun_start = address; 2391 mmun_end = address + HPAGE_PMD_SIZE; 2392 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); 2393 pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */ 2394 /* 2395 * After this gup_fast can't run anymore. This also removes 2396 * any huge TLB entry from the CPU so we won't allow 2397 * huge and small TLB entries for the same virtual address 2398 * to avoid the risk of CPU bugs in that area. 2399 */ 2400 _pmd = pmdp_collapse_flush(vma, address, pmd); 2401 spin_unlock(pmd_ptl); 2402 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); 2403 2404 spin_lock(pte_ptl); 2405 isolated = __collapse_huge_page_isolate(vma, address, pte); 2406 spin_unlock(pte_ptl); 2407 2408 if (unlikely(!isolated)) { 2409 pte_unmap(pte); 2410 spin_lock(pmd_ptl); 2411 BUG_ON(!pmd_none(*pmd)); 2412 /* 2413 * We can only use set_pmd_at when establishing 2414 * hugepmds and never for establishing regular pmds that 2415 * points to regular pagetables. Use pmd_populate for that 2416 */ 2417 pmd_populate(mm, pmd, pmd_pgtable(_pmd)); 2418 spin_unlock(pmd_ptl); 2419 anon_vma_unlock_write(vma->anon_vma); 2420 result = SCAN_FAIL; 2421 goto out; 2422 } 2423 2424 /* 2425 * All pages are isolated and locked so anon_vma rmap 2426 * can't run anymore. 2427 */ 2428 anon_vma_unlock_write(vma->anon_vma); 2429 2430 __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl); 2431 pte_unmap(pte); 2432 __SetPageUptodate(new_page); 2433 pgtable = pmd_pgtable(_pmd); 2434 2435 _pmd = mk_huge_pmd(new_page, vma->vm_page_prot); 2436 _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma); 2437 2438 /* 2439 * spin_lock() below is not the equivalent of smp_wmb(), so 2440 * this is needed to avoid the copy_huge_page writes to become 2441 * visible after the set_pmd_at() write. 2442 */ 2443 smp_wmb(); 2444 2445 spin_lock(pmd_ptl); 2446 BUG_ON(!pmd_none(*pmd)); 2447 page_add_new_anon_rmap(new_page, vma, address, true); 2448 mem_cgroup_commit_charge(new_page, memcg, false, true); 2449 lru_cache_add_active_or_unevictable(new_page, vma); 2450 pgtable_trans_huge_deposit(mm, pmd, pgtable); 2451 set_pmd_at(mm, address, pmd, _pmd); 2452 update_mmu_cache_pmd(vma, address, pmd); 2453 spin_unlock(pmd_ptl); 2454 2455 *hpage = NULL; 2456 2457 khugepaged_pages_collapsed++; 2458 result = SCAN_SUCCEED; 2459 out_up_write: 2460 up_write(&mm->mmap_sem); 2461 trace_mm_collapse_huge_page(mm, isolated, result); 2462 return; 2463 2464 out_nolock: 2465 trace_mm_collapse_huge_page(mm, isolated, result); 2466 return; 2467 out: 2468 mem_cgroup_cancel_charge(new_page, memcg, true); 2469 goto out_up_write; 2470 } 2471 2472 static int khugepaged_scan_pmd(struct mm_struct *mm, 2473 struct vm_area_struct *vma, 2474 unsigned long address, 2475 struct page **hpage) 2476 { 2477 pmd_t *pmd; 2478 pte_t *pte, *_pte; 2479 int ret = 0, none_or_zero = 0, result = 0; 2480 struct page *page = NULL; 2481 unsigned long _address; 2482 spinlock_t *ptl; 2483 int node = NUMA_NO_NODE; 2484 bool writable = false, referenced = false; 2485 2486 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 2487 2488 pmd = mm_find_pmd(mm, address); 2489 if (!pmd) { 2490 result = SCAN_PMD_NULL; 2491 goto out; 2492 } 2493 2494 memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); 2495 pte = pte_offset_map_lock(mm, pmd, address, &ptl); 2496 for (_address = address, _pte = pte; _pte < pte+HPAGE_PMD_NR; 2497 _pte++, _address += PAGE_SIZE) { 2498 pte_t pteval = *_pte; 2499 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { 2500 if (!userfaultfd_armed(vma) && 2501 ++none_or_zero <= khugepaged_max_ptes_none) { 2502 continue; 2503 } else { 2504 result = SCAN_EXCEED_NONE_PTE; 2505 goto out_unmap; 2506 } 2507 } 2508 if (!pte_present(pteval)) { 2509 result = SCAN_PTE_NON_PRESENT; 2510 goto out_unmap; 2511 } 2512 if (pte_write(pteval)) 2513 writable = true; 2514 2515 page = vm_normal_page(vma, _address, pteval); 2516 if (unlikely(!page)) { 2517 result = SCAN_PAGE_NULL; 2518 goto out_unmap; 2519 } 2520 2521 /* TODO: teach khugepaged to collapse THP mapped with pte */ 2522 if (PageCompound(page)) { 2523 result = SCAN_PAGE_COMPOUND; 2524 goto out_unmap; 2525 } 2526 2527 /* 2528 * Record which node the original page is from and save this 2529 * information to khugepaged_node_load[]. 2530 * Khupaged will allocate hugepage from the node has the max 2531 * hit record. 2532 */ 2533 node = page_to_nid(page); 2534 if (khugepaged_scan_abort(node)) { 2535 result = SCAN_SCAN_ABORT; 2536 goto out_unmap; 2537 } 2538 khugepaged_node_load[node]++; 2539 if (!PageLRU(page)) { 2540 result = SCAN_SCAN_ABORT; 2541 goto out_unmap; 2542 } 2543 if (PageLocked(page)) { 2544 result = SCAN_PAGE_LOCK; 2545 goto out_unmap; 2546 } 2547 if (!PageAnon(page)) { 2548 result = SCAN_PAGE_ANON; 2549 goto out_unmap; 2550 } 2551 2552 /* 2553 * cannot use mapcount: can't collapse if there's a gup pin. 2554 * The page must only be referenced by the scanned process 2555 * and page swap cache. 2556 */ 2557 if (page_count(page) != 1 + !!PageSwapCache(page)) { 2558 result = SCAN_PAGE_COUNT; 2559 goto out_unmap; 2560 } 2561 if (pte_young(pteval) || 2562 page_is_young(page) || PageReferenced(page) || 2563 mmu_notifier_test_young(vma->vm_mm, address)) 2564 referenced = true; 2565 } 2566 if (writable) { 2567 if (referenced) { 2568 result = SCAN_SUCCEED; 2569 ret = 1; 2570 } else { 2571 result = SCAN_NO_REFERENCED_PAGE; 2572 } 2573 } else { 2574 result = SCAN_PAGE_RO; 2575 } 2576 out_unmap: 2577 pte_unmap_unlock(pte, ptl); 2578 if (ret) { 2579 node = khugepaged_find_target_node(); 2580 /* collapse_huge_page will return with the mmap_sem released */ 2581 collapse_huge_page(mm, address, hpage, vma, node); 2582 } 2583 out: 2584 trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced, 2585 none_or_zero, result); 2586 return ret; 2587 } 2588 2589 static void collect_mm_slot(struct mm_slot *mm_slot) 2590 { 2591 struct mm_struct *mm = mm_slot->mm; 2592 2593 VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock)); 2594 2595 if (khugepaged_test_exit(mm)) { 2596 /* free mm_slot */ 2597 hash_del(&mm_slot->hash); 2598 list_del(&mm_slot->mm_node); 2599 2600 /* 2601 * Not strictly needed because the mm exited already. 2602 * 2603 * clear_bit(MMF_VM_HUGEPAGE, &mm->flags); 2604 */ 2605 2606 /* khugepaged_mm_lock actually not necessary for the below */ 2607 free_mm_slot(mm_slot); 2608 mmdrop(mm); 2609 } 2610 } 2611 2612 static unsigned int khugepaged_scan_mm_slot(unsigned int pages, 2613 struct page **hpage) 2614 __releases(&khugepaged_mm_lock) 2615 __acquires(&khugepaged_mm_lock) 2616 { 2617 struct mm_slot *mm_slot; 2618 struct mm_struct *mm; 2619 struct vm_area_struct *vma; 2620 int progress = 0; 2621 2622 VM_BUG_ON(!pages); 2623 VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock)); 2624 2625 if (khugepaged_scan.mm_slot) 2626 mm_slot = khugepaged_scan.mm_slot; 2627 else { 2628 mm_slot = list_entry(khugepaged_scan.mm_head.next, 2629 struct mm_slot, mm_node); 2630 khugepaged_scan.address = 0; 2631 khugepaged_scan.mm_slot = mm_slot; 2632 } 2633 spin_unlock(&khugepaged_mm_lock); 2634 2635 mm = mm_slot->mm; 2636 down_read(&mm->mmap_sem); 2637 if (unlikely(khugepaged_test_exit(mm))) 2638 vma = NULL; 2639 else 2640 vma = find_vma(mm, khugepaged_scan.address); 2641 2642 progress++; 2643 for (; vma; vma = vma->vm_next) { 2644 unsigned long hstart, hend; 2645 2646 cond_resched(); 2647 if (unlikely(khugepaged_test_exit(mm))) { 2648 progress++; 2649 break; 2650 } 2651 if (!hugepage_vma_check(vma)) { 2652 skip: 2653 progress++; 2654 continue; 2655 } 2656 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; 2657 hend = vma->vm_end & HPAGE_PMD_MASK; 2658 if (hstart >= hend) 2659 goto skip; 2660 if (khugepaged_scan.address > hend) 2661 goto skip; 2662 if (khugepaged_scan.address < hstart) 2663 khugepaged_scan.address = hstart; 2664 VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK); 2665 2666 while (khugepaged_scan.address < hend) { 2667 int ret; 2668 cond_resched(); 2669 if (unlikely(khugepaged_test_exit(mm))) 2670 goto breakouterloop; 2671 2672 VM_BUG_ON(khugepaged_scan.address < hstart || 2673 khugepaged_scan.address + HPAGE_PMD_SIZE > 2674 hend); 2675 ret = khugepaged_scan_pmd(mm, vma, 2676 khugepaged_scan.address, 2677 hpage); 2678 /* move to next address */ 2679 khugepaged_scan.address += HPAGE_PMD_SIZE; 2680 progress += HPAGE_PMD_NR; 2681 if (ret) 2682 /* we released mmap_sem so break loop */ 2683 goto breakouterloop_mmap_sem; 2684 if (progress >= pages) 2685 goto breakouterloop; 2686 } 2687 } 2688 breakouterloop: 2689 up_read(&mm->mmap_sem); /* exit_mmap will destroy ptes after this */ 2690 breakouterloop_mmap_sem: 2691 2692 spin_lock(&khugepaged_mm_lock); 2693 VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot); 2694 /* 2695 * Release the current mm_slot if this mm is about to die, or 2696 * if we scanned all vmas of this mm. 2697 */ 2698 if (khugepaged_test_exit(mm) || !vma) { 2699 /* 2700 * Make sure that if mm_users is reaching zero while 2701 * khugepaged runs here, khugepaged_exit will find 2702 * mm_slot not pointing to the exiting mm. 2703 */ 2704 if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) { 2705 khugepaged_scan.mm_slot = list_entry( 2706 mm_slot->mm_node.next, 2707 struct mm_slot, mm_node); 2708 khugepaged_scan.address = 0; 2709 } else { 2710 khugepaged_scan.mm_slot = NULL; 2711 khugepaged_full_scans++; 2712 } 2713 2714 collect_mm_slot(mm_slot); 2715 } 2716 2717 return progress; 2718 } 2719 2720 static int khugepaged_has_work(void) 2721 { 2722 return !list_empty(&khugepaged_scan.mm_head) && 2723 khugepaged_enabled(); 2724 } 2725 2726 static int khugepaged_wait_event(void) 2727 { 2728 return !list_empty(&khugepaged_scan.mm_head) || 2729 kthread_should_stop(); 2730 } 2731 2732 static void khugepaged_do_scan(void) 2733 { 2734 struct page *hpage = NULL; 2735 unsigned int progress = 0, pass_through_head = 0; 2736 unsigned int pages = khugepaged_pages_to_scan; 2737 bool wait = true; 2738 2739 barrier(); /* write khugepaged_pages_to_scan to local stack */ 2740 2741 while (progress < pages) { 2742 if (!khugepaged_prealloc_page(&hpage, &wait)) 2743 break; 2744 2745 cond_resched(); 2746 2747 if (unlikely(kthread_should_stop() || try_to_freeze())) 2748 break; 2749 2750 spin_lock(&khugepaged_mm_lock); 2751 if (!khugepaged_scan.mm_slot) 2752 pass_through_head++; 2753 if (khugepaged_has_work() && 2754 pass_through_head < 2) 2755 progress += khugepaged_scan_mm_slot(pages - progress, 2756 &hpage); 2757 else 2758 progress = pages; 2759 spin_unlock(&khugepaged_mm_lock); 2760 } 2761 2762 if (!IS_ERR_OR_NULL(hpage)) 2763 put_page(hpage); 2764 } 2765 2766 static void khugepaged_wait_work(void) 2767 { 2768 if (khugepaged_has_work()) { 2769 if (!khugepaged_scan_sleep_millisecs) 2770 return; 2771 2772 wait_event_freezable_timeout(khugepaged_wait, 2773 kthread_should_stop(), 2774 msecs_to_jiffies(khugepaged_scan_sleep_millisecs)); 2775 return; 2776 } 2777 2778 if (khugepaged_enabled()) 2779 wait_event_freezable(khugepaged_wait, khugepaged_wait_event()); 2780 } 2781 2782 static int khugepaged(void *none) 2783 { 2784 struct mm_slot *mm_slot; 2785 2786 set_freezable(); 2787 set_user_nice(current, MAX_NICE); 2788 2789 while (!kthread_should_stop()) { 2790 khugepaged_do_scan(); 2791 khugepaged_wait_work(); 2792 } 2793 2794 spin_lock(&khugepaged_mm_lock); 2795 mm_slot = khugepaged_scan.mm_slot; 2796 khugepaged_scan.mm_slot = NULL; 2797 if (mm_slot) 2798 collect_mm_slot(mm_slot); 2799 spin_unlock(&khugepaged_mm_lock); 2800 return 0; 2801 } 2802 2803 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 2804 unsigned long haddr, pmd_t *pmd) 2805 { 2806 struct mm_struct *mm = vma->vm_mm; 2807 pgtable_t pgtable; 2808 pmd_t _pmd; 2809 int i; 2810 2811 /* leave pmd empty until pte is filled */ 2812 pmdp_huge_clear_flush_notify(vma, haddr, pmd); 2813 2814 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2815 pmd_populate(mm, &_pmd, pgtable); 2816 2817 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 2818 pte_t *pte, entry; 2819 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot); 2820 entry = pte_mkspecial(entry); 2821 pte = pte_offset_map(&_pmd, haddr); 2822 VM_BUG_ON(!pte_none(*pte)); 2823 set_pte_at(mm, haddr, pte, entry); 2824 pte_unmap(pte); 2825 } 2826 smp_wmb(); /* make pte visible before pmd */ 2827 pmd_populate(mm, pmd, pgtable); 2828 put_huge_zero_page(); 2829 } 2830 2831 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 2832 unsigned long haddr, bool freeze) 2833 { 2834 struct mm_struct *mm = vma->vm_mm; 2835 struct page *page; 2836 pgtable_t pgtable; 2837 pmd_t _pmd; 2838 bool young, write, dirty; 2839 unsigned long addr; 2840 int i; 2841 2842 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 2843 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2844 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 2845 VM_BUG_ON(!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd)); 2846 2847 count_vm_event(THP_SPLIT_PMD); 2848 2849 if (vma_is_dax(vma)) { 2850 pmd_t _pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd); 2851 if (is_huge_zero_pmd(_pmd)) 2852 put_huge_zero_page(); 2853 return; 2854 } else if (is_huge_zero_pmd(*pmd)) { 2855 return __split_huge_zero_page_pmd(vma, haddr, pmd); 2856 } 2857 2858 page = pmd_page(*pmd); 2859 VM_BUG_ON_PAGE(!page_count(page), page); 2860 atomic_add(HPAGE_PMD_NR - 1, &page->_count); 2861 write = pmd_write(*pmd); 2862 young = pmd_young(*pmd); 2863 dirty = pmd_dirty(*pmd); 2864 2865 pmdp_huge_split_prepare(vma, haddr, pmd); 2866 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2867 pmd_populate(mm, &_pmd, pgtable); 2868 2869 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2870 pte_t entry, *pte; 2871 /* 2872 * Note that NUMA hinting access restrictions are not 2873 * transferred to avoid any possibility of altering 2874 * permissions across VMAs. 2875 */ 2876 if (freeze) { 2877 swp_entry_t swp_entry; 2878 swp_entry = make_migration_entry(page + i, write); 2879 entry = swp_entry_to_pte(swp_entry); 2880 } else { 2881 entry = mk_pte(page + i, vma->vm_page_prot); 2882 entry = maybe_mkwrite(entry, vma); 2883 if (!write) 2884 entry = pte_wrprotect(entry); 2885 if (!young) 2886 entry = pte_mkold(entry); 2887 } 2888 if (dirty) 2889 SetPageDirty(page + i); 2890 pte = pte_offset_map(&_pmd, addr); 2891 BUG_ON(!pte_none(*pte)); 2892 set_pte_at(mm, addr, pte, entry); 2893 atomic_inc(&page[i]._mapcount); 2894 pte_unmap(pte); 2895 } 2896 2897 /* 2898 * Set PG_double_map before dropping compound_mapcount to avoid 2899 * false-negative page_mapped(). 2900 */ 2901 if (compound_mapcount(page) > 1 && !TestSetPageDoubleMap(page)) { 2902 for (i = 0; i < HPAGE_PMD_NR; i++) 2903 atomic_inc(&page[i]._mapcount); 2904 } 2905 2906 if (atomic_add_negative(-1, compound_mapcount_ptr(page))) { 2907 /* Last compound_mapcount is gone. */ 2908 __dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES); 2909 if (TestClearPageDoubleMap(page)) { 2910 /* No need in mapcount reference anymore */ 2911 for (i = 0; i < HPAGE_PMD_NR; i++) 2912 atomic_dec(&page[i]._mapcount); 2913 } 2914 } 2915 2916 smp_wmb(); /* make pte visible before pmd */ 2917 /* 2918 * Up to this point the pmd is present and huge and userland has the 2919 * whole access to the hugepage during the split (which happens in 2920 * place). If we overwrite the pmd with the not-huge version pointing 2921 * to the pte here (which of course we could if all CPUs were bug 2922 * free), userland could trigger a small page size TLB miss on the 2923 * small sized TLB while the hugepage TLB entry is still established in 2924 * the huge TLB. Some CPU doesn't like that. 2925 * See http://support.amd.com/us/Processor_TechDocs/41322.pdf, Erratum 2926 * 383 on page 93. Intel should be safe but is also warns that it's 2927 * only safe if the permission and cache attributes of the two entries 2928 * loaded in the two TLB is identical (which should be the case here). 2929 * But it is generally safer to never allow small and huge TLB entries 2930 * for the same virtual address to be loaded simultaneously. So instead 2931 * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the 2932 * current pmd notpresent (atomically because here the pmd_trans_huge 2933 * and pmd_trans_splitting must remain set at all times on the pmd 2934 * until the split is complete for this pmd), then we flush the SMP TLB 2935 * and finally we write the non-huge version of the pmd entry with 2936 * pmd_populate. 2937 */ 2938 pmdp_invalidate(vma, haddr, pmd); 2939 pmd_populate(mm, pmd, pgtable); 2940 2941 if (freeze) { 2942 for (i = 0; i < HPAGE_PMD_NR; i++) { 2943 page_remove_rmap(page + i, false); 2944 put_page(page + i); 2945 } 2946 } 2947 } 2948 2949 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 2950 unsigned long address) 2951 { 2952 spinlock_t *ptl; 2953 struct mm_struct *mm = vma->vm_mm; 2954 struct page *page = NULL; 2955 unsigned long haddr = address & HPAGE_PMD_MASK; 2956 2957 mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE); 2958 ptl = pmd_lock(mm, pmd); 2959 if (pmd_trans_huge(*pmd)) { 2960 page = pmd_page(*pmd); 2961 if (PageMlocked(page)) 2962 get_page(page); 2963 else 2964 page = NULL; 2965 } else if (!pmd_devmap(*pmd)) 2966 goto out; 2967 __split_huge_pmd_locked(vma, pmd, haddr, false); 2968 out: 2969 spin_unlock(ptl); 2970 mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE); 2971 if (page) { 2972 lock_page(page); 2973 munlock_vma_page(page); 2974 unlock_page(page); 2975 put_page(page); 2976 } 2977 } 2978 2979 static void split_huge_pmd_address(struct vm_area_struct *vma, 2980 unsigned long address) 2981 { 2982 pgd_t *pgd; 2983 pud_t *pud; 2984 pmd_t *pmd; 2985 2986 VM_BUG_ON(!(address & ~HPAGE_PMD_MASK)); 2987 2988 pgd = pgd_offset(vma->vm_mm, address); 2989 if (!pgd_present(*pgd)) 2990 return; 2991 2992 pud = pud_offset(pgd, address); 2993 if (!pud_present(*pud)) 2994 return; 2995 2996 pmd = pmd_offset(pud, address); 2997 if (!pmd_present(*pmd) || (!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd))) 2998 return; 2999 /* 3000 * Caller holds the mmap_sem write mode, so a huge pmd cannot 3001 * materialize from under us. 3002 */ 3003 split_huge_pmd(vma, pmd, address); 3004 } 3005 3006 void vma_adjust_trans_huge(struct vm_area_struct *vma, 3007 unsigned long start, 3008 unsigned long end, 3009 long adjust_next) 3010 { 3011 /* 3012 * If the new start address isn't hpage aligned and it could 3013 * previously contain an hugepage: check if we need to split 3014 * an huge pmd. 3015 */ 3016 if (start & ~HPAGE_PMD_MASK && 3017 (start & HPAGE_PMD_MASK) >= vma->vm_start && 3018 (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) 3019 split_huge_pmd_address(vma, start); 3020 3021 /* 3022 * If the new end address isn't hpage aligned and it could 3023 * previously contain an hugepage: check if we need to split 3024 * an huge pmd. 3025 */ 3026 if (end & ~HPAGE_PMD_MASK && 3027 (end & HPAGE_PMD_MASK) >= vma->vm_start && 3028 (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) 3029 split_huge_pmd_address(vma, end); 3030 3031 /* 3032 * If we're also updating the vma->vm_next->vm_start, if the new 3033 * vm_next->vm_start isn't page aligned and it could previously 3034 * contain an hugepage: check if we need to split an huge pmd. 3035 */ 3036 if (adjust_next > 0) { 3037 struct vm_area_struct *next = vma->vm_next; 3038 unsigned long nstart = next->vm_start; 3039 nstart += adjust_next << PAGE_SHIFT; 3040 if (nstart & ~HPAGE_PMD_MASK && 3041 (nstart & HPAGE_PMD_MASK) >= next->vm_start && 3042 (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end) 3043 split_huge_pmd_address(next, nstart); 3044 } 3045 } 3046 3047 static void freeze_page_vma(struct vm_area_struct *vma, struct page *page, 3048 unsigned long address) 3049 { 3050 unsigned long haddr = address & HPAGE_PMD_MASK; 3051 spinlock_t *ptl; 3052 pgd_t *pgd; 3053 pud_t *pud; 3054 pmd_t *pmd; 3055 pte_t *pte; 3056 int i, nr = HPAGE_PMD_NR; 3057 3058 /* Skip pages which doesn't belong to the VMA */ 3059 if (address < vma->vm_start) { 3060 int off = (vma->vm_start - address) >> PAGE_SHIFT; 3061 page += off; 3062 nr -= off; 3063 address = vma->vm_start; 3064 } 3065 3066 pgd = pgd_offset(vma->vm_mm, address); 3067 if (!pgd_present(*pgd)) 3068 return; 3069 pud = pud_offset(pgd, address); 3070 if (!pud_present(*pud)) 3071 return; 3072 pmd = pmd_offset(pud, address); 3073 ptl = pmd_lock(vma->vm_mm, pmd); 3074 if (!pmd_present(*pmd)) { 3075 spin_unlock(ptl); 3076 return; 3077 } 3078 if (pmd_trans_huge(*pmd)) { 3079 if (page == pmd_page(*pmd)) 3080 __split_huge_pmd_locked(vma, pmd, haddr, true); 3081 spin_unlock(ptl); 3082 return; 3083 } 3084 spin_unlock(ptl); 3085 3086 pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl); 3087 for (i = 0; i < nr; i++, address += PAGE_SIZE, page++, pte++) { 3088 pte_t entry, swp_pte; 3089 swp_entry_t swp_entry; 3090 3091 /* 3092 * We've just crossed page table boundary: need to map next one. 3093 * It can happen if THP was mremaped to non PMD-aligned address. 3094 */ 3095 if (unlikely(address == haddr + HPAGE_PMD_SIZE)) { 3096 pte_unmap_unlock(pte - 1, ptl); 3097 pmd = mm_find_pmd(vma->vm_mm, address); 3098 if (!pmd) 3099 return; 3100 pte = pte_offset_map_lock(vma->vm_mm, pmd, 3101 address, &ptl); 3102 } 3103 3104 if (!pte_present(*pte)) 3105 continue; 3106 if (page_to_pfn(page) != pte_pfn(*pte)) 3107 continue; 3108 flush_cache_page(vma, address, page_to_pfn(page)); 3109 entry = ptep_clear_flush(vma, address, pte); 3110 if (pte_dirty(entry)) 3111 SetPageDirty(page); 3112 swp_entry = make_migration_entry(page, pte_write(entry)); 3113 swp_pte = swp_entry_to_pte(swp_entry); 3114 if (pte_soft_dirty(entry)) 3115 swp_pte = pte_swp_mksoft_dirty(swp_pte); 3116 set_pte_at(vma->vm_mm, address, pte, swp_pte); 3117 page_remove_rmap(page, false); 3118 put_page(page); 3119 } 3120 pte_unmap_unlock(pte - 1, ptl); 3121 } 3122 3123 static void freeze_page(struct anon_vma *anon_vma, struct page *page) 3124 { 3125 struct anon_vma_chain *avc; 3126 pgoff_t pgoff = page_to_pgoff(page); 3127 3128 VM_BUG_ON_PAGE(!PageHead(page), page); 3129 3130 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, 3131 pgoff + HPAGE_PMD_NR - 1) { 3132 unsigned long address = __vma_address(page, avc->vma); 3133 3134 mmu_notifier_invalidate_range_start(avc->vma->vm_mm, 3135 address, address + HPAGE_PMD_SIZE); 3136 freeze_page_vma(avc->vma, page, address); 3137 mmu_notifier_invalidate_range_end(avc->vma->vm_mm, 3138 address, address + HPAGE_PMD_SIZE); 3139 } 3140 } 3141 3142 static void unfreeze_page_vma(struct vm_area_struct *vma, struct page *page, 3143 unsigned long address) 3144 { 3145 spinlock_t *ptl; 3146 pmd_t *pmd; 3147 pte_t *pte, entry; 3148 swp_entry_t swp_entry; 3149 unsigned long haddr = address & HPAGE_PMD_MASK; 3150 int i, nr = HPAGE_PMD_NR; 3151 3152 /* Skip pages which doesn't belong to the VMA */ 3153 if (address < vma->vm_start) { 3154 int off = (vma->vm_start - address) >> PAGE_SHIFT; 3155 page += off; 3156 nr -= off; 3157 address = vma->vm_start; 3158 } 3159 3160 pmd = mm_find_pmd(vma->vm_mm, address); 3161 if (!pmd) 3162 return; 3163 3164 pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl); 3165 for (i = 0; i < nr; i++, address += PAGE_SIZE, page++, pte++) { 3166 /* 3167 * We've just crossed page table boundary: need to map next one. 3168 * It can happen if THP was mremaped to non-PMD aligned address. 3169 */ 3170 if (unlikely(address == haddr + HPAGE_PMD_SIZE)) { 3171 pte_unmap_unlock(pte - 1, ptl); 3172 pmd = mm_find_pmd(vma->vm_mm, address); 3173 if (!pmd) 3174 return; 3175 pte = pte_offset_map_lock(vma->vm_mm, pmd, 3176 address, &ptl); 3177 } 3178 3179 if (!is_swap_pte(*pte)) 3180 continue; 3181 3182 swp_entry = pte_to_swp_entry(*pte); 3183 if (!is_migration_entry(swp_entry)) 3184 continue; 3185 if (migration_entry_to_page(swp_entry) != page) 3186 continue; 3187 3188 get_page(page); 3189 page_add_anon_rmap(page, vma, address, false); 3190 3191 entry = pte_mkold(mk_pte(page, vma->vm_page_prot)); 3192 if (PageDirty(page)) 3193 entry = pte_mkdirty(entry); 3194 if (is_write_migration_entry(swp_entry)) 3195 entry = maybe_mkwrite(entry, vma); 3196 3197 flush_dcache_page(page); 3198 set_pte_at(vma->vm_mm, address, pte, entry); 3199 3200 /* No need to invalidate - it was non-present before */ 3201 update_mmu_cache(vma, address, pte); 3202 } 3203 pte_unmap_unlock(pte - 1, ptl); 3204 } 3205 3206 static void unfreeze_page(struct anon_vma *anon_vma, struct page *page) 3207 { 3208 struct anon_vma_chain *avc; 3209 pgoff_t pgoff = page_to_pgoff(page); 3210 3211 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, 3212 pgoff, pgoff + HPAGE_PMD_NR - 1) { 3213 unsigned long address = __vma_address(page, avc->vma); 3214 3215 mmu_notifier_invalidate_range_start(avc->vma->vm_mm, 3216 address, address + HPAGE_PMD_SIZE); 3217 unfreeze_page_vma(avc->vma, page, address); 3218 mmu_notifier_invalidate_range_end(avc->vma->vm_mm, 3219 address, address + HPAGE_PMD_SIZE); 3220 } 3221 } 3222 3223 static void __split_huge_page_tail(struct page *head, int tail, 3224 struct lruvec *lruvec, struct list_head *list) 3225 { 3226 struct page *page_tail = head + tail; 3227 3228 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail); 3229 VM_BUG_ON_PAGE(atomic_read(&page_tail->_count) != 0, page_tail); 3230 3231 /* 3232 * tail_page->_count is zero and not changing from under us. But 3233 * get_page_unless_zero() may be running from under us on the 3234 * tail_page. If we used atomic_set() below instead of atomic_inc(), we 3235 * would then run atomic_set() concurrently with 3236 * get_page_unless_zero(), and atomic_set() is implemented in C not 3237 * using locked ops. spin_unlock on x86 sometime uses locked ops 3238 * because of PPro errata 66, 92, so unless somebody can guarantee 3239 * atomic_set() here would be safe on all archs (and not only on x86), 3240 * it's safer to use atomic_inc(). 3241 */ 3242 atomic_inc(&page_tail->_count); 3243 3244 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 3245 page_tail->flags |= (head->flags & 3246 ((1L << PG_referenced) | 3247 (1L << PG_swapbacked) | 3248 (1L << PG_mlocked) | 3249 (1L << PG_uptodate) | 3250 (1L << PG_active) | 3251 (1L << PG_locked) | 3252 (1L << PG_unevictable) | 3253 (1L << PG_dirty))); 3254 3255 /* 3256 * After clearing PageTail the gup refcount can be released. 3257 * Page flags also must be visible before we make the page non-compound. 3258 */ 3259 smp_wmb(); 3260 3261 clear_compound_head(page_tail); 3262 3263 if (page_is_young(head)) 3264 set_page_young(page_tail); 3265 if (page_is_idle(head)) 3266 set_page_idle(page_tail); 3267 3268 /* ->mapping in first tail page is compound_mapcount */ 3269 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, 3270 page_tail); 3271 page_tail->mapping = head->mapping; 3272 3273 page_tail->index = head->index + tail; 3274 page_cpupid_xchg_last(page_tail, page_cpupid_last(head)); 3275 lru_add_page_tail(head, page_tail, lruvec, list); 3276 } 3277 3278 static void __split_huge_page(struct page *page, struct list_head *list) 3279 { 3280 struct page *head = compound_head(page); 3281 struct zone *zone = page_zone(head); 3282 struct lruvec *lruvec; 3283 int i; 3284 3285 /* prevent PageLRU to go away from under us, and freeze lru stats */ 3286 spin_lock_irq(&zone->lru_lock); 3287 lruvec = mem_cgroup_page_lruvec(head, zone); 3288 3289 /* complete memcg works before add pages to LRU */ 3290 mem_cgroup_split_huge_fixup(head); 3291 3292 for (i = HPAGE_PMD_NR - 1; i >= 1; i--) 3293 __split_huge_page_tail(head, i, lruvec, list); 3294 3295 ClearPageCompound(head); 3296 spin_unlock_irq(&zone->lru_lock); 3297 3298 unfreeze_page(page_anon_vma(head), head); 3299 3300 for (i = 0; i < HPAGE_PMD_NR; i++) { 3301 struct page *subpage = head + i; 3302 if (subpage == page) 3303 continue; 3304 unlock_page(subpage); 3305 3306 /* 3307 * Subpages may be freed if there wasn't any mapping 3308 * like if add_to_swap() is running on a lru page that 3309 * had its mapping zapped. And freeing these pages 3310 * requires taking the lru_lock so we do the put_page 3311 * of the tail pages after the split is complete. 3312 */ 3313 put_page(subpage); 3314 } 3315 } 3316 3317 int total_mapcount(struct page *page) 3318 { 3319 int i, ret; 3320 3321 VM_BUG_ON_PAGE(PageTail(page), page); 3322 3323 if (likely(!PageCompound(page))) 3324 return atomic_read(&page->_mapcount) + 1; 3325 3326 ret = compound_mapcount(page); 3327 if (PageHuge(page)) 3328 return ret; 3329 for (i = 0; i < HPAGE_PMD_NR; i++) 3330 ret += atomic_read(&page[i]._mapcount) + 1; 3331 if (PageDoubleMap(page)) 3332 ret -= HPAGE_PMD_NR; 3333 return ret; 3334 } 3335 3336 /* 3337 * This function splits huge page into normal pages. @page can point to any 3338 * subpage of huge page to split. Split doesn't change the position of @page. 3339 * 3340 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY. 3341 * The huge page must be locked. 3342 * 3343 * If @list is null, tail pages will be added to LRU list, otherwise, to @list. 3344 * 3345 * Both head page and tail pages will inherit mapping, flags, and so on from 3346 * the hugepage. 3347 * 3348 * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if 3349 * they are not mapped. 3350 * 3351 * Returns 0 if the hugepage is split successfully. 3352 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under 3353 * us. 3354 */ 3355 int split_huge_page_to_list(struct page *page, struct list_head *list) 3356 { 3357 struct page *head = compound_head(page); 3358 struct pglist_data *pgdata = NODE_DATA(page_to_nid(head)); 3359 struct anon_vma *anon_vma; 3360 int count, mapcount, ret; 3361 bool mlocked; 3362 unsigned long flags; 3363 3364 VM_BUG_ON_PAGE(is_huge_zero_page(page), page); 3365 VM_BUG_ON_PAGE(!PageAnon(page), page); 3366 VM_BUG_ON_PAGE(!PageLocked(page), page); 3367 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 3368 VM_BUG_ON_PAGE(!PageCompound(page), page); 3369 3370 /* 3371 * The caller does not necessarily hold an mmap_sem that would prevent 3372 * the anon_vma disappearing so we first we take a reference to it 3373 * and then lock the anon_vma for write. This is similar to 3374 * page_lock_anon_vma_read except the write lock is taken to serialise 3375 * against parallel split or collapse operations. 3376 */ 3377 anon_vma = page_get_anon_vma(head); 3378 if (!anon_vma) { 3379 ret = -EBUSY; 3380 goto out; 3381 } 3382 anon_vma_lock_write(anon_vma); 3383 3384 /* 3385 * Racy check if we can split the page, before freeze_page() will 3386 * split PMDs 3387 */ 3388 if (total_mapcount(head) != page_count(head) - 1) { 3389 ret = -EBUSY; 3390 goto out_unlock; 3391 } 3392 3393 mlocked = PageMlocked(page); 3394 freeze_page(anon_vma, head); 3395 VM_BUG_ON_PAGE(compound_mapcount(head), head); 3396 3397 /* Make sure the page is not on per-CPU pagevec as it takes pin */ 3398 if (mlocked) 3399 lru_add_drain(); 3400 3401 /* Prevent deferred_split_scan() touching ->_count */ 3402 spin_lock_irqsave(&pgdata->split_queue_lock, flags); 3403 count = page_count(head); 3404 mapcount = total_mapcount(head); 3405 if (!mapcount && count == 1) { 3406 if (!list_empty(page_deferred_list(head))) { 3407 pgdata->split_queue_len--; 3408 list_del(page_deferred_list(head)); 3409 } 3410 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 3411 __split_huge_page(page, list); 3412 ret = 0; 3413 } else if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount) { 3414 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 3415 pr_alert("total_mapcount: %u, page_count(): %u\n", 3416 mapcount, count); 3417 if (PageTail(page)) 3418 dump_page(head, NULL); 3419 dump_page(page, "total_mapcount(head) > 0"); 3420 BUG(); 3421 } else { 3422 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 3423 unfreeze_page(anon_vma, head); 3424 ret = -EBUSY; 3425 } 3426 3427 out_unlock: 3428 anon_vma_unlock_write(anon_vma); 3429 put_anon_vma(anon_vma); 3430 out: 3431 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); 3432 return ret; 3433 } 3434 3435 void free_transhuge_page(struct page *page) 3436 { 3437 struct pglist_data *pgdata = NODE_DATA(page_to_nid(page)); 3438 unsigned long flags; 3439 3440 spin_lock_irqsave(&pgdata->split_queue_lock, flags); 3441 if (!list_empty(page_deferred_list(page))) { 3442 pgdata->split_queue_len--; 3443 list_del(page_deferred_list(page)); 3444 } 3445 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 3446 free_compound_page(page); 3447 } 3448 3449 void deferred_split_huge_page(struct page *page) 3450 { 3451 struct pglist_data *pgdata = NODE_DATA(page_to_nid(page)); 3452 unsigned long flags; 3453 3454 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 3455 3456 spin_lock_irqsave(&pgdata->split_queue_lock, flags); 3457 if (list_empty(page_deferred_list(page))) { 3458 list_add_tail(page_deferred_list(page), &pgdata->split_queue); 3459 pgdata->split_queue_len++; 3460 } 3461 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 3462 } 3463 3464 static unsigned long deferred_split_count(struct shrinker *shrink, 3465 struct shrink_control *sc) 3466 { 3467 struct pglist_data *pgdata = NODE_DATA(sc->nid); 3468 return ACCESS_ONCE(pgdata->split_queue_len); 3469 } 3470 3471 static unsigned long deferred_split_scan(struct shrinker *shrink, 3472 struct shrink_control *sc) 3473 { 3474 struct pglist_data *pgdata = NODE_DATA(sc->nid); 3475 unsigned long flags; 3476 LIST_HEAD(list), *pos, *next; 3477 struct page *page; 3478 int split = 0; 3479 3480 spin_lock_irqsave(&pgdata->split_queue_lock, flags); 3481 /* Take pin on all head pages to avoid freeing them under us */ 3482 list_for_each_safe(pos, next, &pgdata->split_queue) { 3483 page = list_entry((void *)pos, struct page, mapping); 3484 page = compound_head(page); 3485 if (get_page_unless_zero(page)) { 3486 list_move(page_deferred_list(page), &list); 3487 } else { 3488 /* We lost race with put_compound_page() */ 3489 list_del_init(page_deferred_list(page)); 3490 pgdata->split_queue_len--; 3491 } 3492 if (!--sc->nr_to_scan) 3493 break; 3494 } 3495 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 3496 3497 list_for_each_safe(pos, next, &list) { 3498 page = list_entry((void *)pos, struct page, mapping); 3499 lock_page(page); 3500 /* split_huge_page() removes page from list on success */ 3501 if (!split_huge_page(page)) 3502 split++; 3503 unlock_page(page); 3504 put_page(page); 3505 } 3506 3507 spin_lock_irqsave(&pgdata->split_queue_lock, flags); 3508 list_splice_tail(&list, &pgdata->split_queue); 3509 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 3510 3511 /* 3512 * Stop shrinker if we didn't split any page, but the queue is empty. 3513 * This can happen if pages were freed under us. 3514 */ 3515 if (!split && list_empty(&pgdata->split_queue)) 3516 return SHRINK_STOP; 3517 return split; 3518 } 3519 3520 static struct shrinker deferred_split_shrinker = { 3521 .count_objects = deferred_split_count, 3522 .scan_objects = deferred_split_scan, 3523 .seeks = DEFAULT_SEEKS, 3524 .flags = SHRINKER_NUMA_AWARE, 3525 }; 3526 3527 #ifdef CONFIG_DEBUG_FS 3528 static int split_huge_pages_set(void *data, u64 val) 3529 { 3530 struct zone *zone; 3531 struct page *page; 3532 unsigned long pfn, max_zone_pfn; 3533 unsigned long total = 0, split = 0; 3534 3535 if (val != 1) 3536 return -EINVAL; 3537 3538 for_each_populated_zone(zone) { 3539 max_zone_pfn = zone_end_pfn(zone); 3540 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 3541 if (!pfn_valid(pfn)) 3542 continue; 3543 3544 page = pfn_to_page(pfn); 3545 if (!get_page_unless_zero(page)) 3546 continue; 3547 3548 if (zone != page_zone(page)) 3549 goto next; 3550 3551 if (!PageHead(page) || !PageAnon(page) || 3552 PageHuge(page)) 3553 goto next; 3554 3555 total++; 3556 lock_page(page); 3557 if (!split_huge_page(page)) 3558 split++; 3559 unlock_page(page); 3560 next: 3561 put_page(page); 3562 } 3563 } 3564 3565 pr_info("%lu of %lu THP split", split, total); 3566 3567 return 0; 3568 } 3569 DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set, 3570 "%llu\n"); 3571 3572 static int __init split_huge_pages_debugfs(void) 3573 { 3574 void *ret; 3575 3576 ret = debugfs_create_file("split_huge_pages", 0644, NULL, NULL, 3577 &split_huge_pages_fops); 3578 if (!ret) 3579 pr_warn("Failed to create split_huge_pages in debugfs"); 3580 return 0; 3581 } 3582 late_initcall(split_huge_pages_debugfs); 3583 #endif 3584