1 /* 2 * Copyright (C) 2009 Red Hat, Inc. 3 * 4 * This work is licensed under the terms of the GNU GPL, version 2. See 5 * the COPYING file in the top-level directory. 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 10 #include <linux/mm.h> 11 #include <linux/sched.h> 12 #include <linux/sched/coredump.h> 13 #include <linux/sched/numa_balancing.h> 14 #include <linux/highmem.h> 15 #include <linux/hugetlb.h> 16 #include <linux/mmu_notifier.h> 17 #include <linux/rmap.h> 18 #include <linux/swap.h> 19 #include <linux/shrinker.h> 20 #include <linux/mm_inline.h> 21 #include <linux/swapops.h> 22 #include <linux/dax.h> 23 #include <linux/khugepaged.h> 24 #include <linux/freezer.h> 25 #include <linux/pfn_t.h> 26 #include <linux/mman.h> 27 #include <linux/memremap.h> 28 #include <linux/pagemap.h> 29 #include <linux/debugfs.h> 30 #include <linux/migrate.h> 31 #include <linux/hashtable.h> 32 #include <linux/userfaultfd_k.h> 33 #include <linux/page_idle.h> 34 #include <linux/shmem_fs.h> 35 #include <linux/oom.h> 36 #include <linux/numa.h> 37 38 #include <asm/tlb.h> 39 #include <asm/pgalloc.h> 40 #include "internal.h" 41 42 /* 43 * By default, transparent hugepage support is disabled in order to avoid 44 * risking an increased memory footprint for applications that are not 45 * guaranteed to benefit from it. When transparent hugepage support is 46 * enabled, it is for all mappings, and khugepaged scans all mappings. 47 * Defrag is invoked by khugepaged hugepage allocations and by page faults 48 * for all hugepage allocations. 49 */ 50 unsigned long transparent_hugepage_flags __read_mostly = 51 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS 52 (1<<TRANSPARENT_HUGEPAGE_FLAG)| 53 #endif 54 #ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE 55 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)| 56 #endif 57 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)| 58 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| 59 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 60 61 static struct shrinker deferred_split_shrinker; 62 63 static atomic_t huge_zero_refcount; 64 struct page *huge_zero_page __read_mostly; 65 66 bool transparent_hugepage_enabled(struct vm_area_struct *vma) 67 { 68 if (vma_is_anonymous(vma)) 69 return __transparent_hugepage_enabled(vma); 70 if (vma_is_shmem(vma) && shmem_huge_enabled(vma)) 71 return __transparent_hugepage_enabled(vma); 72 73 return false; 74 } 75 76 static struct page *get_huge_zero_page(void) 77 { 78 struct page *zero_page; 79 retry: 80 if (likely(atomic_inc_not_zero(&huge_zero_refcount))) 81 return READ_ONCE(huge_zero_page); 82 83 zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE, 84 HPAGE_PMD_ORDER); 85 if (!zero_page) { 86 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); 87 return NULL; 88 } 89 count_vm_event(THP_ZERO_PAGE_ALLOC); 90 preempt_disable(); 91 if (cmpxchg(&huge_zero_page, NULL, zero_page)) { 92 preempt_enable(); 93 __free_pages(zero_page, compound_order(zero_page)); 94 goto retry; 95 } 96 97 /* We take additional reference here. It will be put back by shrinker */ 98 atomic_set(&huge_zero_refcount, 2); 99 preempt_enable(); 100 return READ_ONCE(huge_zero_page); 101 } 102 103 static void put_huge_zero_page(void) 104 { 105 /* 106 * Counter should never go to zero here. Only shrinker can put 107 * last reference. 108 */ 109 BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); 110 } 111 112 struct page *mm_get_huge_zero_page(struct mm_struct *mm) 113 { 114 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 115 return READ_ONCE(huge_zero_page); 116 117 if (!get_huge_zero_page()) 118 return NULL; 119 120 if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 121 put_huge_zero_page(); 122 123 return READ_ONCE(huge_zero_page); 124 } 125 126 void mm_put_huge_zero_page(struct mm_struct *mm) 127 { 128 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags)) 129 put_huge_zero_page(); 130 } 131 132 static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink, 133 struct shrink_control *sc) 134 { 135 /* we can free zero page only if last reference remains */ 136 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; 137 } 138 139 static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink, 140 struct shrink_control *sc) 141 { 142 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { 143 struct page *zero_page = xchg(&huge_zero_page, NULL); 144 BUG_ON(zero_page == NULL); 145 __free_pages(zero_page, compound_order(zero_page)); 146 return HPAGE_PMD_NR; 147 } 148 149 return 0; 150 } 151 152 static struct shrinker huge_zero_page_shrinker = { 153 .count_objects = shrink_huge_zero_page_count, 154 .scan_objects = shrink_huge_zero_page_scan, 155 .seeks = DEFAULT_SEEKS, 156 }; 157 158 #ifdef CONFIG_SYSFS 159 static ssize_t enabled_show(struct kobject *kobj, 160 struct kobj_attribute *attr, char *buf) 161 { 162 if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags)) 163 return sprintf(buf, "[always] madvise never\n"); 164 else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags)) 165 return sprintf(buf, "always [madvise] never\n"); 166 else 167 return sprintf(buf, "always madvise [never]\n"); 168 } 169 170 static ssize_t enabled_store(struct kobject *kobj, 171 struct kobj_attribute *attr, 172 const char *buf, size_t count) 173 { 174 ssize_t ret = count; 175 176 if (!memcmp("always", buf, 177 min(sizeof("always")-1, count))) { 178 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 179 set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 180 } else if (!memcmp("madvise", buf, 181 min(sizeof("madvise")-1, count))) { 182 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 183 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 184 } else if (!memcmp("never", buf, 185 min(sizeof("never")-1, count))) { 186 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); 187 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); 188 } else 189 ret = -EINVAL; 190 191 if (ret > 0) { 192 int err = start_stop_khugepaged(); 193 if (err) 194 ret = err; 195 } 196 return ret; 197 } 198 static struct kobj_attribute enabled_attr = 199 __ATTR(enabled, 0644, enabled_show, enabled_store); 200 201 ssize_t single_hugepage_flag_show(struct kobject *kobj, 202 struct kobj_attribute *attr, char *buf, 203 enum transparent_hugepage_flag flag) 204 { 205 return sprintf(buf, "%d\n", 206 !!test_bit(flag, &transparent_hugepage_flags)); 207 } 208 209 ssize_t single_hugepage_flag_store(struct kobject *kobj, 210 struct kobj_attribute *attr, 211 const char *buf, size_t count, 212 enum transparent_hugepage_flag flag) 213 { 214 unsigned long value; 215 int ret; 216 217 ret = kstrtoul(buf, 10, &value); 218 if (ret < 0) 219 return ret; 220 if (value > 1) 221 return -EINVAL; 222 223 if (value) 224 set_bit(flag, &transparent_hugepage_flags); 225 else 226 clear_bit(flag, &transparent_hugepage_flags); 227 228 return count; 229 } 230 231 static ssize_t defrag_show(struct kobject *kobj, 232 struct kobj_attribute *attr, char *buf) 233 { 234 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) 235 return sprintf(buf, "[always] defer defer+madvise madvise never\n"); 236 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) 237 return sprintf(buf, "always [defer] defer+madvise madvise never\n"); 238 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags)) 239 return sprintf(buf, "always defer [defer+madvise] madvise never\n"); 240 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags)) 241 return sprintf(buf, "always defer defer+madvise [madvise] never\n"); 242 return sprintf(buf, "always defer defer+madvise madvise [never]\n"); 243 } 244 245 static ssize_t defrag_store(struct kobject *kobj, 246 struct kobj_attribute *attr, 247 const char *buf, size_t count) 248 { 249 if (!memcmp("always", buf, 250 min(sizeof("always")-1, count))) { 251 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 252 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 253 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 254 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 255 } else if (!memcmp("defer+madvise", buf, 256 min(sizeof("defer+madvise")-1, count))) { 257 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 258 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 259 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 260 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 261 } else if (!memcmp("defer", buf, 262 min(sizeof("defer")-1, count))) { 263 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 264 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 265 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 266 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 267 } else if (!memcmp("madvise", buf, 268 min(sizeof("madvise")-1, count))) { 269 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 270 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 271 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 272 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 273 } else if (!memcmp("never", buf, 274 min(sizeof("never")-1, count))) { 275 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); 276 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); 277 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); 278 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); 279 } else 280 return -EINVAL; 281 282 return count; 283 } 284 static struct kobj_attribute defrag_attr = 285 __ATTR(defrag, 0644, defrag_show, defrag_store); 286 287 static ssize_t use_zero_page_show(struct kobject *kobj, 288 struct kobj_attribute *attr, char *buf) 289 { 290 return single_hugepage_flag_show(kobj, attr, buf, 291 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 292 } 293 static ssize_t use_zero_page_store(struct kobject *kobj, 294 struct kobj_attribute *attr, const char *buf, size_t count) 295 { 296 return single_hugepage_flag_store(kobj, attr, buf, count, 297 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); 298 } 299 static struct kobj_attribute use_zero_page_attr = 300 __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store); 301 302 static ssize_t hpage_pmd_size_show(struct kobject *kobj, 303 struct kobj_attribute *attr, char *buf) 304 { 305 return sprintf(buf, "%lu\n", HPAGE_PMD_SIZE); 306 } 307 static struct kobj_attribute hpage_pmd_size_attr = 308 __ATTR_RO(hpage_pmd_size); 309 310 #ifdef CONFIG_DEBUG_VM 311 static ssize_t debug_cow_show(struct kobject *kobj, 312 struct kobj_attribute *attr, char *buf) 313 { 314 return single_hugepage_flag_show(kobj, attr, buf, 315 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG); 316 } 317 static ssize_t debug_cow_store(struct kobject *kobj, 318 struct kobj_attribute *attr, 319 const char *buf, size_t count) 320 { 321 return single_hugepage_flag_store(kobj, attr, buf, count, 322 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG); 323 } 324 static struct kobj_attribute debug_cow_attr = 325 __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store); 326 #endif /* CONFIG_DEBUG_VM */ 327 328 static struct attribute *hugepage_attr[] = { 329 &enabled_attr.attr, 330 &defrag_attr.attr, 331 &use_zero_page_attr.attr, 332 &hpage_pmd_size_attr.attr, 333 #if defined(CONFIG_SHMEM) && defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) 334 &shmem_enabled_attr.attr, 335 #endif 336 #ifdef CONFIG_DEBUG_VM 337 &debug_cow_attr.attr, 338 #endif 339 NULL, 340 }; 341 342 static const struct attribute_group hugepage_attr_group = { 343 .attrs = hugepage_attr, 344 }; 345 346 static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj) 347 { 348 int err; 349 350 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj); 351 if (unlikely(!*hugepage_kobj)) { 352 pr_err("failed to create transparent hugepage kobject\n"); 353 return -ENOMEM; 354 } 355 356 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group); 357 if (err) { 358 pr_err("failed to register transparent hugepage group\n"); 359 goto delete_obj; 360 } 361 362 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group); 363 if (err) { 364 pr_err("failed to register transparent hugepage group\n"); 365 goto remove_hp_group; 366 } 367 368 return 0; 369 370 remove_hp_group: 371 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group); 372 delete_obj: 373 kobject_put(*hugepage_kobj); 374 return err; 375 } 376 377 static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj) 378 { 379 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group); 380 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group); 381 kobject_put(hugepage_kobj); 382 } 383 #else 384 static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj) 385 { 386 return 0; 387 } 388 389 static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) 390 { 391 } 392 #endif /* CONFIG_SYSFS */ 393 394 static int __init hugepage_init(void) 395 { 396 int err; 397 struct kobject *hugepage_kobj; 398 399 if (!has_transparent_hugepage()) { 400 transparent_hugepage_flags = 0; 401 return -EINVAL; 402 } 403 404 /* 405 * hugepages can't be allocated by the buddy allocator 406 */ 407 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER >= MAX_ORDER); 408 /* 409 * we use page->mapping and page->index in second tail page 410 * as list_head: assuming THP order >= 2 411 */ 412 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER < 2); 413 414 err = hugepage_init_sysfs(&hugepage_kobj); 415 if (err) 416 goto err_sysfs; 417 418 err = khugepaged_init(); 419 if (err) 420 goto err_slab; 421 422 err = register_shrinker(&huge_zero_page_shrinker); 423 if (err) 424 goto err_hzp_shrinker; 425 err = register_shrinker(&deferred_split_shrinker); 426 if (err) 427 goto err_split_shrinker; 428 429 /* 430 * By default disable transparent hugepages on smaller systems, 431 * where the extra memory used could hurt more than TLB overhead 432 * is likely to save. The admin can still enable it through /sys. 433 */ 434 if (totalram_pages() < (512 << (20 - PAGE_SHIFT))) { 435 transparent_hugepage_flags = 0; 436 return 0; 437 } 438 439 err = start_stop_khugepaged(); 440 if (err) 441 goto err_khugepaged; 442 443 return 0; 444 err_khugepaged: 445 unregister_shrinker(&deferred_split_shrinker); 446 err_split_shrinker: 447 unregister_shrinker(&huge_zero_page_shrinker); 448 err_hzp_shrinker: 449 khugepaged_destroy(); 450 err_slab: 451 hugepage_exit_sysfs(hugepage_kobj); 452 err_sysfs: 453 return err; 454 } 455 subsys_initcall(hugepage_init); 456 457 static int __init setup_transparent_hugepage(char *str) 458 { 459 int ret = 0; 460 if (!str) 461 goto out; 462 if (!strcmp(str, "always")) { 463 set_bit(TRANSPARENT_HUGEPAGE_FLAG, 464 &transparent_hugepage_flags); 465 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 466 &transparent_hugepage_flags); 467 ret = 1; 468 } else if (!strcmp(str, "madvise")) { 469 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 470 &transparent_hugepage_flags); 471 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 472 &transparent_hugepage_flags); 473 ret = 1; 474 } else if (!strcmp(str, "never")) { 475 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, 476 &transparent_hugepage_flags); 477 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, 478 &transparent_hugepage_flags); 479 ret = 1; 480 } 481 out: 482 if (!ret) 483 pr_warn("transparent_hugepage= cannot parse, ignored\n"); 484 return ret; 485 } 486 __setup("transparent_hugepage=", setup_transparent_hugepage); 487 488 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 489 { 490 if (likely(vma->vm_flags & VM_WRITE)) 491 pmd = pmd_mkwrite(pmd); 492 return pmd; 493 } 494 495 static inline struct list_head *page_deferred_list(struct page *page) 496 { 497 /* ->lru in the tail pages is occupied by compound_head. */ 498 return &page[2].deferred_list; 499 } 500 501 void prep_transhuge_page(struct page *page) 502 { 503 /* 504 * we use page->mapping and page->indexlru in second tail page 505 * as list_head: assuming THP order >= 2 506 */ 507 508 INIT_LIST_HEAD(page_deferred_list(page)); 509 set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR); 510 } 511 512 unsigned long __thp_get_unmapped_area(struct file *filp, unsigned long len, 513 loff_t off, unsigned long flags, unsigned long size) 514 { 515 unsigned long addr; 516 loff_t off_end = off + len; 517 loff_t off_align = round_up(off, size); 518 unsigned long len_pad; 519 520 if (off_end <= off_align || (off_end - off_align) < size) 521 return 0; 522 523 len_pad = len + size; 524 if (len_pad < len || (off + len_pad) < off) 525 return 0; 526 527 addr = current->mm->get_unmapped_area(filp, 0, len_pad, 528 off >> PAGE_SHIFT, flags); 529 if (IS_ERR_VALUE(addr)) 530 return 0; 531 532 addr += (off - addr) & (size - 1); 533 return addr; 534 } 535 536 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, 537 unsigned long len, unsigned long pgoff, unsigned long flags) 538 { 539 loff_t off = (loff_t)pgoff << PAGE_SHIFT; 540 541 if (addr) 542 goto out; 543 if (!IS_DAX(filp->f_mapping->host) || !IS_ENABLED(CONFIG_FS_DAX_PMD)) 544 goto out; 545 546 addr = __thp_get_unmapped_area(filp, len, off, flags, PMD_SIZE); 547 if (addr) 548 return addr; 549 550 out: 551 return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); 552 } 553 EXPORT_SYMBOL_GPL(thp_get_unmapped_area); 554 555 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf, 556 struct page *page, gfp_t gfp) 557 { 558 struct vm_area_struct *vma = vmf->vma; 559 struct mem_cgroup *memcg; 560 pgtable_t pgtable; 561 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 562 vm_fault_t ret = 0; 563 564 VM_BUG_ON_PAGE(!PageCompound(page), page); 565 566 if (mem_cgroup_try_charge_delay(page, vma->vm_mm, gfp, &memcg, true)) { 567 put_page(page); 568 count_vm_event(THP_FAULT_FALLBACK); 569 return VM_FAULT_FALLBACK; 570 } 571 572 pgtable = pte_alloc_one(vma->vm_mm); 573 if (unlikely(!pgtable)) { 574 ret = VM_FAULT_OOM; 575 goto release; 576 } 577 578 clear_huge_page(page, vmf->address, HPAGE_PMD_NR); 579 /* 580 * The memory barrier inside __SetPageUptodate makes sure that 581 * clear_huge_page writes become visible before the set_pmd_at() 582 * write. 583 */ 584 __SetPageUptodate(page); 585 586 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 587 if (unlikely(!pmd_none(*vmf->pmd))) { 588 goto unlock_release; 589 } else { 590 pmd_t entry; 591 592 ret = check_stable_address_space(vma->vm_mm); 593 if (ret) 594 goto unlock_release; 595 596 /* Deliver the page fault to userland */ 597 if (userfaultfd_missing(vma)) { 598 vm_fault_t ret2; 599 600 spin_unlock(vmf->ptl); 601 mem_cgroup_cancel_charge(page, memcg, true); 602 put_page(page); 603 pte_free(vma->vm_mm, pgtable); 604 ret2 = handle_userfault(vmf, VM_UFFD_MISSING); 605 VM_BUG_ON(ret2 & VM_FAULT_FALLBACK); 606 return ret2; 607 } 608 609 entry = mk_huge_pmd(page, vma->vm_page_prot); 610 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 611 page_add_new_anon_rmap(page, vma, haddr, true); 612 mem_cgroup_commit_charge(page, memcg, false, true); 613 lru_cache_add_active_or_unevictable(page, vma); 614 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); 615 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); 616 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); 617 mm_inc_nr_ptes(vma->vm_mm); 618 spin_unlock(vmf->ptl); 619 count_vm_event(THP_FAULT_ALLOC); 620 } 621 622 return 0; 623 unlock_release: 624 spin_unlock(vmf->ptl); 625 release: 626 if (pgtable) 627 pte_free(vma->vm_mm, pgtable); 628 mem_cgroup_cancel_charge(page, memcg, true); 629 put_page(page); 630 return ret; 631 632 } 633 634 /* 635 * always: directly stall for all thp allocations 636 * defer: wake kswapd and fail if not immediately available 637 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise 638 * fail if not immediately available 639 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately 640 * available 641 * never: never stall for any thp allocation 642 */ 643 static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma) 644 { 645 const bool vma_madvised = !!(vma->vm_flags & VM_HUGEPAGE); 646 647 /* Always do synchronous compaction */ 648 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) 649 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY); 650 651 /* Kick kcompactd and fail quickly */ 652 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) 653 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM; 654 655 /* Synchronous compaction if madvised, otherwise kick kcompactd */ 656 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags)) 657 return GFP_TRANSHUGE_LIGHT | 658 (vma_madvised ? __GFP_DIRECT_RECLAIM : 659 __GFP_KSWAPD_RECLAIM); 660 661 /* Only do synchronous compaction if madvised */ 662 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags)) 663 return GFP_TRANSHUGE_LIGHT | 664 (vma_madvised ? __GFP_DIRECT_RECLAIM : 0); 665 666 return GFP_TRANSHUGE_LIGHT; 667 } 668 669 /* Caller must hold page table lock. */ 670 static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm, 671 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd, 672 struct page *zero_page) 673 { 674 pmd_t entry; 675 if (!pmd_none(*pmd)) 676 return false; 677 entry = mk_pmd(zero_page, vma->vm_page_prot); 678 entry = pmd_mkhuge(entry); 679 if (pgtable) 680 pgtable_trans_huge_deposit(mm, pmd, pgtable); 681 set_pmd_at(mm, haddr, pmd, entry); 682 mm_inc_nr_ptes(mm); 683 return true; 684 } 685 686 vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) 687 { 688 struct vm_area_struct *vma = vmf->vma; 689 gfp_t gfp; 690 struct page *page; 691 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 692 693 if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end) 694 return VM_FAULT_FALLBACK; 695 if (unlikely(anon_vma_prepare(vma))) 696 return VM_FAULT_OOM; 697 if (unlikely(khugepaged_enter(vma, vma->vm_flags))) 698 return VM_FAULT_OOM; 699 if (!(vmf->flags & FAULT_FLAG_WRITE) && 700 !mm_forbids_zeropage(vma->vm_mm) && 701 transparent_hugepage_use_zero_page()) { 702 pgtable_t pgtable; 703 struct page *zero_page; 704 bool set; 705 vm_fault_t ret; 706 pgtable = pte_alloc_one(vma->vm_mm); 707 if (unlikely(!pgtable)) 708 return VM_FAULT_OOM; 709 zero_page = mm_get_huge_zero_page(vma->vm_mm); 710 if (unlikely(!zero_page)) { 711 pte_free(vma->vm_mm, pgtable); 712 count_vm_event(THP_FAULT_FALLBACK); 713 return VM_FAULT_FALLBACK; 714 } 715 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 716 ret = 0; 717 set = false; 718 if (pmd_none(*vmf->pmd)) { 719 ret = check_stable_address_space(vma->vm_mm); 720 if (ret) { 721 spin_unlock(vmf->ptl); 722 } else if (userfaultfd_missing(vma)) { 723 spin_unlock(vmf->ptl); 724 ret = handle_userfault(vmf, VM_UFFD_MISSING); 725 VM_BUG_ON(ret & VM_FAULT_FALLBACK); 726 } else { 727 set_huge_zero_page(pgtable, vma->vm_mm, vma, 728 haddr, vmf->pmd, zero_page); 729 spin_unlock(vmf->ptl); 730 set = true; 731 } 732 } else 733 spin_unlock(vmf->ptl); 734 if (!set) 735 pte_free(vma->vm_mm, pgtable); 736 return ret; 737 } 738 gfp = alloc_hugepage_direct_gfpmask(vma); 739 page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER); 740 if (unlikely(!page)) { 741 count_vm_event(THP_FAULT_FALLBACK); 742 return VM_FAULT_FALLBACK; 743 } 744 prep_transhuge_page(page); 745 return __do_huge_pmd_anonymous_page(vmf, page, gfp); 746 } 747 748 static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 749 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write, 750 pgtable_t pgtable) 751 { 752 struct mm_struct *mm = vma->vm_mm; 753 pmd_t entry; 754 spinlock_t *ptl; 755 756 ptl = pmd_lock(mm, pmd); 757 entry = pmd_mkhuge(pfn_t_pmd(pfn, prot)); 758 if (pfn_t_devmap(pfn)) 759 entry = pmd_mkdevmap(entry); 760 if (write) { 761 entry = pmd_mkyoung(pmd_mkdirty(entry)); 762 entry = maybe_pmd_mkwrite(entry, vma); 763 } 764 765 if (pgtable) { 766 pgtable_trans_huge_deposit(mm, pmd, pgtable); 767 mm_inc_nr_ptes(mm); 768 } 769 770 set_pmd_at(mm, addr, pmd, entry); 771 update_mmu_cache_pmd(vma, addr, pmd); 772 spin_unlock(ptl); 773 } 774 775 vm_fault_t vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, 776 pmd_t *pmd, pfn_t pfn, bool write) 777 { 778 pgprot_t pgprot = vma->vm_page_prot; 779 pgtable_t pgtable = NULL; 780 /* 781 * If we had pmd_special, we could avoid all these restrictions, 782 * but we need to be consistent with PTEs and architectures that 783 * can't support a 'special' bit. 784 */ 785 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && 786 !pfn_t_devmap(pfn)); 787 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 788 (VM_PFNMAP|VM_MIXEDMAP)); 789 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 790 791 if (addr < vma->vm_start || addr >= vma->vm_end) 792 return VM_FAULT_SIGBUS; 793 794 if (arch_needs_pgtable_deposit()) { 795 pgtable = pte_alloc_one(vma->vm_mm); 796 if (!pgtable) 797 return VM_FAULT_OOM; 798 } 799 800 track_pfn_insert(vma, &pgprot, pfn); 801 802 insert_pfn_pmd(vma, addr, pmd, pfn, pgprot, write, pgtable); 803 return VM_FAULT_NOPAGE; 804 } 805 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd); 806 807 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 808 static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma) 809 { 810 if (likely(vma->vm_flags & VM_WRITE)) 811 pud = pud_mkwrite(pud); 812 return pud; 813 } 814 815 static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr, 816 pud_t *pud, pfn_t pfn, pgprot_t prot, bool write) 817 { 818 struct mm_struct *mm = vma->vm_mm; 819 pud_t entry; 820 spinlock_t *ptl; 821 822 ptl = pud_lock(mm, pud); 823 entry = pud_mkhuge(pfn_t_pud(pfn, prot)); 824 if (pfn_t_devmap(pfn)) 825 entry = pud_mkdevmap(entry); 826 if (write) { 827 entry = pud_mkyoung(pud_mkdirty(entry)); 828 entry = maybe_pud_mkwrite(entry, vma); 829 } 830 set_pud_at(mm, addr, pud, entry); 831 update_mmu_cache_pud(vma, addr, pud); 832 spin_unlock(ptl); 833 } 834 835 vm_fault_t vmf_insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr, 836 pud_t *pud, pfn_t pfn, bool write) 837 { 838 pgprot_t pgprot = vma->vm_page_prot; 839 /* 840 * If we had pud_special, we could avoid all these restrictions, 841 * but we need to be consistent with PTEs and architectures that 842 * can't support a 'special' bit. 843 */ 844 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && 845 !pfn_t_devmap(pfn)); 846 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == 847 (VM_PFNMAP|VM_MIXEDMAP)); 848 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); 849 850 if (addr < vma->vm_start || addr >= vma->vm_end) 851 return VM_FAULT_SIGBUS; 852 853 track_pfn_insert(vma, &pgprot, pfn); 854 855 insert_pfn_pud(vma, addr, pud, pfn, pgprot, write); 856 return VM_FAULT_NOPAGE; 857 } 858 EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud); 859 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 860 861 static void touch_pmd(struct vm_area_struct *vma, unsigned long addr, 862 pmd_t *pmd, int flags) 863 { 864 pmd_t _pmd; 865 866 _pmd = pmd_mkyoung(*pmd); 867 if (flags & FOLL_WRITE) 868 _pmd = pmd_mkdirty(_pmd); 869 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, 870 pmd, _pmd, flags & FOLL_WRITE)) 871 update_mmu_cache_pmd(vma, addr, pmd); 872 } 873 874 struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, 875 pmd_t *pmd, int flags, struct dev_pagemap **pgmap) 876 { 877 unsigned long pfn = pmd_pfn(*pmd); 878 struct mm_struct *mm = vma->vm_mm; 879 struct page *page; 880 881 assert_spin_locked(pmd_lockptr(mm, pmd)); 882 883 /* 884 * When we COW a devmap PMD entry, we split it into PTEs, so we should 885 * not be in this function with `flags & FOLL_COW` set. 886 */ 887 WARN_ONCE(flags & FOLL_COW, "mm: In follow_devmap_pmd with FOLL_COW set"); 888 889 if (flags & FOLL_WRITE && !pmd_write(*pmd)) 890 return NULL; 891 892 if (pmd_present(*pmd) && pmd_devmap(*pmd)) 893 /* pass */; 894 else 895 return NULL; 896 897 if (flags & FOLL_TOUCH) 898 touch_pmd(vma, addr, pmd, flags); 899 900 /* 901 * device mapped pages can only be returned if the 902 * caller will manage the page reference count. 903 */ 904 if (!(flags & FOLL_GET)) 905 return ERR_PTR(-EEXIST); 906 907 pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT; 908 *pgmap = get_dev_pagemap(pfn, *pgmap); 909 if (!*pgmap) 910 return ERR_PTR(-EFAULT); 911 page = pfn_to_page(pfn); 912 get_page(page); 913 914 return page; 915 } 916 917 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, 918 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, 919 struct vm_area_struct *vma) 920 { 921 spinlock_t *dst_ptl, *src_ptl; 922 struct page *src_page; 923 pmd_t pmd; 924 pgtable_t pgtable = NULL; 925 int ret = -ENOMEM; 926 927 /* Skip if can be re-fill on fault */ 928 if (!vma_is_anonymous(vma)) 929 return 0; 930 931 pgtable = pte_alloc_one(dst_mm); 932 if (unlikely(!pgtable)) 933 goto out; 934 935 dst_ptl = pmd_lock(dst_mm, dst_pmd); 936 src_ptl = pmd_lockptr(src_mm, src_pmd); 937 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 938 939 ret = -EAGAIN; 940 pmd = *src_pmd; 941 942 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 943 if (unlikely(is_swap_pmd(pmd))) { 944 swp_entry_t entry = pmd_to_swp_entry(pmd); 945 946 VM_BUG_ON(!is_pmd_migration_entry(pmd)); 947 if (is_write_migration_entry(entry)) { 948 make_migration_entry_read(&entry); 949 pmd = swp_entry_to_pmd(entry); 950 if (pmd_swp_soft_dirty(*src_pmd)) 951 pmd = pmd_swp_mksoft_dirty(pmd); 952 set_pmd_at(src_mm, addr, src_pmd, pmd); 953 } 954 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 955 mm_inc_nr_ptes(dst_mm); 956 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 957 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 958 ret = 0; 959 goto out_unlock; 960 } 961 #endif 962 963 if (unlikely(!pmd_trans_huge(pmd))) { 964 pte_free(dst_mm, pgtable); 965 goto out_unlock; 966 } 967 /* 968 * When page table lock is held, the huge zero pmd should not be 969 * under splitting since we don't split the page itself, only pmd to 970 * a page table. 971 */ 972 if (is_huge_zero_pmd(pmd)) { 973 struct page *zero_page; 974 /* 975 * get_huge_zero_page() will never allocate a new page here, 976 * since we already have a zero page to copy. It just takes a 977 * reference. 978 */ 979 zero_page = mm_get_huge_zero_page(dst_mm); 980 set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd, 981 zero_page); 982 ret = 0; 983 goto out_unlock; 984 } 985 986 src_page = pmd_page(pmd); 987 VM_BUG_ON_PAGE(!PageHead(src_page), src_page); 988 get_page(src_page); 989 page_dup_rmap(src_page, true); 990 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); 991 mm_inc_nr_ptes(dst_mm); 992 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); 993 994 pmdp_set_wrprotect(src_mm, addr, src_pmd); 995 pmd = pmd_mkold(pmd_wrprotect(pmd)); 996 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 997 998 ret = 0; 999 out_unlock: 1000 spin_unlock(src_ptl); 1001 spin_unlock(dst_ptl); 1002 out: 1003 return ret; 1004 } 1005 1006 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1007 static void touch_pud(struct vm_area_struct *vma, unsigned long addr, 1008 pud_t *pud, int flags) 1009 { 1010 pud_t _pud; 1011 1012 _pud = pud_mkyoung(*pud); 1013 if (flags & FOLL_WRITE) 1014 _pud = pud_mkdirty(_pud); 1015 if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK, 1016 pud, _pud, flags & FOLL_WRITE)) 1017 update_mmu_cache_pud(vma, addr, pud); 1018 } 1019 1020 struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr, 1021 pud_t *pud, int flags, struct dev_pagemap **pgmap) 1022 { 1023 unsigned long pfn = pud_pfn(*pud); 1024 struct mm_struct *mm = vma->vm_mm; 1025 struct page *page; 1026 1027 assert_spin_locked(pud_lockptr(mm, pud)); 1028 1029 if (flags & FOLL_WRITE && !pud_write(*pud)) 1030 return NULL; 1031 1032 if (pud_present(*pud) && pud_devmap(*pud)) 1033 /* pass */; 1034 else 1035 return NULL; 1036 1037 if (flags & FOLL_TOUCH) 1038 touch_pud(vma, addr, pud, flags); 1039 1040 /* 1041 * device mapped pages can only be returned if the 1042 * caller will manage the page reference count. 1043 */ 1044 if (!(flags & FOLL_GET)) 1045 return ERR_PTR(-EEXIST); 1046 1047 pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT; 1048 *pgmap = get_dev_pagemap(pfn, *pgmap); 1049 if (!*pgmap) 1050 return ERR_PTR(-EFAULT); 1051 page = pfn_to_page(pfn); 1052 get_page(page); 1053 1054 return page; 1055 } 1056 1057 int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, 1058 pud_t *dst_pud, pud_t *src_pud, unsigned long addr, 1059 struct vm_area_struct *vma) 1060 { 1061 spinlock_t *dst_ptl, *src_ptl; 1062 pud_t pud; 1063 int ret; 1064 1065 dst_ptl = pud_lock(dst_mm, dst_pud); 1066 src_ptl = pud_lockptr(src_mm, src_pud); 1067 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); 1068 1069 ret = -EAGAIN; 1070 pud = *src_pud; 1071 if (unlikely(!pud_trans_huge(pud) && !pud_devmap(pud))) 1072 goto out_unlock; 1073 1074 /* 1075 * When page table lock is held, the huge zero pud should not be 1076 * under splitting since we don't split the page itself, only pud to 1077 * a page table. 1078 */ 1079 if (is_huge_zero_pud(pud)) { 1080 /* No huge zero pud yet */ 1081 } 1082 1083 pudp_set_wrprotect(src_mm, addr, src_pud); 1084 pud = pud_mkold(pud_wrprotect(pud)); 1085 set_pud_at(dst_mm, addr, dst_pud, pud); 1086 1087 ret = 0; 1088 out_unlock: 1089 spin_unlock(src_ptl); 1090 spin_unlock(dst_ptl); 1091 return ret; 1092 } 1093 1094 void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud) 1095 { 1096 pud_t entry; 1097 unsigned long haddr; 1098 bool write = vmf->flags & FAULT_FLAG_WRITE; 1099 1100 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); 1101 if (unlikely(!pud_same(*vmf->pud, orig_pud))) 1102 goto unlock; 1103 1104 entry = pud_mkyoung(orig_pud); 1105 if (write) 1106 entry = pud_mkdirty(entry); 1107 haddr = vmf->address & HPAGE_PUD_MASK; 1108 if (pudp_set_access_flags(vmf->vma, haddr, vmf->pud, entry, write)) 1109 update_mmu_cache_pud(vmf->vma, vmf->address, vmf->pud); 1110 1111 unlock: 1112 spin_unlock(vmf->ptl); 1113 } 1114 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 1115 1116 void huge_pmd_set_accessed(struct vm_fault *vmf, pmd_t orig_pmd) 1117 { 1118 pmd_t entry; 1119 unsigned long haddr; 1120 bool write = vmf->flags & FAULT_FLAG_WRITE; 1121 1122 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd); 1123 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) 1124 goto unlock; 1125 1126 entry = pmd_mkyoung(orig_pmd); 1127 if (write) 1128 entry = pmd_mkdirty(entry); 1129 haddr = vmf->address & HPAGE_PMD_MASK; 1130 if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write)) 1131 update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd); 1132 1133 unlock: 1134 spin_unlock(vmf->ptl); 1135 } 1136 1137 static vm_fault_t do_huge_pmd_wp_page_fallback(struct vm_fault *vmf, 1138 pmd_t orig_pmd, struct page *page) 1139 { 1140 struct vm_area_struct *vma = vmf->vma; 1141 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1142 struct mem_cgroup *memcg; 1143 pgtable_t pgtable; 1144 pmd_t _pmd; 1145 int i; 1146 vm_fault_t ret = 0; 1147 struct page **pages; 1148 struct mmu_notifier_range range; 1149 1150 pages = kmalloc_array(HPAGE_PMD_NR, sizeof(struct page *), 1151 GFP_KERNEL); 1152 if (unlikely(!pages)) { 1153 ret |= VM_FAULT_OOM; 1154 goto out; 1155 } 1156 1157 for (i = 0; i < HPAGE_PMD_NR; i++) { 1158 pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE, vma, 1159 vmf->address, page_to_nid(page)); 1160 if (unlikely(!pages[i] || 1161 mem_cgroup_try_charge_delay(pages[i], vma->vm_mm, 1162 GFP_KERNEL, &memcg, false))) { 1163 if (pages[i]) 1164 put_page(pages[i]); 1165 while (--i >= 0) { 1166 memcg = (void *)page_private(pages[i]); 1167 set_page_private(pages[i], 0); 1168 mem_cgroup_cancel_charge(pages[i], memcg, 1169 false); 1170 put_page(pages[i]); 1171 } 1172 kfree(pages); 1173 ret |= VM_FAULT_OOM; 1174 goto out; 1175 } 1176 set_page_private(pages[i], (unsigned long)memcg); 1177 } 1178 1179 for (i = 0; i < HPAGE_PMD_NR; i++) { 1180 copy_user_highpage(pages[i], page + i, 1181 haddr + PAGE_SIZE * i, vma); 1182 __SetPageUptodate(pages[i]); 1183 cond_resched(); 1184 } 1185 1186 mmu_notifier_range_init(&range, vma->vm_mm, haddr, 1187 haddr + HPAGE_PMD_SIZE); 1188 mmu_notifier_invalidate_range_start(&range); 1189 1190 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1191 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) 1192 goto out_free_pages; 1193 VM_BUG_ON_PAGE(!PageHead(page), page); 1194 1195 /* 1196 * Leave pmd empty until pte is filled note we must notify here as 1197 * concurrent CPU thread might write to new page before the call to 1198 * mmu_notifier_invalidate_range_end() happens which can lead to a 1199 * device seeing memory write in different order than CPU. 1200 * 1201 * See Documentation/vm/mmu_notifier.rst 1202 */ 1203 pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd); 1204 1205 pgtable = pgtable_trans_huge_withdraw(vma->vm_mm, vmf->pmd); 1206 pmd_populate(vma->vm_mm, &_pmd, pgtable); 1207 1208 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 1209 pte_t entry; 1210 entry = mk_pte(pages[i], vma->vm_page_prot); 1211 entry = maybe_mkwrite(pte_mkdirty(entry), vma); 1212 memcg = (void *)page_private(pages[i]); 1213 set_page_private(pages[i], 0); 1214 page_add_new_anon_rmap(pages[i], vmf->vma, haddr, false); 1215 mem_cgroup_commit_charge(pages[i], memcg, false, false); 1216 lru_cache_add_active_or_unevictable(pages[i], vma); 1217 vmf->pte = pte_offset_map(&_pmd, haddr); 1218 VM_BUG_ON(!pte_none(*vmf->pte)); 1219 set_pte_at(vma->vm_mm, haddr, vmf->pte, entry); 1220 pte_unmap(vmf->pte); 1221 } 1222 kfree(pages); 1223 1224 smp_wmb(); /* make pte visible before pmd */ 1225 pmd_populate(vma->vm_mm, vmf->pmd, pgtable); 1226 page_remove_rmap(page, true); 1227 spin_unlock(vmf->ptl); 1228 1229 /* 1230 * No need to double call mmu_notifier->invalidate_range() callback as 1231 * the above pmdp_huge_clear_flush_notify() did already call it. 1232 */ 1233 mmu_notifier_invalidate_range_only_end(&range); 1234 1235 ret |= VM_FAULT_WRITE; 1236 put_page(page); 1237 1238 out: 1239 return ret; 1240 1241 out_free_pages: 1242 spin_unlock(vmf->ptl); 1243 mmu_notifier_invalidate_range_end(&range); 1244 for (i = 0; i < HPAGE_PMD_NR; i++) { 1245 memcg = (void *)page_private(pages[i]); 1246 set_page_private(pages[i], 0); 1247 mem_cgroup_cancel_charge(pages[i], memcg, false); 1248 put_page(pages[i]); 1249 } 1250 kfree(pages); 1251 goto out; 1252 } 1253 1254 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd) 1255 { 1256 struct vm_area_struct *vma = vmf->vma; 1257 struct page *page = NULL, *new_page; 1258 struct mem_cgroup *memcg; 1259 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1260 struct mmu_notifier_range range; 1261 gfp_t huge_gfp; /* for allocation and charge */ 1262 vm_fault_t ret = 0; 1263 1264 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd); 1265 VM_BUG_ON_VMA(!vma->anon_vma, vma); 1266 if (is_huge_zero_pmd(orig_pmd)) 1267 goto alloc; 1268 spin_lock(vmf->ptl); 1269 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) 1270 goto out_unlock; 1271 1272 page = pmd_page(orig_pmd); 1273 VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page); 1274 /* 1275 * We can only reuse the page if nobody else maps the huge page or it's 1276 * part. 1277 */ 1278 if (!trylock_page(page)) { 1279 get_page(page); 1280 spin_unlock(vmf->ptl); 1281 lock_page(page); 1282 spin_lock(vmf->ptl); 1283 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1284 unlock_page(page); 1285 put_page(page); 1286 goto out_unlock; 1287 } 1288 put_page(page); 1289 } 1290 if (reuse_swap_page(page, NULL)) { 1291 pmd_t entry; 1292 entry = pmd_mkyoung(orig_pmd); 1293 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1294 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1)) 1295 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1296 ret |= VM_FAULT_WRITE; 1297 unlock_page(page); 1298 goto out_unlock; 1299 } 1300 unlock_page(page); 1301 get_page(page); 1302 spin_unlock(vmf->ptl); 1303 alloc: 1304 if (__transparent_hugepage_enabled(vma) && 1305 !transparent_hugepage_debug_cow()) { 1306 huge_gfp = alloc_hugepage_direct_gfpmask(vma); 1307 new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER); 1308 } else 1309 new_page = NULL; 1310 1311 if (likely(new_page)) { 1312 prep_transhuge_page(new_page); 1313 } else { 1314 if (!page) { 1315 split_huge_pmd(vma, vmf->pmd, vmf->address); 1316 ret |= VM_FAULT_FALLBACK; 1317 } else { 1318 ret = do_huge_pmd_wp_page_fallback(vmf, orig_pmd, page); 1319 if (ret & VM_FAULT_OOM) { 1320 split_huge_pmd(vma, vmf->pmd, vmf->address); 1321 ret |= VM_FAULT_FALLBACK; 1322 } 1323 put_page(page); 1324 } 1325 count_vm_event(THP_FAULT_FALLBACK); 1326 goto out; 1327 } 1328 1329 if (unlikely(mem_cgroup_try_charge_delay(new_page, vma->vm_mm, 1330 huge_gfp, &memcg, true))) { 1331 put_page(new_page); 1332 split_huge_pmd(vma, vmf->pmd, vmf->address); 1333 if (page) 1334 put_page(page); 1335 ret |= VM_FAULT_FALLBACK; 1336 count_vm_event(THP_FAULT_FALLBACK); 1337 goto out; 1338 } 1339 1340 count_vm_event(THP_FAULT_ALLOC); 1341 1342 if (!page) 1343 clear_huge_page(new_page, vmf->address, HPAGE_PMD_NR); 1344 else 1345 copy_user_huge_page(new_page, page, vmf->address, 1346 vma, HPAGE_PMD_NR); 1347 __SetPageUptodate(new_page); 1348 1349 mmu_notifier_range_init(&range, vma->vm_mm, haddr, 1350 haddr + HPAGE_PMD_SIZE); 1351 mmu_notifier_invalidate_range_start(&range); 1352 1353 spin_lock(vmf->ptl); 1354 if (page) 1355 put_page(page); 1356 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { 1357 spin_unlock(vmf->ptl); 1358 mem_cgroup_cancel_charge(new_page, memcg, true); 1359 put_page(new_page); 1360 goto out_mn; 1361 } else { 1362 pmd_t entry; 1363 entry = mk_huge_pmd(new_page, vma->vm_page_prot); 1364 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); 1365 pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd); 1366 page_add_new_anon_rmap(new_page, vma, haddr, true); 1367 mem_cgroup_commit_charge(new_page, memcg, false, true); 1368 lru_cache_add_active_or_unevictable(new_page, vma); 1369 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry); 1370 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1371 if (!page) { 1372 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); 1373 } else { 1374 VM_BUG_ON_PAGE(!PageHead(page), page); 1375 page_remove_rmap(page, true); 1376 put_page(page); 1377 } 1378 ret |= VM_FAULT_WRITE; 1379 } 1380 spin_unlock(vmf->ptl); 1381 out_mn: 1382 /* 1383 * No need to double call mmu_notifier->invalidate_range() callback as 1384 * the above pmdp_huge_clear_flush_notify() did already call it. 1385 */ 1386 mmu_notifier_invalidate_range_only_end(&range); 1387 out: 1388 return ret; 1389 out_unlock: 1390 spin_unlock(vmf->ptl); 1391 return ret; 1392 } 1393 1394 /* 1395 * FOLL_FORCE can write to even unwritable pmd's, but only 1396 * after we've gone through a COW cycle and they are dirty. 1397 */ 1398 static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags) 1399 { 1400 return pmd_write(pmd) || 1401 ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd)); 1402 } 1403 1404 struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, 1405 unsigned long addr, 1406 pmd_t *pmd, 1407 unsigned int flags) 1408 { 1409 struct mm_struct *mm = vma->vm_mm; 1410 struct page *page = NULL; 1411 1412 assert_spin_locked(pmd_lockptr(mm, pmd)); 1413 1414 if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags)) 1415 goto out; 1416 1417 /* Avoid dumping huge zero page */ 1418 if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd)) 1419 return ERR_PTR(-EFAULT); 1420 1421 /* Full NUMA hinting faults to serialise migration in fault paths */ 1422 if ((flags & FOLL_NUMA) && pmd_protnone(*pmd)) 1423 goto out; 1424 1425 page = pmd_page(*pmd); 1426 VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page); 1427 if (flags & FOLL_TOUCH) 1428 touch_pmd(vma, addr, pmd, flags); 1429 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) { 1430 /* 1431 * We don't mlock() pte-mapped THPs. This way we can avoid 1432 * leaking mlocked pages into non-VM_LOCKED VMAs. 1433 * 1434 * For anon THP: 1435 * 1436 * In most cases the pmd is the only mapping of the page as we 1437 * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for 1438 * writable private mappings in populate_vma_page_range(). 1439 * 1440 * The only scenario when we have the page shared here is if we 1441 * mlocking read-only mapping shared over fork(). We skip 1442 * mlocking such pages. 1443 * 1444 * For file THP: 1445 * 1446 * We can expect PageDoubleMap() to be stable under page lock: 1447 * for file pages we set it in page_add_file_rmap(), which 1448 * requires page to be locked. 1449 */ 1450 1451 if (PageAnon(page) && compound_mapcount(page) != 1) 1452 goto skip_mlock; 1453 if (PageDoubleMap(page) || !page->mapping) 1454 goto skip_mlock; 1455 if (!trylock_page(page)) 1456 goto skip_mlock; 1457 lru_add_drain(); 1458 if (page->mapping && !PageDoubleMap(page)) 1459 mlock_vma_page(page); 1460 unlock_page(page); 1461 } 1462 skip_mlock: 1463 page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT; 1464 VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page); 1465 if (flags & FOLL_GET) 1466 get_page(page); 1467 1468 out: 1469 return page; 1470 } 1471 1472 /* NUMA hinting page fault entry point for trans huge pmds */ 1473 vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd) 1474 { 1475 struct vm_area_struct *vma = vmf->vma; 1476 struct anon_vma *anon_vma = NULL; 1477 struct page *page; 1478 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; 1479 int page_nid = NUMA_NO_NODE, this_nid = numa_node_id(); 1480 int target_nid, last_cpupid = -1; 1481 bool page_locked; 1482 bool migrated = false; 1483 bool was_writable; 1484 int flags = 0; 1485 1486 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); 1487 if (unlikely(!pmd_same(pmd, *vmf->pmd))) 1488 goto out_unlock; 1489 1490 /* 1491 * If there are potential migrations, wait for completion and retry 1492 * without disrupting NUMA hinting information. Do not relock and 1493 * check_same as the page may no longer be mapped. 1494 */ 1495 if (unlikely(pmd_trans_migrating(*vmf->pmd))) { 1496 page = pmd_page(*vmf->pmd); 1497 if (!get_page_unless_zero(page)) 1498 goto out_unlock; 1499 spin_unlock(vmf->ptl); 1500 put_and_wait_on_page_locked(page); 1501 goto out; 1502 } 1503 1504 page = pmd_page(pmd); 1505 BUG_ON(is_huge_zero_page(page)); 1506 page_nid = page_to_nid(page); 1507 last_cpupid = page_cpupid_last(page); 1508 count_vm_numa_event(NUMA_HINT_FAULTS); 1509 if (page_nid == this_nid) { 1510 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL); 1511 flags |= TNF_FAULT_LOCAL; 1512 } 1513 1514 /* See similar comment in do_numa_page for explanation */ 1515 if (!pmd_savedwrite(pmd)) 1516 flags |= TNF_NO_GROUP; 1517 1518 /* 1519 * Acquire the page lock to serialise THP migrations but avoid dropping 1520 * page_table_lock if at all possible 1521 */ 1522 page_locked = trylock_page(page); 1523 target_nid = mpol_misplaced(page, vma, haddr); 1524 if (target_nid == NUMA_NO_NODE) { 1525 /* If the page was locked, there are no parallel migrations */ 1526 if (page_locked) 1527 goto clear_pmdnuma; 1528 } 1529 1530 /* Migration could have started since the pmd_trans_migrating check */ 1531 if (!page_locked) { 1532 page_nid = NUMA_NO_NODE; 1533 if (!get_page_unless_zero(page)) 1534 goto out_unlock; 1535 spin_unlock(vmf->ptl); 1536 put_and_wait_on_page_locked(page); 1537 goto out; 1538 } 1539 1540 /* 1541 * Page is misplaced. Page lock serialises migrations. Acquire anon_vma 1542 * to serialises splits 1543 */ 1544 get_page(page); 1545 spin_unlock(vmf->ptl); 1546 anon_vma = page_lock_anon_vma_read(page); 1547 1548 /* Confirm the PMD did not change while page_table_lock was released */ 1549 spin_lock(vmf->ptl); 1550 if (unlikely(!pmd_same(pmd, *vmf->pmd))) { 1551 unlock_page(page); 1552 put_page(page); 1553 page_nid = NUMA_NO_NODE; 1554 goto out_unlock; 1555 } 1556 1557 /* Bail if we fail to protect against THP splits for any reason */ 1558 if (unlikely(!anon_vma)) { 1559 put_page(page); 1560 page_nid = NUMA_NO_NODE; 1561 goto clear_pmdnuma; 1562 } 1563 1564 /* 1565 * Since we took the NUMA fault, we must have observed the !accessible 1566 * bit. Make sure all other CPUs agree with that, to avoid them 1567 * modifying the page we're about to migrate. 1568 * 1569 * Must be done under PTL such that we'll observe the relevant 1570 * inc_tlb_flush_pending(). 1571 * 1572 * We are not sure a pending tlb flush here is for a huge page 1573 * mapping or not. Hence use the tlb range variant 1574 */ 1575 if (mm_tlb_flush_pending(vma->vm_mm)) { 1576 flush_tlb_range(vma, haddr, haddr + HPAGE_PMD_SIZE); 1577 /* 1578 * change_huge_pmd() released the pmd lock before 1579 * invalidating the secondary MMUs sharing the primary 1580 * MMU pagetables (with ->invalidate_range()). The 1581 * mmu_notifier_invalidate_range_end() (which 1582 * internally calls ->invalidate_range()) in 1583 * change_pmd_range() will run after us, so we can't 1584 * rely on it here and we need an explicit invalidate. 1585 */ 1586 mmu_notifier_invalidate_range(vma->vm_mm, haddr, 1587 haddr + HPAGE_PMD_SIZE); 1588 } 1589 1590 /* 1591 * Migrate the THP to the requested node, returns with page unlocked 1592 * and access rights restored. 1593 */ 1594 spin_unlock(vmf->ptl); 1595 1596 migrated = migrate_misplaced_transhuge_page(vma->vm_mm, vma, 1597 vmf->pmd, pmd, vmf->address, page, target_nid); 1598 if (migrated) { 1599 flags |= TNF_MIGRATED; 1600 page_nid = target_nid; 1601 } else 1602 flags |= TNF_MIGRATE_FAIL; 1603 1604 goto out; 1605 clear_pmdnuma: 1606 BUG_ON(!PageLocked(page)); 1607 was_writable = pmd_savedwrite(pmd); 1608 pmd = pmd_modify(pmd, vma->vm_page_prot); 1609 pmd = pmd_mkyoung(pmd); 1610 if (was_writable) 1611 pmd = pmd_mkwrite(pmd); 1612 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd); 1613 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); 1614 unlock_page(page); 1615 out_unlock: 1616 spin_unlock(vmf->ptl); 1617 1618 out: 1619 if (anon_vma) 1620 page_unlock_anon_vma_read(anon_vma); 1621 1622 if (page_nid != NUMA_NO_NODE) 1623 task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, 1624 flags); 1625 1626 return 0; 1627 } 1628 1629 /* 1630 * Return true if we do MADV_FREE successfully on entire pmd page. 1631 * Otherwise, return false. 1632 */ 1633 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1634 pmd_t *pmd, unsigned long addr, unsigned long next) 1635 { 1636 spinlock_t *ptl; 1637 pmd_t orig_pmd; 1638 struct page *page; 1639 struct mm_struct *mm = tlb->mm; 1640 bool ret = false; 1641 1642 tlb_remove_check_page_size_change(tlb, HPAGE_PMD_SIZE); 1643 1644 ptl = pmd_trans_huge_lock(pmd, vma); 1645 if (!ptl) 1646 goto out_unlocked; 1647 1648 orig_pmd = *pmd; 1649 if (is_huge_zero_pmd(orig_pmd)) 1650 goto out; 1651 1652 if (unlikely(!pmd_present(orig_pmd))) { 1653 VM_BUG_ON(thp_migration_supported() && 1654 !is_pmd_migration_entry(orig_pmd)); 1655 goto out; 1656 } 1657 1658 page = pmd_page(orig_pmd); 1659 /* 1660 * If other processes are mapping this page, we couldn't discard 1661 * the page unless they all do MADV_FREE so let's skip the page. 1662 */ 1663 if (page_mapcount(page) != 1) 1664 goto out; 1665 1666 if (!trylock_page(page)) 1667 goto out; 1668 1669 /* 1670 * If user want to discard part-pages of THP, split it so MADV_FREE 1671 * will deactivate only them. 1672 */ 1673 if (next - addr != HPAGE_PMD_SIZE) { 1674 get_page(page); 1675 spin_unlock(ptl); 1676 split_huge_page(page); 1677 unlock_page(page); 1678 put_page(page); 1679 goto out_unlocked; 1680 } 1681 1682 if (PageDirty(page)) 1683 ClearPageDirty(page); 1684 unlock_page(page); 1685 1686 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) { 1687 pmdp_invalidate(vma, addr, pmd); 1688 orig_pmd = pmd_mkold(orig_pmd); 1689 orig_pmd = pmd_mkclean(orig_pmd); 1690 1691 set_pmd_at(mm, addr, pmd, orig_pmd); 1692 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 1693 } 1694 1695 mark_page_lazyfree(page); 1696 ret = true; 1697 out: 1698 spin_unlock(ptl); 1699 out_unlocked: 1700 return ret; 1701 } 1702 1703 static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd) 1704 { 1705 pgtable_t pgtable; 1706 1707 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 1708 pte_free(mm, pgtable); 1709 mm_dec_nr_ptes(mm); 1710 } 1711 1712 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, 1713 pmd_t *pmd, unsigned long addr) 1714 { 1715 pmd_t orig_pmd; 1716 spinlock_t *ptl; 1717 1718 tlb_remove_check_page_size_change(tlb, HPAGE_PMD_SIZE); 1719 1720 ptl = __pmd_trans_huge_lock(pmd, vma); 1721 if (!ptl) 1722 return 0; 1723 /* 1724 * For architectures like ppc64 we look at deposited pgtable 1725 * when calling pmdp_huge_get_and_clear. So do the 1726 * pgtable_trans_huge_withdraw after finishing pmdp related 1727 * operations. 1728 */ 1729 orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd, 1730 tlb->fullmm); 1731 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 1732 if (vma_is_dax(vma)) { 1733 if (arch_needs_pgtable_deposit()) 1734 zap_deposited_table(tlb->mm, pmd); 1735 spin_unlock(ptl); 1736 if (is_huge_zero_pmd(orig_pmd)) 1737 tlb_remove_page_size(tlb, pmd_page(orig_pmd), HPAGE_PMD_SIZE); 1738 } else if (is_huge_zero_pmd(orig_pmd)) { 1739 zap_deposited_table(tlb->mm, pmd); 1740 spin_unlock(ptl); 1741 tlb_remove_page_size(tlb, pmd_page(orig_pmd), HPAGE_PMD_SIZE); 1742 } else { 1743 struct page *page = NULL; 1744 int flush_needed = 1; 1745 1746 if (pmd_present(orig_pmd)) { 1747 page = pmd_page(orig_pmd); 1748 page_remove_rmap(page, true); 1749 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page); 1750 VM_BUG_ON_PAGE(!PageHead(page), page); 1751 } else if (thp_migration_supported()) { 1752 swp_entry_t entry; 1753 1754 VM_BUG_ON(!is_pmd_migration_entry(orig_pmd)); 1755 entry = pmd_to_swp_entry(orig_pmd); 1756 page = pfn_to_page(swp_offset(entry)); 1757 flush_needed = 0; 1758 } else 1759 WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!"); 1760 1761 if (PageAnon(page)) { 1762 zap_deposited_table(tlb->mm, pmd); 1763 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); 1764 } else { 1765 if (arch_needs_pgtable_deposit()) 1766 zap_deposited_table(tlb->mm, pmd); 1767 add_mm_counter(tlb->mm, mm_counter_file(page), -HPAGE_PMD_NR); 1768 } 1769 1770 spin_unlock(ptl); 1771 if (flush_needed) 1772 tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE); 1773 } 1774 return 1; 1775 } 1776 1777 #ifndef pmd_move_must_withdraw 1778 static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl, 1779 spinlock_t *old_pmd_ptl, 1780 struct vm_area_struct *vma) 1781 { 1782 /* 1783 * With split pmd lock we also need to move preallocated 1784 * PTE page table if new_pmd is on different PMD page table. 1785 * 1786 * We also don't deposit and withdraw tables for file pages. 1787 */ 1788 return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma); 1789 } 1790 #endif 1791 1792 static pmd_t move_soft_dirty_pmd(pmd_t pmd) 1793 { 1794 #ifdef CONFIG_MEM_SOFT_DIRTY 1795 if (unlikely(is_pmd_migration_entry(pmd))) 1796 pmd = pmd_swp_mksoft_dirty(pmd); 1797 else if (pmd_present(pmd)) 1798 pmd = pmd_mksoft_dirty(pmd); 1799 #endif 1800 return pmd; 1801 } 1802 1803 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, 1804 unsigned long new_addr, unsigned long old_end, 1805 pmd_t *old_pmd, pmd_t *new_pmd) 1806 { 1807 spinlock_t *old_ptl, *new_ptl; 1808 pmd_t pmd; 1809 struct mm_struct *mm = vma->vm_mm; 1810 bool force_flush = false; 1811 1812 if ((old_addr & ~HPAGE_PMD_MASK) || 1813 (new_addr & ~HPAGE_PMD_MASK) || 1814 old_end - old_addr < HPAGE_PMD_SIZE) 1815 return false; 1816 1817 /* 1818 * The destination pmd shouldn't be established, free_pgtables() 1819 * should have release it. 1820 */ 1821 if (WARN_ON(!pmd_none(*new_pmd))) { 1822 VM_BUG_ON(pmd_trans_huge(*new_pmd)); 1823 return false; 1824 } 1825 1826 /* 1827 * We don't have to worry about the ordering of src and dst 1828 * ptlocks because exclusive mmap_sem prevents deadlock. 1829 */ 1830 old_ptl = __pmd_trans_huge_lock(old_pmd, vma); 1831 if (old_ptl) { 1832 new_ptl = pmd_lockptr(mm, new_pmd); 1833 if (new_ptl != old_ptl) 1834 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); 1835 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); 1836 if (pmd_present(pmd)) 1837 force_flush = true; 1838 VM_BUG_ON(!pmd_none(*new_pmd)); 1839 1840 if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) { 1841 pgtable_t pgtable; 1842 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd); 1843 pgtable_trans_huge_deposit(mm, new_pmd, pgtable); 1844 } 1845 pmd = move_soft_dirty_pmd(pmd); 1846 set_pmd_at(mm, new_addr, new_pmd, pmd); 1847 if (force_flush) 1848 flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE); 1849 if (new_ptl != old_ptl) 1850 spin_unlock(new_ptl); 1851 spin_unlock(old_ptl); 1852 return true; 1853 } 1854 return false; 1855 } 1856 1857 /* 1858 * Returns 1859 * - 0 if PMD could not be locked 1860 * - 1 if PMD was locked but protections unchange and TLB flush unnecessary 1861 * - HPAGE_PMD_NR is protections changed and TLB flush necessary 1862 */ 1863 int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 1864 unsigned long addr, pgprot_t newprot, int prot_numa) 1865 { 1866 struct mm_struct *mm = vma->vm_mm; 1867 spinlock_t *ptl; 1868 pmd_t entry; 1869 bool preserve_write; 1870 int ret; 1871 1872 ptl = __pmd_trans_huge_lock(pmd, vma); 1873 if (!ptl) 1874 return 0; 1875 1876 preserve_write = prot_numa && pmd_write(*pmd); 1877 ret = 1; 1878 1879 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 1880 if (is_swap_pmd(*pmd)) { 1881 swp_entry_t entry = pmd_to_swp_entry(*pmd); 1882 1883 VM_BUG_ON(!is_pmd_migration_entry(*pmd)); 1884 if (is_write_migration_entry(entry)) { 1885 pmd_t newpmd; 1886 /* 1887 * A protection check is difficult so 1888 * just be safe and disable write 1889 */ 1890 make_migration_entry_read(&entry); 1891 newpmd = swp_entry_to_pmd(entry); 1892 if (pmd_swp_soft_dirty(*pmd)) 1893 newpmd = pmd_swp_mksoft_dirty(newpmd); 1894 set_pmd_at(mm, addr, pmd, newpmd); 1895 } 1896 goto unlock; 1897 } 1898 #endif 1899 1900 /* 1901 * Avoid trapping faults against the zero page. The read-only 1902 * data is likely to be read-cached on the local CPU and 1903 * local/remote hits to the zero page are not interesting. 1904 */ 1905 if (prot_numa && is_huge_zero_pmd(*pmd)) 1906 goto unlock; 1907 1908 if (prot_numa && pmd_protnone(*pmd)) 1909 goto unlock; 1910 1911 /* 1912 * In case prot_numa, we are under down_read(mmap_sem). It's critical 1913 * to not clear pmd intermittently to avoid race with MADV_DONTNEED 1914 * which is also under down_read(mmap_sem): 1915 * 1916 * CPU0: CPU1: 1917 * change_huge_pmd(prot_numa=1) 1918 * pmdp_huge_get_and_clear_notify() 1919 * madvise_dontneed() 1920 * zap_pmd_range() 1921 * pmd_trans_huge(*pmd) == 0 (without ptl) 1922 * // skip the pmd 1923 * set_pmd_at(); 1924 * // pmd is re-established 1925 * 1926 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it 1927 * which may break userspace. 1928 * 1929 * pmdp_invalidate() is required to make sure we don't miss 1930 * dirty/young flags set by hardware. 1931 */ 1932 entry = pmdp_invalidate(vma, addr, pmd); 1933 1934 entry = pmd_modify(entry, newprot); 1935 if (preserve_write) 1936 entry = pmd_mk_savedwrite(entry); 1937 ret = HPAGE_PMD_NR; 1938 set_pmd_at(mm, addr, pmd, entry); 1939 BUG_ON(vma_is_anonymous(vma) && !preserve_write && pmd_write(entry)); 1940 unlock: 1941 spin_unlock(ptl); 1942 return ret; 1943 } 1944 1945 /* 1946 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise. 1947 * 1948 * Note that if it returns page table lock pointer, this routine returns without 1949 * unlocking page table lock. So callers must unlock it. 1950 */ 1951 spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) 1952 { 1953 spinlock_t *ptl; 1954 ptl = pmd_lock(vma->vm_mm, pmd); 1955 if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || 1956 pmd_devmap(*pmd))) 1957 return ptl; 1958 spin_unlock(ptl); 1959 return NULL; 1960 } 1961 1962 /* 1963 * Returns true if a given pud maps a thp, false otherwise. 1964 * 1965 * Note that if it returns true, this routine returns without unlocking page 1966 * table lock. So callers must unlock it. 1967 */ 1968 spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma) 1969 { 1970 spinlock_t *ptl; 1971 1972 ptl = pud_lock(vma->vm_mm, pud); 1973 if (likely(pud_trans_huge(*pud) || pud_devmap(*pud))) 1974 return ptl; 1975 spin_unlock(ptl); 1976 return NULL; 1977 } 1978 1979 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 1980 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, 1981 pud_t *pud, unsigned long addr) 1982 { 1983 pud_t orig_pud; 1984 spinlock_t *ptl; 1985 1986 ptl = __pud_trans_huge_lock(pud, vma); 1987 if (!ptl) 1988 return 0; 1989 /* 1990 * For architectures like ppc64 we look at deposited pgtable 1991 * when calling pudp_huge_get_and_clear. So do the 1992 * pgtable_trans_huge_withdraw after finishing pudp related 1993 * operations. 1994 */ 1995 orig_pud = pudp_huge_get_and_clear_full(tlb->mm, addr, pud, 1996 tlb->fullmm); 1997 tlb_remove_pud_tlb_entry(tlb, pud, addr); 1998 if (vma_is_dax(vma)) { 1999 spin_unlock(ptl); 2000 /* No zero page support yet */ 2001 } else { 2002 /* No support for anonymous PUD pages yet */ 2003 BUG(); 2004 } 2005 return 1; 2006 } 2007 2008 static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud, 2009 unsigned long haddr) 2010 { 2011 VM_BUG_ON(haddr & ~HPAGE_PUD_MASK); 2012 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2013 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma); 2014 VM_BUG_ON(!pud_trans_huge(*pud) && !pud_devmap(*pud)); 2015 2016 count_vm_event(THP_SPLIT_PUD); 2017 2018 pudp_huge_clear_flush_notify(vma, haddr, pud); 2019 } 2020 2021 void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, 2022 unsigned long address) 2023 { 2024 spinlock_t *ptl; 2025 struct mmu_notifier_range range; 2026 2027 mmu_notifier_range_init(&range, vma->vm_mm, address & HPAGE_PUD_MASK, 2028 (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE); 2029 mmu_notifier_invalidate_range_start(&range); 2030 ptl = pud_lock(vma->vm_mm, pud); 2031 if (unlikely(!pud_trans_huge(*pud) && !pud_devmap(*pud))) 2032 goto out; 2033 __split_huge_pud_locked(vma, pud, range.start); 2034 2035 out: 2036 spin_unlock(ptl); 2037 /* 2038 * No need to double call mmu_notifier->invalidate_range() callback as 2039 * the above pudp_huge_clear_flush_notify() did already call it. 2040 */ 2041 mmu_notifier_invalidate_range_only_end(&range); 2042 } 2043 #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 2044 2045 static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, 2046 unsigned long haddr, pmd_t *pmd) 2047 { 2048 struct mm_struct *mm = vma->vm_mm; 2049 pgtable_t pgtable; 2050 pmd_t _pmd; 2051 int i; 2052 2053 /* 2054 * Leave pmd empty until pte is filled note that it is fine to delay 2055 * notification until mmu_notifier_invalidate_range_end() as we are 2056 * replacing a zero pmd write protected page with a zero pte write 2057 * protected page. 2058 * 2059 * See Documentation/vm/mmu_notifier.rst 2060 */ 2061 pmdp_huge_clear_flush(vma, haddr, pmd); 2062 2063 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2064 pmd_populate(mm, &_pmd, pgtable); 2065 2066 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 2067 pte_t *pte, entry; 2068 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot); 2069 entry = pte_mkspecial(entry); 2070 pte = pte_offset_map(&_pmd, haddr); 2071 VM_BUG_ON(!pte_none(*pte)); 2072 set_pte_at(mm, haddr, pte, entry); 2073 pte_unmap(pte); 2074 } 2075 smp_wmb(); /* make pte visible before pmd */ 2076 pmd_populate(mm, pmd, pgtable); 2077 } 2078 2079 static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, 2080 unsigned long haddr, bool freeze) 2081 { 2082 struct mm_struct *mm = vma->vm_mm; 2083 struct page *page; 2084 pgtable_t pgtable; 2085 pmd_t old_pmd, _pmd; 2086 bool young, write, soft_dirty, pmd_migration = false; 2087 unsigned long addr; 2088 int i; 2089 2090 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); 2091 VM_BUG_ON_VMA(vma->vm_start > haddr, vma); 2092 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); 2093 VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd) 2094 && !pmd_devmap(*pmd)); 2095 2096 count_vm_event(THP_SPLIT_PMD); 2097 2098 if (!vma_is_anonymous(vma)) { 2099 _pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd); 2100 /* 2101 * We are going to unmap this huge page. So 2102 * just go ahead and zap it 2103 */ 2104 if (arch_needs_pgtable_deposit()) 2105 zap_deposited_table(mm, pmd); 2106 if (vma_is_dax(vma)) 2107 return; 2108 page = pmd_page(_pmd); 2109 if (!PageDirty(page) && pmd_dirty(_pmd)) 2110 set_page_dirty(page); 2111 if (!PageReferenced(page) && pmd_young(_pmd)) 2112 SetPageReferenced(page); 2113 page_remove_rmap(page, true); 2114 put_page(page); 2115 add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR); 2116 return; 2117 } else if (is_huge_zero_pmd(*pmd)) { 2118 /* 2119 * FIXME: Do we want to invalidate secondary mmu by calling 2120 * mmu_notifier_invalidate_range() see comments below inside 2121 * __split_huge_pmd() ? 2122 * 2123 * We are going from a zero huge page write protected to zero 2124 * small page also write protected so it does not seems useful 2125 * to invalidate secondary mmu at this time. 2126 */ 2127 return __split_huge_zero_page_pmd(vma, haddr, pmd); 2128 } 2129 2130 /* 2131 * Up to this point the pmd is present and huge and userland has the 2132 * whole access to the hugepage during the split (which happens in 2133 * place). If we overwrite the pmd with the not-huge version pointing 2134 * to the pte here (which of course we could if all CPUs were bug 2135 * free), userland could trigger a small page size TLB miss on the 2136 * small sized TLB while the hugepage TLB entry is still established in 2137 * the huge TLB. Some CPU doesn't like that. 2138 * See http://support.amd.com/us/Processor_TechDocs/41322.pdf, Erratum 2139 * 383 on page 93. Intel should be safe but is also warns that it's 2140 * only safe if the permission and cache attributes of the two entries 2141 * loaded in the two TLB is identical (which should be the case here). 2142 * But it is generally safer to never allow small and huge TLB entries 2143 * for the same virtual address to be loaded simultaneously. So instead 2144 * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the 2145 * current pmd notpresent (atomically because here the pmd_trans_huge 2146 * must remain set at all times on the pmd until the split is complete 2147 * for this pmd), then we flush the SMP TLB and finally we write the 2148 * non-huge version of the pmd entry with pmd_populate. 2149 */ 2150 old_pmd = pmdp_invalidate(vma, haddr, pmd); 2151 2152 pmd_migration = is_pmd_migration_entry(old_pmd); 2153 if (unlikely(pmd_migration)) { 2154 swp_entry_t entry; 2155 2156 entry = pmd_to_swp_entry(old_pmd); 2157 page = pfn_to_page(swp_offset(entry)); 2158 write = is_write_migration_entry(entry); 2159 young = false; 2160 soft_dirty = pmd_swp_soft_dirty(old_pmd); 2161 } else { 2162 page = pmd_page(old_pmd); 2163 if (pmd_dirty(old_pmd)) 2164 SetPageDirty(page); 2165 write = pmd_write(old_pmd); 2166 young = pmd_young(old_pmd); 2167 soft_dirty = pmd_soft_dirty(old_pmd); 2168 } 2169 VM_BUG_ON_PAGE(!page_count(page), page); 2170 page_ref_add(page, HPAGE_PMD_NR - 1); 2171 2172 /* 2173 * Withdraw the table only after we mark the pmd entry invalid. 2174 * This's critical for some architectures (Power). 2175 */ 2176 pgtable = pgtable_trans_huge_withdraw(mm, pmd); 2177 pmd_populate(mm, &_pmd, pgtable); 2178 2179 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { 2180 pte_t entry, *pte; 2181 /* 2182 * Note that NUMA hinting access restrictions are not 2183 * transferred to avoid any possibility of altering 2184 * permissions across VMAs. 2185 */ 2186 if (freeze || pmd_migration) { 2187 swp_entry_t swp_entry; 2188 swp_entry = make_migration_entry(page + i, write); 2189 entry = swp_entry_to_pte(swp_entry); 2190 if (soft_dirty) 2191 entry = pte_swp_mksoft_dirty(entry); 2192 } else { 2193 entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot)); 2194 entry = maybe_mkwrite(entry, vma); 2195 if (!write) 2196 entry = pte_wrprotect(entry); 2197 if (!young) 2198 entry = pte_mkold(entry); 2199 if (soft_dirty) 2200 entry = pte_mksoft_dirty(entry); 2201 } 2202 pte = pte_offset_map(&_pmd, addr); 2203 BUG_ON(!pte_none(*pte)); 2204 set_pte_at(mm, addr, pte, entry); 2205 atomic_inc(&page[i]._mapcount); 2206 pte_unmap(pte); 2207 } 2208 2209 /* 2210 * Set PG_double_map before dropping compound_mapcount to avoid 2211 * false-negative page_mapped(). 2212 */ 2213 if (compound_mapcount(page) > 1 && !TestSetPageDoubleMap(page)) { 2214 for (i = 0; i < HPAGE_PMD_NR; i++) 2215 atomic_inc(&page[i]._mapcount); 2216 } 2217 2218 if (atomic_add_negative(-1, compound_mapcount_ptr(page))) { 2219 /* Last compound_mapcount is gone. */ 2220 __dec_node_page_state(page, NR_ANON_THPS); 2221 if (TestClearPageDoubleMap(page)) { 2222 /* No need in mapcount reference anymore */ 2223 for (i = 0; i < HPAGE_PMD_NR; i++) 2224 atomic_dec(&page[i]._mapcount); 2225 } 2226 } 2227 2228 smp_wmb(); /* make pte visible before pmd */ 2229 pmd_populate(mm, pmd, pgtable); 2230 2231 if (freeze) { 2232 for (i = 0; i < HPAGE_PMD_NR; i++) { 2233 page_remove_rmap(page + i, false); 2234 put_page(page + i); 2235 } 2236 } 2237 } 2238 2239 void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, 2240 unsigned long address, bool freeze, struct page *page) 2241 { 2242 spinlock_t *ptl; 2243 struct mmu_notifier_range range; 2244 2245 mmu_notifier_range_init(&range, vma->vm_mm, address & HPAGE_PMD_MASK, 2246 (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE); 2247 mmu_notifier_invalidate_range_start(&range); 2248 ptl = pmd_lock(vma->vm_mm, pmd); 2249 2250 /* 2251 * If caller asks to setup a migration entries, we need a page to check 2252 * pmd against. Otherwise we can end up replacing wrong page. 2253 */ 2254 VM_BUG_ON(freeze && !page); 2255 if (page && page != pmd_page(*pmd)) 2256 goto out; 2257 2258 if (pmd_trans_huge(*pmd)) { 2259 page = pmd_page(*pmd); 2260 if (PageMlocked(page)) 2261 clear_page_mlock(page); 2262 } else if (!(pmd_devmap(*pmd) || is_pmd_migration_entry(*pmd))) 2263 goto out; 2264 __split_huge_pmd_locked(vma, pmd, range.start, freeze); 2265 out: 2266 spin_unlock(ptl); 2267 /* 2268 * No need to double call mmu_notifier->invalidate_range() callback. 2269 * They are 3 cases to consider inside __split_huge_pmd_locked(): 2270 * 1) pmdp_huge_clear_flush_notify() call invalidate_range() obvious 2271 * 2) __split_huge_zero_page_pmd() read only zero page and any write 2272 * fault will trigger a flush_notify before pointing to a new page 2273 * (it is fine if the secondary mmu keeps pointing to the old zero 2274 * page in the meantime) 2275 * 3) Split a huge pmd into pte pointing to the same page. No need 2276 * to invalidate secondary tlb entry they are all still valid. 2277 * any further changes to individual pte will notify. So no need 2278 * to call mmu_notifier->invalidate_range() 2279 */ 2280 mmu_notifier_invalidate_range_only_end(&range); 2281 } 2282 2283 void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, 2284 bool freeze, struct page *page) 2285 { 2286 pgd_t *pgd; 2287 p4d_t *p4d; 2288 pud_t *pud; 2289 pmd_t *pmd; 2290 2291 pgd = pgd_offset(vma->vm_mm, address); 2292 if (!pgd_present(*pgd)) 2293 return; 2294 2295 p4d = p4d_offset(pgd, address); 2296 if (!p4d_present(*p4d)) 2297 return; 2298 2299 pud = pud_offset(p4d, address); 2300 if (!pud_present(*pud)) 2301 return; 2302 2303 pmd = pmd_offset(pud, address); 2304 2305 __split_huge_pmd(vma, pmd, address, freeze, page); 2306 } 2307 2308 void vma_adjust_trans_huge(struct vm_area_struct *vma, 2309 unsigned long start, 2310 unsigned long end, 2311 long adjust_next) 2312 { 2313 /* 2314 * If the new start address isn't hpage aligned and it could 2315 * previously contain an hugepage: check if we need to split 2316 * an huge pmd. 2317 */ 2318 if (start & ~HPAGE_PMD_MASK && 2319 (start & HPAGE_PMD_MASK) >= vma->vm_start && 2320 (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) 2321 split_huge_pmd_address(vma, start, false, NULL); 2322 2323 /* 2324 * If the new end address isn't hpage aligned and it could 2325 * previously contain an hugepage: check if we need to split 2326 * an huge pmd. 2327 */ 2328 if (end & ~HPAGE_PMD_MASK && 2329 (end & HPAGE_PMD_MASK) >= vma->vm_start && 2330 (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end) 2331 split_huge_pmd_address(vma, end, false, NULL); 2332 2333 /* 2334 * If we're also updating the vma->vm_next->vm_start, if the new 2335 * vm_next->vm_start isn't page aligned and it could previously 2336 * contain an hugepage: check if we need to split an huge pmd. 2337 */ 2338 if (adjust_next > 0) { 2339 struct vm_area_struct *next = vma->vm_next; 2340 unsigned long nstart = next->vm_start; 2341 nstart += adjust_next << PAGE_SHIFT; 2342 if (nstart & ~HPAGE_PMD_MASK && 2343 (nstart & HPAGE_PMD_MASK) >= next->vm_start && 2344 (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end) 2345 split_huge_pmd_address(next, nstart, false, NULL); 2346 } 2347 } 2348 2349 static void unmap_page(struct page *page) 2350 { 2351 enum ttu_flags ttu_flags = TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS | 2352 TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD; 2353 bool unmap_success; 2354 2355 VM_BUG_ON_PAGE(!PageHead(page), page); 2356 2357 if (PageAnon(page)) 2358 ttu_flags |= TTU_SPLIT_FREEZE; 2359 2360 unmap_success = try_to_unmap(page, ttu_flags); 2361 VM_BUG_ON_PAGE(!unmap_success, page); 2362 } 2363 2364 static void remap_page(struct page *page) 2365 { 2366 int i; 2367 if (PageTransHuge(page)) { 2368 remove_migration_ptes(page, page, true); 2369 } else { 2370 for (i = 0; i < HPAGE_PMD_NR; i++) 2371 remove_migration_ptes(page + i, page + i, true); 2372 } 2373 } 2374 2375 static void __split_huge_page_tail(struct page *head, int tail, 2376 struct lruvec *lruvec, struct list_head *list) 2377 { 2378 struct page *page_tail = head + tail; 2379 2380 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail); 2381 2382 /* 2383 * Clone page flags before unfreezing refcount. 2384 * 2385 * After successful get_page_unless_zero() might follow flags change, 2386 * for exmaple lock_page() which set PG_waiters. 2387 */ 2388 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; 2389 page_tail->flags |= (head->flags & 2390 ((1L << PG_referenced) | 2391 (1L << PG_swapbacked) | 2392 (1L << PG_swapcache) | 2393 (1L << PG_mlocked) | 2394 (1L << PG_uptodate) | 2395 (1L << PG_active) | 2396 (1L << PG_workingset) | 2397 (1L << PG_locked) | 2398 (1L << PG_unevictable) | 2399 (1L << PG_dirty))); 2400 2401 /* ->mapping in first tail page is compound_mapcount */ 2402 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, 2403 page_tail); 2404 page_tail->mapping = head->mapping; 2405 page_tail->index = head->index + tail; 2406 2407 /* Page flags must be visible before we make the page non-compound. */ 2408 smp_wmb(); 2409 2410 /* 2411 * Clear PageTail before unfreezing page refcount. 2412 * 2413 * After successful get_page_unless_zero() might follow put_page() 2414 * which needs correct compound_head(). 2415 */ 2416 clear_compound_head(page_tail); 2417 2418 /* Finally unfreeze refcount. Additional reference from page cache. */ 2419 page_ref_unfreeze(page_tail, 1 + (!PageAnon(head) || 2420 PageSwapCache(head))); 2421 2422 if (page_is_young(head)) 2423 set_page_young(page_tail); 2424 if (page_is_idle(head)) 2425 set_page_idle(page_tail); 2426 2427 page_cpupid_xchg_last(page_tail, page_cpupid_last(head)); 2428 2429 /* 2430 * always add to the tail because some iterators expect new 2431 * pages to show after the currently processed elements - e.g. 2432 * migrate_pages 2433 */ 2434 lru_add_page_tail(head, page_tail, lruvec, list); 2435 } 2436 2437 static void __split_huge_page(struct page *page, struct list_head *list, 2438 pgoff_t end, unsigned long flags) 2439 { 2440 struct page *head = compound_head(page); 2441 struct zone *zone = page_zone(head); 2442 struct lruvec *lruvec; 2443 int i; 2444 2445 lruvec = mem_cgroup_page_lruvec(head, zone->zone_pgdat); 2446 2447 /* complete memcg works before add pages to LRU */ 2448 mem_cgroup_split_huge_fixup(head); 2449 2450 for (i = HPAGE_PMD_NR - 1; i >= 1; i--) { 2451 __split_huge_page_tail(head, i, lruvec, list); 2452 /* Some pages can be beyond i_size: drop them from page cache */ 2453 if (head[i].index >= end) { 2454 ClearPageDirty(head + i); 2455 __delete_from_page_cache(head + i, NULL); 2456 if (IS_ENABLED(CONFIG_SHMEM) && PageSwapBacked(head)) 2457 shmem_uncharge(head->mapping->host, 1); 2458 put_page(head + i); 2459 } 2460 } 2461 2462 ClearPageCompound(head); 2463 /* See comment in __split_huge_page_tail() */ 2464 if (PageAnon(head)) { 2465 /* Additional pin to swap cache */ 2466 if (PageSwapCache(head)) 2467 page_ref_add(head, 2); 2468 else 2469 page_ref_inc(head); 2470 } else { 2471 /* Additional pin to page cache */ 2472 page_ref_add(head, 2); 2473 xa_unlock(&head->mapping->i_pages); 2474 } 2475 2476 spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags); 2477 2478 remap_page(head); 2479 2480 for (i = 0; i < HPAGE_PMD_NR; i++) { 2481 struct page *subpage = head + i; 2482 if (subpage == page) 2483 continue; 2484 unlock_page(subpage); 2485 2486 /* 2487 * Subpages may be freed if there wasn't any mapping 2488 * like if add_to_swap() is running on a lru page that 2489 * had its mapping zapped. And freeing these pages 2490 * requires taking the lru_lock so we do the put_page 2491 * of the tail pages after the split is complete. 2492 */ 2493 put_page(subpage); 2494 } 2495 } 2496 2497 int total_mapcount(struct page *page) 2498 { 2499 int i, compound, ret; 2500 2501 VM_BUG_ON_PAGE(PageTail(page), page); 2502 2503 if (likely(!PageCompound(page))) 2504 return atomic_read(&page->_mapcount) + 1; 2505 2506 compound = compound_mapcount(page); 2507 if (PageHuge(page)) 2508 return compound; 2509 ret = compound; 2510 for (i = 0; i < HPAGE_PMD_NR; i++) 2511 ret += atomic_read(&page[i]._mapcount) + 1; 2512 /* File pages has compound_mapcount included in _mapcount */ 2513 if (!PageAnon(page)) 2514 return ret - compound * HPAGE_PMD_NR; 2515 if (PageDoubleMap(page)) 2516 ret -= HPAGE_PMD_NR; 2517 return ret; 2518 } 2519 2520 /* 2521 * This calculates accurately how many mappings a transparent hugepage 2522 * has (unlike page_mapcount() which isn't fully accurate). This full 2523 * accuracy is primarily needed to know if copy-on-write faults can 2524 * reuse the page and change the mapping to read-write instead of 2525 * copying them. At the same time this returns the total_mapcount too. 2526 * 2527 * The function returns the highest mapcount any one of the subpages 2528 * has. If the return value is one, even if different processes are 2529 * mapping different subpages of the transparent hugepage, they can 2530 * all reuse it, because each process is reusing a different subpage. 2531 * 2532 * The total_mapcount is instead counting all virtual mappings of the 2533 * subpages. If the total_mapcount is equal to "one", it tells the 2534 * caller all mappings belong to the same "mm" and in turn the 2535 * anon_vma of the transparent hugepage can become the vma->anon_vma 2536 * local one as no other process may be mapping any of the subpages. 2537 * 2538 * It would be more accurate to replace page_mapcount() with 2539 * page_trans_huge_mapcount(), however we only use 2540 * page_trans_huge_mapcount() in the copy-on-write faults where we 2541 * need full accuracy to avoid breaking page pinning, because 2542 * page_trans_huge_mapcount() is slower than page_mapcount(). 2543 */ 2544 int page_trans_huge_mapcount(struct page *page, int *total_mapcount) 2545 { 2546 int i, ret, _total_mapcount, mapcount; 2547 2548 /* hugetlbfs shouldn't call it */ 2549 VM_BUG_ON_PAGE(PageHuge(page), page); 2550 2551 if (likely(!PageTransCompound(page))) { 2552 mapcount = atomic_read(&page->_mapcount) + 1; 2553 if (total_mapcount) 2554 *total_mapcount = mapcount; 2555 return mapcount; 2556 } 2557 2558 page = compound_head(page); 2559 2560 _total_mapcount = ret = 0; 2561 for (i = 0; i < HPAGE_PMD_NR; i++) { 2562 mapcount = atomic_read(&page[i]._mapcount) + 1; 2563 ret = max(ret, mapcount); 2564 _total_mapcount += mapcount; 2565 } 2566 if (PageDoubleMap(page)) { 2567 ret -= 1; 2568 _total_mapcount -= HPAGE_PMD_NR; 2569 } 2570 mapcount = compound_mapcount(page); 2571 ret += mapcount; 2572 _total_mapcount += mapcount; 2573 if (total_mapcount) 2574 *total_mapcount = _total_mapcount; 2575 return ret; 2576 } 2577 2578 /* Racy check whether the huge page can be split */ 2579 bool can_split_huge_page(struct page *page, int *pextra_pins) 2580 { 2581 int extra_pins; 2582 2583 /* Additional pins from page cache */ 2584 if (PageAnon(page)) 2585 extra_pins = PageSwapCache(page) ? HPAGE_PMD_NR : 0; 2586 else 2587 extra_pins = HPAGE_PMD_NR; 2588 if (pextra_pins) 2589 *pextra_pins = extra_pins; 2590 return total_mapcount(page) == page_count(page) - extra_pins - 1; 2591 } 2592 2593 /* 2594 * This function splits huge page into normal pages. @page can point to any 2595 * subpage of huge page to split. Split doesn't change the position of @page. 2596 * 2597 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY. 2598 * The huge page must be locked. 2599 * 2600 * If @list is null, tail pages will be added to LRU list, otherwise, to @list. 2601 * 2602 * Both head page and tail pages will inherit mapping, flags, and so on from 2603 * the hugepage. 2604 * 2605 * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if 2606 * they are not mapped. 2607 * 2608 * Returns 0 if the hugepage is split successfully. 2609 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under 2610 * us. 2611 */ 2612 int split_huge_page_to_list(struct page *page, struct list_head *list) 2613 { 2614 struct page *head = compound_head(page); 2615 struct pglist_data *pgdata = NODE_DATA(page_to_nid(head)); 2616 struct anon_vma *anon_vma = NULL; 2617 struct address_space *mapping = NULL; 2618 int count, mapcount, extra_pins, ret; 2619 bool mlocked; 2620 unsigned long flags; 2621 pgoff_t end; 2622 2623 VM_BUG_ON_PAGE(is_huge_zero_page(page), page); 2624 VM_BUG_ON_PAGE(!PageLocked(page), page); 2625 VM_BUG_ON_PAGE(!PageCompound(page), page); 2626 2627 if (PageWriteback(page)) 2628 return -EBUSY; 2629 2630 if (PageAnon(head)) { 2631 /* 2632 * The caller does not necessarily hold an mmap_sem that would 2633 * prevent the anon_vma disappearing so we first we take a 2634 * reference to it and then lock the anon_vma for write. This 2635 * is similar to page_lock_anon_vma_read except the write lock 2636 * is taken to serialise against parallel split or collapse 2637 * operations. 2638 */ 2639 anon_vma = page_get_anon_vma(head); 2640 if (!anon_vma) { 2641 ret = -EBUSY; 2642 goto out; 2643 } 2644 end = -1; 2645 mapping = NULL; 2646 anon_vma_lock_write(anon_vma); 2647 } else { 2648 mapping = head->mapping; 2649 2650 /* Truncated ? */ 2651 if (!mapping) { 2652 ret = -EBUSY; 2653 goto out; 2654 } 2655 2656 anon_vma = NULL; 2657 i_mmap_lock_read(mapping); 2658 2659 /* 2660 *__split_huge_page() may need to trim off pages beyond EOF: 2661 * but on 32-bit, i_size_read() takes an irq-unsafe seqlock, 2662 * which cannot be nested inside the page tree lock. So note 2663 * end now: i_size itself may be changed at any moment, but 2664 * head page lock is good enough to serialize the trimming. 2665 */ 2666 end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); 2667 } 2668 2669 /* 2670 * Racy check if we can split the page, before unmap_page() will 2671 * split PMDs 2672 */ 2673 if (!can_split_huge_page(head, &extra_pins)) { 2674 ret = -EBUSY; 2675 goto out_unlock; 2676 } 2677 2678 mlocked = PageMlocked(page); 2679 unmap_page(head); 2680 VM_BUG_ON_PAGE(compound_mapcount(head), head); 2681 2682 /* Make sure the page is not on per-CPU pagevec as it takes pin */ 2683 if (mlocked) 2684 lru_add_drain(); 2685 2686 /* prevent PageLRU to go away from under us, and freeze lru stats */ 2687 spin_lock_irqsave(zone_lru_lock(page_zone(head)), flags); 2688 2689 if (mapping) { 2690 XA_STATE(xas, &mapping->i_pages, page_index(head)); 2691 2692 /* 2693 * Check if the head page is present in page cache. 2694 * We assume all tail are present too, if head is there. 2695 */ 2696 xa_lock(&mapping->i_pages); 2697 if (xas_load(&xas) != head) 2698 goto fail; 2699 } 2700 2701 /* Prevent deferred_split_scan() touching ->_refcount */ 2702 spin_lock(&pgdata->split_queue_lock); 2703 count = page_count(head); 2704 mapcount = total_mapcount(head); 2705 if (!mapcount && page_ref_freeze(head, 1 + extra_pins)) { 2706 if (!list_empty(page_deferred_list(head))) { 2707 pgdata->split_queue_len--; 2708 list_del(page_deferred_list(head)); 2709 } 2710 if (mapping) 2711 __dec_node_page_state(page, NR_SHMEM_THPS); 2712 spin_unlock(&pgdata->split_queue_lock); 2713 __split_huge_page(page, list, end, flags); 2714 if (PageSwapCache(head)) { 2715 swp_entry_t entry = { .val = page_private(head) }; 2716 2717 ret = split_swap_cluster(entry); 2718 } else 2719 ret = 0; 2720 } else { 2721 if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount) { 2722 pr_alert("total_mapcount: %u, page_count(): %u\n", 2723 mapcount, count); 2724 if (PageTail(page)) 2725 dump_page(head, NULL); 2726 dump_page(page, "total_mapcount(head) > 0"); 2727 BUG(); 2728 } 2729 spin_unlock(&pgdata->split_queue_lock); 2730 fail: if (mapping) 2731 xa_unlock(&mapping->i_pages); 2732 spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags); 2733 remap_page(head); 2734 ret = -EBUSY; 2735 } 2736 2737 out_unlock: 2738 if (anon_vma) { 2739 anon_vma_unlock_write(anon_vma); 2740 put_anon_vma(anon_vma); 2741 } 2742 if (mapping) 2743 i_mmap_unlock_read(mapping); 2744 out: 2745 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); 2746 return ret; 2747 } 2748 2749 void free_transhuge_page(struct page *page) 2750 { 2751 struct pglist_data *pgdata = NODE_DATA(page_to_nid(page)); 2752 unsigned long flags; 2753 2754 spin_lock_irqsave(&pgdata->split_queue_lock, flags); 2755 if (!list_empty(page_deferred_list(page))) { 2756 pgdata->split_queue_len--; 2757 list_del(page_deferred_list(page)); 2758 } 2759 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 2760 free_compound_page(page); 2761 } 2762 2763 void deferred_split_huge_page(struct page *page) 2764 { 2765 struct pglist_data *pgdata = NODE_DATA(page_to_nid(page)); 2766 unsigned long flags; 2767 2768 VM_BUG_ON_PAGE(!PageTransHuge(page), page); 2769 2770 spin_lock_irqsave(&pgdata->split_queue_lock, flags); 2771 if (list_empty(page_deferred_list(page))) { 2772 count_vm_event(THP_DEFERRED_SPLIT_PAGE); 2773 list_add_tail(page_deferred_list(page), &pgdata->split_queue); 2774 pgdata->split_queue_len++; 2775 } 2776 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 2777 } 2778 2779 static unsigned long deferred_split_count(struct shrinker *shrink, 2780 struct shrink_control *sc) 2781 { 2782 struct pglist_data *pgdata = NODE_DATA(sc->nid); 2783 return READ_ONCE(pgdata->split_queue_len); 2784 } 2785 2786 static unsigned long deferred_split_scan(struct shrinker *shrink, 2787 struct shrink_control *sc) 2788 { 2789 struct pglist_data *pgdata = NODE_DATA(sc->nid); 2790 unsigned long flags; 2791 LIST_HEAD(list), *pos, *next; 2792 struct page *page; 2793 int split = 0; 2794 2795 spin_lock_irqsave(&pgdata->split_queue_lock, flags); 2796 /* Take pin on all head pages to avoid freeing them under us */ 2797 list_for_each_safe(pos, next, &pgdata->split_queue) { 2798 page = list_entry((void *)pos, struct page, mapping); 2799 page = compound_head(page); 2800 if (get_page_unless_zero(page)) { 2801 list_move(page_deferred_list(page), &list); 2802 } else { 2803 /* We lost race with put_compound_page() */ 2804 list_del_init(page_deferred_list(page)); 2805 pgdata->split_queue_len--; 2806 } 2807 if (!--sc->nr_to_scan) 2808 break; 2809 } 2810 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 2811 2812 list_for_each_safe(pos, next, &list) { 2813 page = list_entry((void *)pos, struct page, mapping); 2814 if (!trylock_page(page)) 2815 goto next; 2816 /* split_huge_page() removes page from list on success */ 2817 if (!split_huge_page(page)) 2818 split++; 2819 unlock_page(page); 2820 next: 2821 put_page(page); 2822 } 2823 2824 spin_lock_irqsave(&pgdata->split_queue_lock, flags); 2825 list_splice_tail(&list, &pgdata->split_queue); 2826 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags); 2827 2828 /* 2829 * Stop shrinker if we didn't split any page, but the queue is empty. 2830 * This can happen if pages were freed under us. 2831 */ 2832 if (!split && list_empty(&pgdata->split_queue)) 2833 return SHRINK_STOP; 2834 return split; 2835 } 2836 2837 static struct shrinker deferred_split_shrinker = { 2838 .count_objects = deferred_split_count, 2839 .scan_objects = deferred_split_scan, 2840 .seeks = DEFAULT_SEEKS, 2841 .flags = SHRINKER_NUMA_AWARE, 2842 }; 2843 2844 #ifdef CONFIG_DEBUG_FS 2845 static int split_huge_pages_set(void *data, u64 val) 2846 { 2847 struct zone *zone; 2848 struct page *page; 2849 unsigned long pfn, max_zone_pfn; 2850 unsigned long total = 0, split = 0; 2851 2852 if (val != 1) 2853 return -EINVAL; 2854 2855 for_each_populated_zone(zone) { 2856 max_zone_pfn = zone_end_pfn(zone); 2857 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { 2858 if (!pfn_valid(pfn)) 2859 continue; 2860 2861 page = pfn_to_page(pfn); 2862 if (!get_page_unless_zero(page)) 2863 continue; 2864 2865 if (zone != page_zone(page)) 2866 goto next; 2867 2868 if (!PageHead(page) || PageHuge(page) || !PageLRU(page)) 2869 goto next; 2870 2871 total++; 2872 lock_page(page); 2873 if (!split_huge_page(page)) 2874 split++; 2875 unlock_page(page); 2876 next: 2877 put_page(page); 2878 } 2879 } 2880 2881 pr_info("%lu of %lu THP split\n", split, total); 2882 2883 return 0; 2884 } 2885 DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set, 2886 "%llu\n"); 2887 2888 static int __init split_huge_pages_debugfs(void) 2889 { 2890 debugfs_create_file("split_huge_pages", 0200, NULL, NULL, 2891 &split_huge_pages_fops); 2892 return 0; 2893 } 2894 late_initcall(split_huge_pages_debugfs); 2895 #endif 2896 2897 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 2898 void set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 2899 struct page *page) 2900 { 2901 struct vm_area_struct *vma = pvmw->vma; 2902 struct mm_struct *mm = vma->vm_mm; 2903 unsigned long address = pvmw->address; 2904 pmd_t pmdval; 2905 swp_entry_t entry; 2906 pmd_t pmdswp; 2907 2908 if (!(pvmw->pmd && !pvmw->pte)) 2909 return; 2910 2911 flush_cache_range(vma, address, address + HPAGE_PMD_SIZE); 2912 pmdval = *pvmw->pmd; 2913 pmdp_invalidate(vma, address, pvmw->pmd); 2914 if (pmd_dirty(pmdval)) 2915 set_page_dirty(page); 2916 entry = make_migration_entry(page, pmd_write(pmdval)); 2917 pmdswp = swp_entry_to_pmd(entry); 2918 if (pmd_soft_dirty(pmdval)) 2919 pmdswp = pmd_swp_mksoft_dirty(pmdswp); 2920 set_pmd_at(mm, address, pvmw->pmd, pmdswp); 2921 page_remove_rmap(page, true); 2922 put_page(page); 2923 } 2924 2925 void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) 2926 { 2927 struct vm_area_struct *vma = pvmw->vma; 2928 struct mm_struct *mm = vma->vm_mm; 2929 unsigned long address = pvmw->address; 2930 unsigned long mmun_start = address & HPAGE_PMD_MASK; 2931 pmd_t pmde; 2932 swp_entry_t entry; 2933 2934 if (!(pvmw->pmd && !pvmw->pte)) 2935 return; 2936 2937 entry = pmd_to_swp_entry(*pvmw->pmd); 2938 get_page(new); 2939 pmde = pmd_mkold(mk_huge_pmd(new, vma->vm_page_prot)); 2940 if (pmd_swp_soft_dirty(*pvmw->pmd)) 2941 pmde = pmd_mksoft_dirty(pmde); 2942 if (is_write_migration_entry(entry)) 2943 pmde = maybe_pmd_mkwrite(pmde, vma); 2944 2945 flush_cache_range(vma, mmun_start, mmun_start + HPAGE_PMD_SIZE); 2946 if (PageAnon(new)) 2947 page_add_anon_rmap(new, vma, mmun_start, true); 2948 else 2949 page_add_file_rmap(new, true); 2950 set_pmd_at(mm, mmun_start, pvmw->pmd, pmde); 2951 if ((vma->vm_flags & VM_LOCKED) && !PageDoubleMap(new)) 2952 mlock_vma_page(new); 2953 update_mmu_cache_pmd(vma, address, pvmw->pmd); 2954 } 2955 #endif 2956