1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_HUGETLB_H 3 #define _LINUX_HUGETLB_H 4 5 #include <linux/mm.h> 6 #include <linux/mm_types.h> 7 #include <linux/mmdebug.h> 8 #include <linux/fs.h> 9 #include <linux/hugetlb_inline.h> 10 #include <linux/cgroup.h> 11 #include <linux/page_ref.h> 12 #include <linux/list.h> 13 #include <linux/kref.h> 14 #include <linux/pgtable.h> 15 #include <linux/gfp.h> 16 #include <linux/userfaultfd_k.h> 17 18 struct ctl_table; 19 struct user_struct; 20 struct mmu_gather; 21 struct node; 22 23 #ifndef CONFIG_ARCH_HAS_HUGEPD 24 typedef struct { unsigned long pd; } hugepd_t; 25 #define is_hugepd(hugepd) (0) 26 #define __hugepd(x) ((hugepd_t) { (x) }) 27 #endif 28 29 void free_huge_folio(struct folio *folio); 30 31 #ifdef CONFIG_HUGETLB_PAGE 32 33 #include <linux/pagemap.h> 34 #include <linux/shm.h> 35 #include <asm/tlbflush.h> 36 37 /* 38 * For HugeTLB page, there are more metadata to save in the struct page. But 39 * the head struct page cannot meet our needs, so we have to abuse other tail 40 * struct page to store the metadata. 41 */ 42 #define __NR_USED_SUBPAGE 3 43 44 struct hugepage_subpool { 45 spinlock_t lock; 46 long count; 47 long max_hpages; /* Maximum huge pages or -1 if no maximum. */ 48 long used_hpages; /* Used count against maximum, includes */ 49 /* both allocated and reserved pages. */ 50 struct hstate *hstate; 51 long min_hpages; /* Minimum huge pages or -1 if no minimum. */ 52 long rsv_hpages; /* Pages reserved against global pool to */ 53 /* satisfy minimum size. */ 54 }; 55 56 struct resv_map { 57 struct kref refs; 58 spinlock_t lock; 59 struct list_head regions; 60 long adds_in_progress; 61 struct list_head region_cache; 62 long region_cache_count; 63 struct rw_semaphore rw_sema; 64 #ifdef CONFIG_CGROUP_HUGETLB 65 /* 66 * On private mappings, the counter to uncharge reservations is stored 67 * here. If these fields are 0, then either the mapping is shared, or 68 * cgroup accounting is disabled for this resv_map. 69 */ 70 struct page_counter *reservation_counter; 71 unsigned long pages_per_hpage; 72 struct cgroup_subsys_state *css; 73 #endif 74 }; 75 76 /* 77 * Region tracking -- allows tracking of reservations and instantiated pages 78 * across the pages in a mapping. 79 * 80 * The region data structures are embedded into a resv_map and protected 81 * by a resv_map's lock. The set of regions within the resv_map represent 82 * reservations for huge pages, or huge pages that have already been 83 * instantiated within the map. The from and to elements are huge page 84 * indices into the associated mapping. from indicates the starting index 85 * of the region. to represents the first index past the end of the region. 86 * 87 * For example, a file region structure with from == 0 and to == 4 represents 88 * four huge pages in a mapping. It is important to note that the to element 89 * represents the first element past the end of the region. This is used in 90 * arithmetic as 4(to) - 0(from) = 4 huge pages in the region. 91 * 92 * Interval notation of the form [from, to) will be used to indicate that 93 * the endpoint from is inclusive and to is exclusive. 94 */ 95 struct file_region { 96 struct list_head link; 97 long from; 98 long to; 99 #ifdef CONFIG_CGROUP_HUGETLB 100 /* 101 * On shared mappings, each reserved region appears as a struct 102 * file_region in resv_map. These fields hold the info needed to 103 * uncharge each reservation. 104 */ 105 struct page_counter *reservation_counter; 106 struct cgroup_subsys_state *css; 107 #endif 108 }; 109 110 struct hugetlb_vma_lock { 111 struct kref refs; 112 struct rw_semaphore rw_sema; 113 struct vm_area_struct *vma; 114 }; 115 116 extern struct resv_map *resv_map_alloc(void); 117 void resv_map_release(struct kref *ref); 118 119 extern spinlock_t hugetlb_lock; 120 extern int hugetlb_max_hstate __read_mostly; 121 #define for_each_hstate(h) \ 122 for ((h) = hstates; (h) < &hstates[hugetlb_max_hstate]; (h)++) 123 124 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages, 125 long min_hpages); 126 void hugepage_put_subpool(struct hugepage_subpool *spool); 127 128 void hugetlb_dup_vma_private(struct vm_area_struct *vma); 129 void clear_vma_resv_huge_pages(struct vm_area_struct *vma); 130 int move_hugetlb_page_tables(struct vm_area_struct *vma, 131 struct vm_area_struct *new_vma, 132 unsigned long old_addr, unsigned long new_addr, 133 unsigned long len); 134 int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *, 135 struct vm_area_struct *, struct vm_area_struct *); 136 struct page *hugetlb_follow_page_mask(struct vm_area_struct *vma, 137 unsigned long address, unsigned int flags, 138 unsigned int *page_mask); 139 void unmap_hugepage_range(struct vm_area_struct *, 140 unsigned long, unsigned long, struct page *, 141 zap_flags_t); 142 void __unmap_hugepage_range(struct mmu_gather *tlb, 143 struct vm_area_struct *vma, 144 unsigned long start, unsigned long end, 145 struct page *ref_page, zap_flags_t zap_flags); 146 void hugetlb_report_meminfo(struct seq_file *); 147 int hugetlb_report_node_meminfo(char *buf, int len, int nid); 148 void hugetlb_show_meminfo_node(int nid); 149 unsigned long hugetlb_total_pages(void); 150 vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, 151 unsigned long address, unsigned int flags); 152 #ifdef CONFIG_USERFAULTFD 153 int hugetlb_mfill_atomic_pte(pte_t *dst_pte, 154 struct vm_area_struct *dst_vma, 155 unsigned long dst_addr, 156 unsigned long src_addr, 157 uffd_flags_t flags, 158 struct folio **foliop); 159 #endif /* CONFIG_USERFAULTFD */ 160 bool hugetlb_reserve_pages(struct inode *inode, long from, long to, 161 struct vm_area_struct *vma, 162 vm_flags_t vm_flags); 163 long hugetlb_unreserve_pages(struct inode *inode, long start, long end, 164 long freed); 165 bool isolate_hugetlb(struct folio *folio, struct list_head *list); 166 int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison); 167 int get_huge_page_for_hwpoison(unsigned long pfn, int flags, 168 bool *migratable_cleared); 169 void folio_putback_active_hugetlb(struct folio *folio); 170 void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason); 171 void hugetlb_fix_reserve_counts(struct inode *inode); 172 extern struct mutex *hugetlb_fault_mutex_table; 173 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx); 174 175 pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, 176 unsigned long addr, pud_t *pud); 177 bool hugetlbfs_pagecache_present(struct hstate *h, 178 struct vm_area_struct *vma, 179 unsigned long address); 180 181 struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio); 182 183 extern int sysctl_hugetlb_shm_group; 184 extern struct list_head huge_boot_pages[MAX_NUMNODES]; 185 186 /* arch callbacks */ 187 188 #ifndef CONFIG_HIGHPTE 189 /* 190 * pte_offset_huge() and pte_alloc_huge() are helpers for those architectures 191 * which may go down to the lowest PTE level in their huge_pte_offset() and 192 * huge_pte_alloc(): to avoid reliance on pte_offset_map() without pte_unmap(). 193 */ 194 static inline pte_t *pte_offset_huge(pmd_t *pmd, unsigned long address) 195 { 196 return pte_offset_kernel(pmd, address); 197 } 198 static inline pte_t *pte_alloc_huge(struct mm_struct *mm, pmd_t *pmd, 199 unsigned long address) 200 { 201 return pte_alloc(mm, pmd) ? NULL : pte_offset_huge(pmd, address); 202 } 203 #endif 204 205 pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, 206 unsigned long addr, unsigned long sz); 207 /* 208 * huge_pte_offset(): Walk the hugetlb pgtable until the last level PTE. 209 * Returns the pte_t* if found, or NULL if the address is not mapped. 210 * 211 * IMPORTANT: we should normally not directly call this function, instead 212 * this is only a common interface to implement arch-specific 213 * walker. Please use hugetlb_walk() instead, because that will attempt to 214 * verify the locking for you. 215 * 216 * Since this function will walk all the pgtable pages (including not only 217 * high-level pgtable page, but also PUD entry that can be unshared 218 * concurrently for VM_SHARED), the caller of this function should be 219 * responsible of its thread safety. One can follow this rule: 220 * 221 * (1) For private mappings: pmd unsharing is not possible, so holding the 222 * mmap_lock for either read or write is sufficient. Most callers 223 * already hold the mmap_lock, so normally, no special action is 224 * required. 225 * 226 * (2) For shared mappings: pmd unsharing is possible (so the PUD-ranged 227 * pgtable page can go away from under us! It can be done by a pmd 228 * unshare with a follow up munmap() on the other process), then we 229 * need either: 230 * 231 * (2.1) hugetlb vma lock read or write held, to make sure pmd unshare 232 * won't happen upon the range (it also makes sure the pte_t we 233 * read is the right and stable one), or, 234 * 235 * (2.2) hugetlb mapping i_mmap_rwsem lock held read or write, to make 236 * sure even if unshare happened the racy unmap() will wait until 237 * i_mmap_rwsem is released. 238 * 239 * Option (2.1) is the safest, which guarantees pte stability from pmd 240 * sharing pov, until the vma lock released. Option (2.2) doesn't protect 241 * a concurrent pmd unshare, but it makes sure the pgtable page is safe to 242 * access. 243 */ 244 pte_t *huge_pte_offset(struct mm_struct *mm, 245 unsigned long addr, unsigned long sz); 246 unsigned long hugetlb_mask_last_page(struct hstate *h); 247 int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, 248 unsigned long addr, pte_t *ptep); 249 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, 250 unsigned long *start, unsigned long *end); 251 252 extern void __hugetlb_zap_begin(struct vm_area_struct *vma, 253 unsigned long *begin, unsigned long *end); 254 extern void __hugetlb_zap_end(struct vm_area_struct *vma, 255 struct zap_details *details); 256 257 static inline void hugetlb_zap_begin(struct vm_area_struct *vma, 258 unsigned long *start, unsigned long *end) 259 { 260 if (is_vm_hugetlb_page(vma)) 261 __hugetlb_zap_begin(vma, start, end); 262 } 263 264 static inline void hugetlb_zap_end(struct vm_area_struct *vma, 265 struct zap_details *details) 266 { 267 if (is_vm_hugetlb_page(vma)) 268 __hugetlb_zap_end(vma, details); 269 } 270 271 void hugetlb_vma_lock_read(struct vm_area_struct *vma); 272 void hugetlb_vma_unlock_read(struct vm_area_struct *vma); 273 void hugetlb_vma_lock_write(struct vm_area_struct *vma); 274 void hugetlb_vma_unlock_write(struct vm_area_struct *vma); 275 int hugetlb_vma_trylock_write(struct vm_area_struct *vma); 276 void hugetlb_vma_assert_locked(struct vm_area_struct *vma); 277 void hugetlb_vma_lock_release(struct kref *kref); 278 long hugetlb_change_protection(struct vm_area_struct *vma, 279 unsigned long address, unsigned long end, pgprot_t newprot, 280 unsigned long cp_flags); 281 bool is_hugetlb_entry_migration(pte_t pte); 282 bool is_hugetlb_entry_hwpoisoned(pte_t pte); 283 void hugetlb_unshare_all_pmds(struct vm_area_struct *vma); 284 285 #else /* !CONFIG_HUGETLB_PAGE */ 286 287 static inline void hugetlb_dup_vma_private(struct vm_area_struct *vma) 288 { 289 } 290 291 static inline void clear_vma_resv_huge_pages(struct vm_area_struct *vma) 292 { 293 } 294 295 static inline unsigned long hugetlb_total_pages(void) 296 { 297 return 0; 298 } 299 300 static inline struct address_space *hugetlb_folio_mapping_lock_write( 301 struct folio *folio) 302 { 303 return NULL; 304 } 305 306 static inline int huge_pmd_unshare(struct mm_struct *mm, 307 struct vm_area_struct *vma, 308 unsigned long addr, pte_t *ptep) 309 { 310 return 0; 311 } 312 313 static inline void adjust_range_if_pmd_sharing_possible( 314 struct vm_area_struct *vma, 315 unsigned long *start, unsigned long *end) 316 { 317 } 318 319 static inline void hugetlb_zap_begin( 320 struct vm_area_struct *vma, 321 unsigned long *start, unsigned long *end) 322 { 323 } 324 325 static inline void hugetlb_zap_end( 326 struct vm_area_struct *vma, 327 struct zap_details *details) 328 { 329 } 330 331 static inline int copy_hugetlb_page_range(struct mm_struct *dst, 332 struct mm_struct *src, 333 struct vm_area_struct *dst_vma, 334 struct vm_area_struct *src_vma) 335 { 336 BUG(); 337 return 0; 338 } 339 340 static inline int move_hugetlb_page_tables(struct vm_area_struct *vma, 341 struct vm_area_struct *new_vma, 342 unsigned long old_addr, 343 unsigned long new_addr, 344 unsigned long len) 345 { 346 BUG(); 347 return 0; 348 } 349 350 static inline void hugetlb_report_meminfo(struct seq_file *m) 351 { 352 } 353 354 static inline int hugetlb_report_node_meminfo(char *buf, int len, int nid) 355 { 356 return 0; 357 } 358 359 static inline void hugetlb_show_meminfo_node(int nid) 360 { 361 } 362 363 static inline int prepare_hugepage_range(struct file *file, 364 unsigned long addr, unsigned long len) 365 { 366 return -EINVAL; 367 } 368 369 static inline void hugetlb_vma_lock_read(struct vm_area_struct *vma) 370 { 371 } 372 373 static inline void hugetlb_vma_unlock_read(struct vm_area_struct *vma) 374 { 375 } 376 377 static inline void hugetlb_vma_lock_write(struct vm_area_struct *vma) 378 { 379 } 380 381 static inline void hugetlb_vma_unlock_write(struct vm_area_struct *vma) 382 { 383 } 384 385 static inline int hugetlb_vma_trylock_write(struct vm_area_struct *vma) 386 { 387 return 1; 388 } 389 390 static inline void hugetlb_vma_assert_locked(struct vm_area_struct *vma) 391 { 392 } 393 394 static inline int is_hugepage_only_range(struct mm_struct *mm, 395 unsigned long addr, unsigned long len) 396 { 397 return 0; 398 } 399 400 static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb, 401 unsigned long addr, unsigned long end, 402 unsigned long floor, unsigned long ceiling) 403 { 404 BUG(); 405 } 406 407 #ifdef CONFIG_USERFAULTFD 408 static inline int hugetlb_mfill_atomic_pte(pte_t *dst_pte, 409 struct vm_area_struct *dst_vma, 410 unsigned long dst_addr, 411 unsigned long src_addr, 412 uffd_flags_t flags, 413 struct folio **foliop) 414 { 415 BUG(); 416 return 0; 417 } 418 #endif /* CONFIG_USERFAULTFD */ 419 420 static inline pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, 421 unsigned long sz) 422 { 423 return NULL; 424 } 425 426 static inline bool isolate_hugetlb(struct folio *folio, struct list_head *list) 427 { 428 return false; 429 } 430 431 static inline int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison) 432 { 433 return 0; 434 } 435 436 static inline int get_huge_page_for_hwpoison(unsigned long pfn, int flags, 437 bool *migratable_cleared) 438 { 439 return 0; 440 } 441 442 static inline void folio_putback_active_hugetlb(struct folio *folio) 443 { 444 } 445 446 static inline void move_hugetlb_state(struct folio *old_folio, 447 struct folio *new_folio, int reason) 448 { 449 } 450 451 static inline long hugetlb_change_protection( 452 struct vm_area_struct *vma, unsigned long address, 453 unsigned long end, pgprot_t newprot, 454 unsigned long cp_flags) 455 { 456 return 0; 457 } 458 459 static inline void __unmap_hugepage_range(struct mmu_gather *tlb, 460 struct vm_area_struct *vma, unsigned long start, 461 unsigned long end, struct page *ref_page, 462 zap_flags_t zap_flags) 463 { 464 BUG(); 465 } 466 467 static inline vm_fault_t hugetlb_fault(struct mm_struct *mm, 468 struct vm_area_struct *vma, unsigned long address, 469 unsigned int flags) 470 { 471 BUG(); 472 return 0; 473 } 474 475 static inline void hugetlb_unshare_all_pmds(struct vm_area_struct *vma) { } 476 477 #endif /* !CONFIG_HUGETLB_PAGE */ 478 479 #ifndef pgd_write 480 static inline int pgd_write(pgd_t pgd) 481 { 482 BUG(); 483 return 0; 484 } 485 #endif 486 487 #define HUGETLB_ANON_FILE "anon_hugepage" 488 489 enum { 490 /* 491 * The file will be used as an shm file so shmfs accounting rules 492 * apply 493 */ 494 HUGETLB_SHMFS_INODE = 1, 495 /* 496 * The file is being created on the internal vfs mount and shmfs 497 * accounting rules do not apply 498 */ 499 HUGETLB_ANONHUGE_INODE = 2, 500 }; 501 502 #ifdef CONFIG_HUGETLBFS 503 struct hugetlbfs_sb_info { 504 long max_inodes; /* inodes allowed */ 505 long free_inodes; /* inodes free */ 506 spinlock_t stat_lock; 507 struct hstate *hstate; 508 struct hugepage_subpool *spool; 509 kuid_t uid; 510 kgid_t gid; 511 umode_t mode; 512 }; 513 514 static inline struct hugetlbfs_sb_info *HUGETLBFS_SB(struct super_block *sb) 515 { 516 return sb->s_fs_info; 517 } 518 519 struct hugetlbfs_inode_info { 520 struct inode vfs_inode; 521 unsigned int seals; 522 }; 523 524 static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode) 525 { 526 return container_of(inode, struct hugetlbfs_inode_info, vfs_inode); 527 } 528 529 extern const struct vm_operations_struct hugetlb_vm_ops; 530 struct file *hugetlb_file_setup(const char *name, size_t size, vm_flags_t acct, 531 int creat_flags, int page_size_log); 532 533 static inline bool is_file_hugepages(const struct file *file) 534 { 535 return file->f_op->fop_flags & FOP_HUGE_PAGES; 536 } 537 538 static inline struct hstate *hstate_inode(struct inode *i) 539 { 540 return HUGETLBFS_SB(i->i_sb)->hstate; 541 } 542 #else /* !CONFIG_HUGETLBFS */ 543 544 #define is_file_hugepages(file) false 545 static inline struct file * 546 hugetlb_file_setup(const char *name, size_t size, vm_flags_t acctflag, 547 int creat_flags, int page_size_log) 548 { 549 return ERR_PTR(-ENOSYS); 550 } 551 552 static inline struct hstate *hstate_inode(struct inode *i) 553 { 554 return NULL; 555 } 556 #endif /* !CONFIG_HUGETLBFS */ 557 558 #ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA 559 unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, 560 unsigned long len, unsigned long pgoff, 561 unsigned long flags); 562 #endif /* HAVE_ARCH_HUGETLB_UNMAPPED_AREA */ 563 564 unsigned long 565 generic_hugetlb_get_unmapped_area(struct file *file, unsigned long addr, 566 unsigned long len, unsigned long pgoff, 567 unsigned long flags); 568 569 /* 570 * huegtlb page specific state flags. These flags are located in page.private 571 * of the hugetlb head page. Functions created via the below macros should be 572 * used to manipulate these flags. 573 * 574 * HPG_restore_reserve - Set when a hugetlb page consumes a reservation at 575 * allocation time. Cleared when page is fully instantiated. Free 576 * routine checks flag to restore a reservation on error paths. 577 * Synchronization: Examined or modified by code that knows it has 578 * the only reference to page. i.e. After allocation but before use 579 * or when the page is being freed. 580 * HPG_migratable - Set after a newly allocated page is added to the page 581 * cache and/or page tables. Indicates the page is a candidate for 582 * migration. 583 * Synchronization: Initially set after new page allocation with no 584 * locking. When examined and modified during migration processing 585 * (isolate, migrate, putback) the hugetlb_lock is held. 586 * HPG_temporary - Set on a page that is temporarily allocated from the buddy 587 * allocator. Typically used for migration target pages when no pages 588 * are available in the pool. The hugetlb free page path will 589 * immediately free pages with this flag set to the buddy allocator. 590 * Synchronization: Can be set after huge page allocation from buddy when 591 * code knows it has only reference. All other examinations and 592 * modifications require hugetlb_lock. 593 * HPG_freed - Set when page is on the free lists. 594 * Synchronization: hugetlb_lock held for examination and modification. 595 * HPG_vmemmap_optimized - Set when the vmemmap pages of the page are freed. 596 * HPG_raw_hwp_unreliable - Set when the hugetlb page has a hwpoison sub-page 597 * that is not tracked by raw_hwp_page list. 598 */ 599 enum hugetlb_page_flags { 600 HPG_restore_reserve = 0, 601 HPG_migratable, 602 HPG_temporary, 603 HPG_freed, 604 HPG_vmemmap_optimized, 605 HPG_raw_hwp_unreliable, 606 __NR_HPAGEFLAGS, 607 }; 608 609 /* 610 * Macros to create test, set and clear function definitions for 611 * hugetlb specific page flags. 612 */ 613 #ifdef CONFIG_HUGETLB_PAGE 614 #define TESTHPAGEFLAG(uname, flname) \ 615 static __always_inline \ 616 bool folio_test_hugetlb_##flname(struct folio *folio) \ 617 { void *private = &folio->private; \ 618 return test_bit(HPG_##flname, private); \ 619 } \ 620 static inline int HPage##uname(struct page *page) \ 621 { return test_bit(HPG_##flname, &(page->private)); } 622 623 #define SETHPAGEFLAG(uname, flname) \ 624 static __always_inline \ 625 void folio_set_hugetlb_##flname(struct folio *folio) \ 626 { void *private = &folio->private; \ 627 set_bit(HPG_##flname, private); \ 628 } \ 629 static inline void SetHPage##uname(struct page *page) \ 630 { set_bit(HPG_##flname, &(page->private)); } 631 632 #define CLEARHPAGEFLAG(uname, flname) \ 633 static __always_inline \ 634 void folio_clear_hugetlb_##flname(struct folio *folio) \ 635 { void *private = &folio->private; \ 636 clear_bit(HPG_##flname, private); \ 637 } \ 638 static inline void ClearHPage##uname(struct page *page) \ 639 { clear_bit(HPG_##flname, &(page->private)); } 640 #else 641 #define TESTHPAGEFLAG(uname, flname) \ 642 static inline bool \ 643 folio_test_hugetlb_##flname(struct folio *folio) \ 644 { return 0; } \ 645 static inline int HPage##uname(struct page *page) \ 646 { return 0; } 647 648 #define SETHPAGEFLAG(uname, flname) \ 649 static inline void \ 650 folio_set_hugetlb_##flname(struct folio *folio) \ 651 { } \ 652 static inline void SetHPage##uname(struct page *page) \ 653 { } 654 655 #define CLEARHPAGEFLAG(uname, flname) \ 656 static inline void \ 657 folio_clear_hugetlb_##flname(struct folio *folio) \ 658 { } \ 659 static inline void ClearHPage##uname(struct page *page) \ 660 { } 661 #endif 662 663 #define HPAGEFLAG(uname, flname) \ 664 TESTHPAGEFLAG(uname, flname) \ 665 SETHPAGEFLAG(uname, flname) \ 666 CLEARHPAGEFLAG(uname, flname) \ 667 668 /* 669 * Create functions associated with hugetlb page flags 670 */ 671 HPAGEFLAG(RestoreReserve, restore_reserve) 672 HPAGEFLAG(Migratable, migratable) 673 HPAGEFLAG(Temporary, temporary) 674 HPAGEFLAG(Freed, freed) 675 HPAGEFLAG(VmemmapOptimized, vmemmap_optimized) 676 HPAGEFLAG(RawHwpUnreliable, raw_hwp_unreliable) 677 678 #ifdef CONFIG_HUGETLB_PAGE 679 680 #define HSTATE_NAME_LEN 32 681 /* Defines one hugetlb page size */ 682 struct hstate { 683 struct mutex resize_lock; 684 int next_nid_to_alloc; 685 int next_nid_to_free; 686 unsigned int order; 687 unsigned int demote_order; 688 unsigned long mask; 689 unsigned long max_huge_pages; 690 unsigned long nr_huge_pages; 691 unsigned long free_huge_pages; 692 unsigned long resv_huge_pages; 693 unsigned long surplus_huge_pages; 694 unsigned long nr_overcommit_huge_pages; 695 struct list_head hugepage_activelist; 696 struct list_head hugepage_freelists[MAX_NUMNODES]; 697 unsigned int max_huge_pages_node[MAX_NUMNODES]; 698 unsigned int nr_huge_pages_node[MAX_NUMNODES]; 699 unsigned int free_huge_pages_node[MAX_NUMNODES]; 700 unsigned int surplus_huge_pages_node[MAX_NUMNODES]; 701 #ifdef CONFIG_CGROUP_HUGETLB 702 /* cgroup control files */ 703 struct cftype cgroup_files_dfl[8]; 704 struct cftype cgroup_files_legacy[10]; 705 #endif 706 char name[HSTATE_NAME_LEN]; 707 }; 708 709 struct huge_bootmem_page { 710 struct list_head list; 711 struct hstate *hstate; 712 }; 713 714 int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list); 715 struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, 716 unsigned long addr, int avoid_reserve); 717 struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, 718 nodemask_t *nmask, gfp_t gfp_mask, 719 bool allow_alloc_fallback); 720 int hugetlb_add_to_page_cache(struct folio *folio, struct address_space *mapping, 721 pgoff_t idx); 722 void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma, 723 unsigned long address, struct folio *folio); 724 725 /* arch callback */ 726 int __init __alloc_bootmem_huge_page(struct hstate *h, int nid); 727 int __init alloc_bootmem_huge_page(struct hstate *h, int nid); 728 bool __init hugetlb_node_alloc_supported(void); 729 730 void __init hugetlb_add_hstate(unsigned order); 731 bool __init arch_hugetlb_valid_size(unsigned long size); 732 struct hstate *size_to_hstate(unsigned long size); 733 734 #ifndef HUGE_MAX_HSTATE 735 #define HUGE_MAX_HSTATE 1 736 #endif 737 738 extern struct hstate hstates[HUGE_MAX_HSTATE]; 739 extern unsigned int default_hstate_idx; 740 741 #define default_hstate (hstates[default_hstate_idx]) 742 743 static inline struct hugepage_subpool *hugetlb_folio_subpool(struct folio *folio) 744 { 745 return folio->_hugetlb_subpool; 746 } 747 748 static inline void hugetlb_set_folio_subpool(struct folio *folio, 749 struct hugepage_subpool *subpool) 750 { 751 folio->_hugetlb_subpool = subpool; 752 } 753 754 static inline struct hstate *hstate_file(struct file *f) 755 { 756 return hstate_inode(file_inode(f)); 757 } 758 759 static inline struct hstate *hstate_sizelog(int page_size_log) 760 { 761 if (!page_size_log) 762 return &default_hstate; 763 764 if (page_size_log < BITS_PER_LONG) 765 return size_to_hstate(1UL << page_size_log); 766 767 return NULL; 768 } 769 770 static inline struct hstate *hstate_vma(struct vm_area_struct *vma) 771 { 772 return hstate_file(vma->vm_file); 773 } 774 775 static inline unsigned long huge_page_size(const struct hstate *h) 776 { 777 return (unsigned long)PAGE_SIZE << h->order; 778 } 779 780 extern unsigned long vma_kernel_pagesize(struct vm_area_struct *vma); 781 782 extern unsigned long vma_mmu_pagesize(struct vm_area_struct *vma); 783 784 static inline unsigned long huge_page_mask(struct hstate *h) 785 { 786 return h->mask; 787 } 788 789 static inline unsigned int huge_page_order(struct hstate *h) 790 { 791 return h->order; 792 } 793 794 static inline unsigned huge_page_shift(struct hstate *h) 795 { 796 return h->order + PAGE_SHIFT; 797 } 798 799 static inline bool hstate_is_gigantic(struct hstate *h) 800 { 801 return huge_page_order(h) > MAX_PAGE_ORDER; 802 } 803 804 static inline unsigned int pages_per_huge_page(const struct hstate *h) 805 { 806 return 1 << h->order; 807 } 808 809 static inline unsigned int blocks_per_huge_page(struct hstate *h) 810 { 811 return huge_page_size(h) / 512; 812 } 813 814 static inline struct folio *filemap_lock_hugetlb_folio(struct hstate *h, 815 struct address_space *mapping, pgoff_t idx) 816 { 817 return filemap_lock_folio(mapping, idx << huge_page_order(h)); 818 } 819 820 #include <asm/hugetlb.h> 821 822 #ifndef is_hugepage_only_range 823 static inline int is_hugepage_only_range(struct mm_struct *mm, 824 unsigned long addr, unsigned long len) 825 { 826 return 0; 827 } 828 #define is_hugepage_only_range is_hugepage_only_range 829 #endif 830 831 #ifndef arch_clear_hugetlb_flags 832 static inline void arch_clear_hugetlb_flags(struct folio *folio) { } 833 #define arch_clear_hugetlb_flags arch_clear_hugetlb_flags 834 #endif 835 836 #ifndef arch_make_huge_pte 837 static inline pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, 838 vm_flags_t flags) 839 { 840 return pte_mkhuge(entry); 841 } 842 #endif 843 844 static inline struct hstate *folio_hstate(struct folio *folio) 845 { 846 VM_BUG_ON_FOLIO(!folio_test_hugetlb(folio), folio); 847 return size_to_hstate(folio_size(folio)); 848 } 849 850 static inline unsigned hstate_index_to_shift(unsigned index) 851 { 852 return hstates[index].order + PAGE_SHIFT; 853 } 854 855 static inline int hstate_index(struct hstate *h) 856 { 857 return h - hstates; 858 } 859 860 int dissolve_free_hugetlb_folio(struct folio *folio); 861 int dissolve_free_hugetlb_folios(unsigned long start_pfn, 862 unsigned long end_pfn); 863 864 #ifdef CONFIG_MEMORY_FAILURE 865 extern void folio_clear_hugetlb_hwpoison(struct folio *folio); 866 #else 867 static inline void folio_clear_hugetlb_hwpoison(struct folio *folio) 868 { 869 } 870 #endif 871 872 #ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION 873 #ifndef arch_hugetlb_migration_supported 874 static inline bool arch_hugetlb_migration_supported(struct hstate *h) 875 { 876 if ((huge_page_shift(h) == PMD_SHIFT) || 877 (huge_page_shift(h) == PUD_SHIFT) || 878 (huge_page_shift(h) == PGDIR_SHIFT)) 879 return true; 880 else 881 return false; 882 } 883 #endif 884 #else 885 static inline bool arch_hugetlb_migration_supported(struct hstate *h) 886 { 887 return false; 888 } 889 #endif 890 891 static inline bool hugepage_migration_supported(struct hstate *h) 892 { 893 return arch_hugetlb_migration_supported(h); 894 } 895 896 /* 897 * Movability check is different as compared to migration check. 898 * It determines whether or not a huge page should be placed on 899 * movable zone or not. Movability of any huge page should be 900 * required only if huge page size is supported for migration. 901 * There won't be any reason for the huge page to be movable if 902 * it is not migratable to start with. Also the size of the huge 903 * page should be large enough to be placed under a movable zone 904 * and still feasible enough to be migratable. Just the presence 905 * in movable zone does not make the migration feasible. 906 * 907 * So even though large huge page sizes like the gigantic ones 908 * are migratable they should not be movable because its not 909 * feasible to migrate them from movable zone. 910 */ 911 static inline bool hugepage_movable_supported(struct hstate *h) 912 { 913 if (!hugepage_migration_supported(h)) 914 return false; 915 916 if (hstate_is_gigantic(h)) 917 return false; 918 return true; 919 } 920 921 /* Movability of hugepages depends on migration support. */ 922 static inline gfp_t htlb_alloc_mask(struct hstate *h) 923 { 924 if (hugepage_movable_supported(h)) 925 return GFP_HIGHUSER_MOVABLE; 926 else 927 return GFP_HIGHUSER; 928 } 929 930 static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask) 931 { 932 gfp_t modified_mask = htlb_alloc_mask(h); 933 934 /* Some callers might want to enforce node */ 935 modified_mask |= (gfp_mask & __GFP_THISNODE); 936 937 modified_mask |= (gfp_mask & __GFP_NOWARN); 938 939 return modified_mask; 940 } 941 942 static inline bool htlb_allow_alloc_fallback(int reason) 943 { 944 bool allowed_fallback = false; 945 946 /* 947 * Note: the memory offline, memory failure and migration syscalls will 948 * be allowed to fallback to other nodes due to lack of a better chioce, 949 * that might break the per-node hugetlb pool. While other cases will 950 * set the __GFP_THISNODE to avoid breaking the per-node hugetlb pool. 951 */ 952 switch (reason) { 953 case MR_MEMORY_HOTPLUG: 954 case MR_MEMORY_FAILURE: 955 case MR_SYSCALL: 956 case MR_MEMPOLICY_MBIND: 957 allowed_fallback = true; 958 break; 959 default: 960 break; 961 } 962 963 return allowed_fallback; 964 } 965 966 static inline spinlock_t *huge_pte_lockptr(struct hstate *h, 967 struct mm_struct *mm, pte_t *pte) 968 { 969 if (huge_page_size(h) == PMD_SIZE) 970 return pmd_lockptr(mm, (pmd_t *) pte); 971 VM_BUG_ON(huge_page_size(h) == PAGE_SIZE); 972 return &mm->page_table_lock; 973 } 974 975 #ifndef hugepages_supported 976 /* 977 * Some platform decide whether they support huge pages at boot 978 * time. Some of them, such as powerpc, set HPAGE_SHIFT to 0 979 * when there is no such support 980 */ 981 #define hugepages_supported() (HPAGE_SHIFT != 0) 982 #endif 983 984 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm); 985 986 static inline void hugetlb_count_init(struct mm_struct *mm) 987 { 988 atomic_long_set(&mm->hugetlb_usage, 0); 989 } 990 991 static inline void hugetlb_count_add(long l, struct mm_struct *mm) 992 { 993 atomic_long_add(l, &mm->hugetlb_usage); 994 } 995 996 static inline void hugetlb_count_sub(long l, struct mm_struct *mm) 997 { 998 atomic_long_sub(l, &mm->hugetlb_usage); 999 } 1000 1001 #ifndef huge_ptep_modify_prot_start 1002 #define huge_ptep_modify_prot_start huge_ptep_modify_prot_start 1003 static inline pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma, 1004 unsigned long addr, pte_t *ptep) 1005 { 1006 return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); 1007 } 1008 #endif 1009 1010 #ifndef huge_ptep_modify_prot_commit 1011 #define huge_ptep_modify_prot_commit huge_ptep_modify_prot_commit 1012 static inline void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, 1013 unsigned long addr, pte_t *ptep, 1014 pte_t old_pte, pte_t pte) 1015 { 1016 unsigned long psize = huge_page_size(hstate_vma(vma)); 1017 1018 set_huge_pte_at(vma->vm_mm, addr, ptep, pte, psize); 1019 } 1020 #endif 1021 1022 #ifdef CONFIG_NUMA 1023 void hugetlb_register_node(struct node *node); 1024 void hugetlb_unregister_node(struct node *node); 1025 #endif 1026 1027 /* 1028 * Check if a given raw @page in a hugepage is HWPOISON. 1029 */ 1030 bool is_raw_hwpoison_page_in_hugepage(struct page *page); 1031 1032 #else /* CONFIG_HUGETLB_PAGE */ 1033 struct hstate {}; 1034 1035 static inline struct hugepage_subpool *hugetlb_folio_subpool(struct folio *folio) 1036 { 1037 return NULL; 1038 } 1039 1040 static inline struct folio *filemap_lock_hugetlb_folio(struct hstate *h, 1041 struct address_space *mapping, pgoff_t idx) 1042 { 1043 return NULL; 1044 } 1045 1046 static inline int isolate_or_dissolve_huge_page(struct page *page, 1047 struct list_head *list) 1048 { 1049 return -ENOMEM; 1050 } 1051 1052 static inline struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, 1053 unsigned long addr, 1054 int avoid_reserve) 1055 { 1056 return NULL; 1057 } 1058 1059 static inline struct folio * 1060 alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, 1061 nodemask_t *nmask, gfp_t gfp_mask, 1062 bool allow_alloc_fallback) 1063 { 1064 return NULL; 1065 } 1066 1067 static inline int __alloc_bootmem_huge_page(struct hstate *h) 1068 { 1069 return 0; 1070 } 1071 1072 static inline struct hstate *hstate_file(struct file *f) 1073 { 1074 return NULL; 1075 } 1076 1077 static inline struct hstate *hstate_sizelog(int page_size_log) 1078 { 1079 return NULL; 1080 } 1081 1082 static inline struct hstate *hstate_vma(struct vm_area_struct *vma) 1083 { 1084 return NULL; 1085 } 1086 1087 static inline struct hstate *folio_hstate(struct folio *folio) 1088 { 1089 return NULL; 1090 } 1091 1092 static inline struct hstate *size_to_hstate(unsigned long size) 1093 { 1094 return NULL; 1095 } 1096 1097 static inline unsigned long huge_page_size(struct hstate *h) 1098 { 1099 return PAGE_SIZE; 1100 } 1101 1102 static inline unsigned long huge_page_mask(struct hstate *h) 1103 { 1104 return PAGE_MASK; 1105 } 1106 1107 static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) 1108 { 1109 return PAGE_SIZE; 1110 } 1111 1112 static inline unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) 1113 { 1114 return PAGE_SIZE; 1115 } 1116 1117 static inline unsigned int huge_page_order(struct hstate *h) 1118 { 1119 return 0; 1120 } 1121 1122 static inline unsigned int huge_page_shift(struct hstate *h) 1123 { 1124 return PAGE_SHIFT; 1125 } 1126 1127 static inline bool hstate_is_gigantic(struct hstate *h) 1128 { 1129 return false; 1130 } 1131 1132 static inline unsigned int pages_per_huge_page(struct hstate *h) 1133 { 1134 return 1; 1135 } 1136 1137 static inline unsigned hstate_index_to_shift(unsigned index) 1138 { 1139 return 0; 1140 } 1141 1142 static inline int hstate_index(struct hstate *h) 1143 { 1144 return 0; 1145 } 1146 1147 static inline int dissolve_free_hugetlb_folio(struct folio *folio) 1148 { 1149 return 0; 1150 } 1151 1152 static inline int dissolve_free_hugetlb_folios(unsigned long start_pfn, 1153 unsigned long end_pfn) 1154 { 1155 return 0; 1156 } 1157 1158 static inline bool hugepage_migration_supported(struct hstate *h) 1159 { 1160 return false; 1161 } 1162 1163 static inline bool hugepage_movable_supported(struct hstate *h) 1164 { 1165 return false; 1166 } 1167 1168 static inline gfp_t htlb_alloc_mask(struct hstate *h) 1169 { 1170 return 0; 1171 } 1172 1173 static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask) 1174 { 1175 return 0; 1176 } 1177 1178 static inline bool htlb_allow_alloc_fallback(int reason) 1179 { 1180 return false; 1181 } 1182 1183 static inline spinlock_t *huge_pte_lockptr(struct hstate *h, 1184 struct mm_struct *mm, pte_t *pte) 1185 { 1186 return &mm->page_table_lock; 1187 } 1188 1189 static inline void hugetlb_count_init(struct mm_struct *mm) 1190 { 1191 } 1192 1193 static inline void hugetlb_report_usage(struct seq_file *f, struct mm_struct *m) 1194 { 1195 } 1196 1197 static inline void hugetlb_count_sub(long l, struct mm_struct *mm) 1198 { 1199 } 1200 1201 static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, 1202 unsigned long addr, pte_t *ptep) 1203 { 1204 #ifdef CONFIG_MMU 1205 return ptep_get(ptep); 1206 #else 1207 return *ptep; 1208 #endif 1209 } 1210 1211 static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, 1212 pte_t *ptep, pte_t pte, unsigned long sz) 1213 { 1214 } 1215 1216 static inline void hugetlb_register_node(struct node *node) 1217 { 1218 } 1219 1220 static inline void hugetlb_unregister_node(struct node *node) 1221 { 1222 } 1223 1224 static inline bool hugetlbfs_pagecache_present( 1225 struct hstate *h, struct vm_area_struct *vma, unsigned long address) 1226 { 1227 return false; 1228 } 1229 #endif /* CONFIG_HUGETLB_PAGE */ 1230 1231 static inline spinlock_t *huge_pte_lock(struct hstate *h, 1232 struct mm_struct *mm, pte_t *pte) 1233 { 1234 spinlock_t *ptl; 1235 1236 ptl = huge_pte_lockptr(h, mm, pte); 1237 spin_lock(ptl); 1238 return ptl; 1239 } 1240 1241 #if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_CMA) 1242 extern void __init hugetlb_cma_reserve(int order); 1243 #else 1244 static inline __init void hugetlb_cma_reserve(int order) 1245 { 1246 } 1247 #endif 1248 1249 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE 1250 static inline bool hugetlb_pmd_shared(pte_t *pte) 1251 { 1252 return page_count(virt_to_page(pte)) > 1; 1253 } 1254 #else 1255 static inline bool hugetlb_pmd_shared(pte_t *pte) 1256 { 1257 return false; 1258 } 1259 #endif 1260 1261 bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr); 1262 1263 #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE 1264 /* 1265 * ARCHes with special requirements for evicting HUGETLB backing TLB entries can 1266 * implement this. 1267 */ 1268 #define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) 1269 #endif 1270 1271 static inline bool __vma_shareable_lock(struct vm_area_struct *vma) 1272 { 1273 return (vma->vm_flags & VM_MAYSHARE) && vma->vm_private_data; 1274 } 1275 1276 bool __vma_private_lock(struct vm_area_struct *vma); 1277 1278 /* 1279 * Safe version of huge_pte_offset() to check the locks. See comments 1280 * above huge_pte_offset(). 1281 */ 1282 static inline pte_t * 1283 hugetlb_walk(struct vm_area_struct *vma, unsigned long addr, unsigned long sz) 1284 { 1285 #if defined(CONFIG_HUGETLB_PAGE) && \ 1286 defined(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) && defined(CONFIG_LOCKDEP) 1287 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; 1288 1289 /* 1290 * If pmd sharing possible, locking needed to safely walk the 1291 * hugetlb pgtables. More information can be found at the comment 1292 * above huge_pte_offset() in the same file. 1293 * 1294 * NOTE: lockdep_is_held() is only defined with CONFIG_LOCKDEP. 1295 */ 1296 if (__vma_shareable_lock(vma)) 1297 WARN_ON_ONCE(!lockdep_is_held(&vma_lock->rw_sema) && 1298 !lockdep_is_held( 1299 &vma->vm_file->f_mapping->i_mmap_rwsem)); 1300 #endif 1301 return huge_pte_offset(vma->vm_mm, addr, sz); 1302 } 1303 1304 #endif /* _LINUX_HUGETLB_H */ 1305