1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_HUGETLB_H
3 #define _LINUX_HUGETLB_H
4
5 #include <linux/mm.h>
6 #include <linux/mm_types.h>
7 #include <linux/mmdebug.h>
8 #include <linux/fs.h>
9 #include <linux/hugetlb_inline.h>
10 #include <linux/cgroup.h>
11 #include <linux/page_ref.h>
12 #include <linux/list.h>
13 #include <linux/kref.h>
14 #include <linux/pgtable.h>
15 #include <linux/gfp.h>
16 #include <linux/userfaultfd_k.h>
17
18 struct ctl_table;
19 struct user_struct;
20 struct mmu_gather;
21 struct node;
22
23 void free_huge_folio(struct folio *folio);
24
25 #ifdef CONFIG_HUGETLB_PAGE
26
27 #include <linux/pagemap.h>
28 #include <linux/shm.h>
29 #include <asm/tlbflush.h>
30
31 /*
32 * For HugeTLB page, there are more metadata to save in the struct page. But
33 * the head struct page cannot meet our needs, so we have to abuse other tail
34 * struct page to store the metadata.
35 */
36 #define __NR_USED_SUBPAGE 3
37
38 struct hugepage_subpool {
39 spinlock_t lock;
40 long count;
41 long max_hpages; /* Maximum huge pages or -1 if no maximum. */
42 long used_hpages; /* Used count against maximum, includes */
43 /* both allocated and reserved pages. */
44 struct hstate *hstate;
45 long min_hpages; /* Minimum huge pages or -1 if no minimum. */
46 long rsv_hpages; /* Pages reserved against global pool to */
47 /* satisfy minimum size. */
48 };
49
50 struct resv_map {
51 struct kref refs;
52 spinlock_t lock;
53 struct list_head regions;
54 long adds_in_progress;
55 struct list_head region_cache;
56 long region_cache_count;
57 struct rw_semaphore rw_sema;
58 #ifdef CONFIG_CGROUP_HUGETLB
59 /*
60 * On private mappings, the counter to uncharge reservations is stored
61 * here. If these fields are 0, then either the mapping is shared, or
62 * cgroup accounting is disabled for this resv_map.
63 */
64 struct page_counter *reservation_counter;
65 unsigned long pages_per_hpage;
66 struct cgroup_subsys_state *css;
67 #endif
68 };
69
70 /*
71 * Region tracking -- allows tracking of reservations and instantiated pages
72 * across the pages in a mapping.
73 *
74 * The region data structures are embedded into a resv_map and protected
75 * by a resv_map's lock. The set of regions within the resv_map represent
76 * reservations for huge pages, or huge pages that have already been
77 * instantiated within the map. The from and to elements are huge page
78 * indices into the associated mapping. from indicates the starting index
79 * of the region. to represents the first index past the end of the region.
80 *
81 * For example, a file region structure with from == 0 and to == 4 represents
82 * four huge pages in a mapping. It is important to note that the to element
83 * represents the first element past the end of the region. This is used in
84 * arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
85 *
86 * Interval notation of the form [from, to) will be used to indicate that
87 * the endpoint from is inclusive and to is exclusive.
88 */
89 struct file_region {
90 struct list_head link;
91 long from;
92 long to;
93 #ifdef CONFIG_CGROUP_HUGETLB
94 /*
95 * On shared mappings, each reserved region appears as a struct
96 * file_region in resv_map. These fields hold the info needed to
97 * uncharge each reservation.
98 */
99 struct page_counter *reservation_counter;
100 struct cgroup_subsys_state *css;
101 #endif
102 };
103
104 struct hugetlb_vma_lock {
105 struct kref refs;
106 struct rw_semaphore rw_sema;
107 struct vm_area_struct *vma;
108 };
109
110 extern struct resv_map *resv_map_alloc(void);
111 void resv_map_release(struct kref *ref);
112
113 extern spinlock_t hugetlb_lock;
114 extern int hugetlb_max_hstate __read_mostly;
115 #define for_each_hstate(h) \
116 for ((h) = hstates; (h) < &hstates[hugetlb_max_hstate]; (h)++)
117
118 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
119 long min_hpages);
120 void hugepage_put_subpool(struct hugepage_subpool *spool);
121
122 void hugetlb_dup_vma_private(struct vm_area_struct *vma);
123 void clear_vma_resv_huge_pages(struct vm_area_struct *vma);
124 int move_hugetlb_page_tables(struct vm_area_struct *vma,
125 struct vm_area_struct *new_vma,
126 unsigned long old_addr, unsigned long new_addr,
127 unsigned long len);
128 int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *,
129 struct vm_area_struct *, struct vm_area_struct *);
130 void unmap_hugepage_range(struct vm_area_struct *,
131 unsigned long, unsigned long, struct page *,
132 zap_flags_t);
133 void __unmap_hugepage_range(struct mmu_gather *tlb,
134 struct vm_area_struct *vma,
135 unsigned long start, unsigned long end,
136 struct page *ref_page, zap_flags_t zap_flags);
137 void hugetlb_report_meminfo(struct seq_file *);
138 int hugetlb_report_node_meminfo(char *buf, int len, int nid);
139 void hugetlb_show_meminfo_node(int nid);
140 unsigned long hugetlb_total_pages(void);
141 vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
142 unsigned long address, unsigned int flags);
143 #ifdef CONFIG_USERFAULTFD
144 int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
145 struct vm_area_struct *dst_vma,
146 unsigned long dst_addr,
147 unsigned long src_addr,
148 uffd_flags_t flags,
149 struct folio **foliop);
150 #endif /* CONFIG_USERFAULTFD */
151 bool hugetlb_reserve_pages(struct inode *inode, long from, long to,
152 struct vm_area_struct *vma,
153 vm_flags_t vm_flags);
154 long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
155 long freed);
156 bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list);
157 int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison);
158 int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
159 bool *migratable_cleared);
160 void folio_putback_hugetlb(struct folio *folio);
161 void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason);
162 void hugetlb_fix_reserve_counts(struct inode *inode);
163 extern struct mutex *hugetlb_fault_mutex_table;
164 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx);
165
166 pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
167 unsigned long addr, pud_t *pud);
168 bool hugetlbfs_pagecache_present(struct hstate *h,
169 struct vm_area_struct *vma,
170 unsigned long address);
171
172 struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio);
173
174 extern int sysctl_hugetlb_shm_group;
175 extern struct list_head huge_boot_pages[MAX_NUMNODES];
176
177 /* arch callbacks */
178
179 #ifndef CONFIG_HIGHPTE
180 /*
181 * pte_offset_huge() and pte_alloc_huge() are helpers for those architectures
182 * which may go down to the lowest PTE level in their huge_pte_offset() and
183 * huge_pte_alloc(): to avoid reliance on pte_offset_map() without pte_unmap().
184 */
pte_offset_huge(pmd_t * pmd,unsigned long address)185 static inline pte_t *pte_offset_huge(pmd_t *pmd, unsigned long address)
186 {
187 return pte_offset_kernel(pmd, address);
188 }
pte_alloc_huge(struct mm_struct * mm,pmd_t * pmd,unsigned long address)189 static inline pte_t *pte_alloc_huge(struct mm_struct *mm, pmd_t *pmd,
190 unsigned long address)
191 {
192 return pte_alloc(mm, pmd) ? NULL : pte_offset_huge(pmd, address);
193 }
194 #endif
195
196 pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
197 unsigned long addr, unsigned long sz);
198 /*
199 * huge_pte_offset(): Walk the hugetlb pgtable until the last level PTE.
200 * Returns the pte_t* if found, or NULL if the address is not mapped.
201 *
202 * IMPORTANT: we should normally not directly call this function, instead
203 * this is only a common interface to implement arch-specific
204 * walker. Please use hugetlb_walk() instead, because that will attempt to
205 * verify the locking for you.
206 *
207 * Since this function will walk all the pgtable pages (including not only
208 * high-level pgtable page, but also PUD entry that can be unshared
209 * concurrently for VM_SHARED), the caller of this function should be
210 * responsible of its thread safety. One can follow this rule:
211 *
212 * (1) For private mappings: pmd unsharing is not possible, so holding the
213 * mmap_lock for either read or write is sufficient. Most callers
214 * already hold the mmap_lock, so normally, no special action is
215 * required.
216 *
217 * (2) For shared mappings: pmd unsharing is possible (so the PUD-ranged
218 * pgtable page can go away from under us! It can be done by a pmd
219 * unshare with a follow up munmap() on the other process), then we
220 * need either:
221 *
222 * (2.1) hugetlb vma lock read or write held, to make sure pmd unshare
223 * won't happen upon the range (it also makes sure the pte_t we
224 * read is the right and stable one), or,
225 *
226 * (2.2) hugetlb mapping i_mmap_rwsem lock held read or write, to make
227 * sure even if unshare happened the racy unmap() will wait until
228 * i_mmap_rwsem is released.
229 *
230 * Option (2.1) is the safest, which guarantees pte stability from pmd
231 * sharing pov, until the vma lock released. Option (2.2) doesn't protect
232 * a concurrent pmd unshare, but it makes sure the pgtable page is safe to
233 * access.
234 */
235 pte_t *huge_pte_offset(struct mm_struct *mm,
236 unsigned long addr, unsigned long sz);
237 unsigned long hugetlb_mask_last_page(struct hstate *h);
238 int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
239 unsigned long addr, pte_t *ptep);
240 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
241 unsigned long *start, unsigned long *end);
242
243 extern void __hugetlb_zap_begin(struct vm_area_struct *vma,
244 unsigned long *begin, unsigned long *end);
245 extern void __hugetlb_zap_end(struct vm_area_struct *vma,
246 struct zap_details *details);
247
hugetlb_zap_begin(struct vm_area_struct * vma,unsigned long * start,unsigned long * end)248 static inline void hugetlb_zap_begin(struct vm_area_struct *vma,
249 unsigned long *start, unsigned long *end)
250 {
251 if (is_vm_hugetlb_page(vma))
252 __hugetlb_zap_begin(vma, start, end);
253 }
254
hugetlb_zap_end(struct vm_area_struct * vma,struct zap_details * details)255 static inline void hugetlb_zap_end(struct vm_area_struct *vma,
256 struct zap_details *details)
257 {
258 if (is_vm_hugetlb_page(vma))
259 __hugetlb_zap_end(vma, details);
260 }
261
262 void hugetlb_vma_lock_read(struct vm_area_struct *vma);
263 void hugetlb_vma_unlock_read(struct vm_area_struct *vma);
264 void hugetlb_vma_lock_write(struct vm_area_struct *vma);
265 void hugetlb_vma_unlock_write(struct vm_area_struct *vma);
266 int hugetlb_vma_trylock_write(struct vm_area_struct *vma);
267 void hugetlb_vma_assert_locked(struct vm_area_struct *vma);
268 void hugetlb_vma_lock_release(struct kref *kref);
269 long hugetlb_change_protection(struct vm_area_struct *vma,
270 unsigned long address, unsigned long end, pgprot_t newprot,
271 unsigned long cp_flags);
272 bool is_hugetlb_entry_migration(pte_t pte);
273 bool is_hugetlb_entry_hwpoisoned(pte_t pte);
274 void hugetlb_unshare_all_pmds(struct vm_area_struct *vma);
275
276 #else /* !CONFIG_HUGETLB_PAGE */
277
hugetlb_dup_vma_private(struct vm_area_struct * vma)278 static inline void hugetlb_dup_vma_private(struct vm_area_struct *vma)
279 {
280 }
281
clear_vma_resv_huge_pages(struct vm_area_struct * vma)282 static inline void clear_vma_resv_huge_pages(struct vm_area_struct *vma)
283 {
284 }
285
hugetlb_total_pages(void)286 static inline unsigned long hugetlb_total_pages(void)
287 {
288 return 0;
289 }
290
hugetlb_folio_mapping_lock_write(struct folio * folio)291 static inline struct address_space *hugetlb_folio_mapping_lock_write(
292 struct folio *folio)
293 {
294 return NULL;
295 }
296
huge_pmd_unshare(struct mm_struct * mm,struct vm_area_struct * vma,unsigned long addr,pte_t * ptep)297 static inline int huge_pmd_unshare(struct mm_struct *mm,
298 struct vm_area_struct *vma,
299 unsigned long addr, pte_t *ptep)
300 {
301 return 0;
302 }
303
adjust_range_if_pmd_sharing_possible(struct vm_area_struct * vma,unsigned long * start,unsigned long * end)304 static inline void adjust_range_if_pmd_sharing_possible(
305 struct vm_area_struct *vma,
306 unsigned long *start, unsigned long *end)
307 {
308 }
309
hugetlb_zap_begin(struct vm_area_struct * vma,unsigned long * start,unsigned long * end)310 static inline void hugetlb_zap_begin(
311 struct vm_area_struct *vma,
312 unsigned long *start, unsigned long *end)
313 {
314 }
315
hugetlb_zap_end(struct vm_area_struct * vma,struct zap_details * details)316 static inline void hugetlb_zap_end(
317 struct vm_area_struct *vma,
318 struct zap_details *details)
319 {
320 }
321
copy_hugetlb_page_range(struct mm_struct * dst,struct mm_struct * src,struct vm_area_struct * dst_vma,struct vm_area_struct * src_vma)322 static inline int copy_hugetlb_page_range(struct mm_struct *dst,
323 struct mm_struct *src,
324 struct vm_area_struct *dst_vma,
325 struct vm_area_struct *src_vma)
326 {
327 BUG();
328 return 0;
329 }
330
move_hugetlb_page_tables(struct vm_area_struct * vma,struct vm_area_struct * new_vma,unsigned long old_addr,unsigned long new_addr,unsigned long len)331 static inline int move_hugetlb_page_tables(struct vm_area_struct *vma,
332 struct vm_area_struct *new_vma,
333 unsigned long old_addr,
334 unsigned long new_addr,
335 unsigned long len)
336 {
337 BUG();
338 return 0;
339 }
340
hugetlb_report_meminfo(struct seq_file * m)341 static inline void hugetlb_report_meminfo(struct seq_file *m)
342 {
343 }
344
hugetlb_report_node_meminfo(char * buf,int len,int nid)345 static inline int hugetlb_report_node_meminfo(char *buf, int len, int nid)
346 {
347 return 0;
348 }
349
hugetlb_show_meminfo_node(int nid)350 static inline void hugetlb_show_meminfo_node(int nid)
351 {
352 }
353
prepare_hugepage_range(struct file * file,unsigned long addr,unsigned long len)354 static inline int prepare_hugepage_range(struct file *file,
355 unsigned long addr, unsigned long len)
356 {
357 return -EINVAL;
358 }
359
hugetlb_vma_lock_read(struct vm_area_struct * vma)360 static inline void hugetlb_vma_lock_read(struct vm_area_struct *vma)
361 {
362 }
363
hugetlb_vma_unlock_read(struct vm_area_struct * vma)364 static inline void hugetlb_vma_unlock_read(struct vm_area_struct *vma)
365 {
366 }
367
hugetlb_vma_lock_write(struct vm_area_struct * vma)368 static inline void hugetlb_vma_lock_write(struct vm_area_struct *vma)
369 {
370 }
371
hugetlb_vma_unlock_write(struct vm_area_struct * vma)372 static inline void hugetlb_vma_unlock_write(struct vm_area_struct *vma)
373 {
374 }
375
hugetlb_vma_trylock_write(struct vm_area_struct * vma)376 static inline int hugetlb_vma_trylock_write(struct vm_area_struct *vma)
377 {
378 return 1;
379 }
380
hugetlb_vma_assert_locked(struct vm_area_struct * vma)381 static inline void hugetlb_vma_assert_locked(struct vm_area_struct *vma)
382 {
383 }
384
is_hugepage_only_range(struct mm_struct * mm,unsigned long addr,unsigned long len)385 static inline int is_hugepage_only_range(struct mm_struct *mm,
386 unsigned long addr, unsigned long len)
387 {
388 return 0;
389 }
390
hugetlb_free_pgd_range(struct mmu_gather * tlb,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling)391 static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb,
392 unsigned long addr, unsigned long end,
393 unsigned long floor, unsigned long ceiling)
394 {
395 BUG();
396 }
397
398 #ifdef CONFIG_USERFAULTFD
hugetlb_mfill_atomic_pte(pte_t * dst_pte,struct vm_area_struct * dst_vma,unsigned long dst_addr,unsigned long src_addr,uffd_flags_t flags,struct folio ** foliop)399 static inline int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
400 struct vm_area_struct *dst_vma,
401 unsigned long dst_addr,
402 unsigned long src_addr,
403 uffd_flags_t flags,
404 struct folio **foliop)
405 {
406 BUG();
407 return 0;
408 }
409 #endif /* CONFIG_USERFAULTFD */
410
huge_pte_offset(struct mm_struct * mm,unsigned long addr,unsigned long sz)411 static inline pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr,
412 unsigned long sz)
413 {
414 return NULL;
415 }
416
folio_isolate_hugetlb(struct folio * folio,struct list_head * list)417 static inline bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list)
418 {
419 return false;
420 }
421
get_hwpoison_hugetlb_folio(struct folio * folio,bool * hugetlb,bool unpoison)422 static inline int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison)
423 {
424 return 0;
425 }
426
get_huge_page_for_hwpoison(unsigned long pfn,int flags,bool * migratable_cleared)427 static inline int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
428 bool *migratable_cleared)
429 {
430 return 0;
431 }
432
folio_putback_hugetlb(struct folio * folio)433 static inline void folio_putback_hugetlb(struct folio *folio)
434 {
435 }
436
move_hugetlb_state(struct folio * old_folio,struct folio * new_folio,int reason)437 static inline void move_hugetlb_state(struct folio *old_folio,
438 struct folio *new_folio, int reason)
439 {
440 }
441
hugetlb_change_protection(struct vm_area_struct * vma,unsigned long address,unsigned long end,pgprot_t newprot,unsigned long cp_flags)442 static inline long hugetlb_change_protection(
443 struct vm_area_struct *vma, unsigned long address,
444 unsigned long end, pgprot_t newprot,
445 unsigned long cp_flags)
446 {
447 return 0;
448 }
449
__unmap_hugepage_range(struct mmu_gather * tlb,struct vm_area_struct * vma,unsigned long start,unsigned long end,struct page * ref_page,zap_flags_t zap_flags)450 static inline void __unmap_hugepage_range(struct mmu_gather *tlb,
451 struct vm_area_struct *vma, unsigned long start,
452 unsigned long end, struct page *ref_page,
453 zap_flags_t zap_flags)
454 {
455 BUG();
456 }
457
hugetlb_fault(struct mm_struct * mm,struct vm_area_struct * vma,unsigned long address,unsigned int flags)458 static inline vm_fault_t hugetlb_fault(struct mm_struct *mm,
459 struct vm_area_struct *vma, unsigned long address,
460 unsigned int flags)
461 {
462 BUG();
463 return 0;
464 }
465
hugetlb_unshare_all_pmds(struct vm_area_struct * vma)466 static inline void hugetlb_unshare_all_pmds(struct vm_area_struct *vma) { }
467
468 #endif /* !CONFIG_HUGETLB_PAGE */
469
470 #ifndef pgd_write
pgd_write(pgd_t pgd)471 static inline int pgd_write(pgd_t pgd)
472 {
473 BUG();
474 return 0;
475 }
476 #endif
477
478 #define HUGETLB_ANON_FILE "anon_hugepage"
479
480 enum {
481 /*
482 * The file will be used as an shm file so shmfs accounting rules
483 * apply
484 */
485 HUGETLB_SHMFS_INODE = 1,
486 /*
487 * The file is being created on the internal vfs mount and shmfs
488 * accounting rules do not apply
489 */
490 HUGETLB_ANONHUGE_INODE = 2,
491 };
492
493 #ifdef CONFIG_HUGETLBFS
494 struct hugetlbfs_sb_info {
495 long max_inodes; /* inodes allowed */
496 long free_inodes; /* inodes free */
497 spinlock_t stat_lock;
498 struct hstate *hstate;
499 struct hugepage_subpool *spool;
500 kuid_t uid;
501 kgid_t gid;
502 umode_t mode;
503 };
504
HUGETLBFS_SB(struct super_block * sb)505 static inline struct hugetlbfs_sb_info *HUGETLBFS_SB(struct super_block *sb)
506 {
507 return sb->s_fs_info;
508 }
509
510 struct hugetlbfs_inode_info {
511 struct inode vfs_inode;
512 unsigned int seals;
513 };
514
HUGETLBFS_I(struct inode * inode)515 static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode)
516 {
517 return container_of(inode, struct hugetlbfs_inode_info, vfs_inode);
518 }
519
520 extern const struct vm_operations_struct hugetlb_vm_ops;
521 struct file *hugetlb_file_setup(const char *name, size_t size, vm_flags_t acct,
522 int creat_flags, int page_size_log);
523
is_file_hugepages(const struct file * file)524 static inline bool is_file_hugepages(const struct file *file)
525 {
526 return file->f_op->fop_flags & FOP_HUGE_PAGES;
527 }
528
hstate_inode(struct inode * i)529 static inline struct hstate *hstate_inode(struct inode *i)
530 {
531 return HUGETLBFS_SB(i->i_sb)->hstate;
532 }
533 #else /* !CONFIG_HUGETLBFS */
534
535 #define is_file_hugepages(file) false
536 static inline struct file *
hugetlb_file_setup(const char * name,size_t size,vm_flags_t acctflag,int creat_flags,int page_size_log)537 hugetlb_file_setup(const char *name, size_t size, vm_flags_t acctflag,
538 int creat_flags, int page_size_log)
539 {
540 return ERR_PTR(-ENOSYS);
541 }
542
hstate_inode(struct inode * i)543 static inline struct hstate *hstate_inode(struct inode *i)
544 {
545 return NULL;
546 }
547 #endif /* !CONFIG_HUGETLBFS */
548
549 unsigned long
550 hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
551 unsigned long len, unsigned long pgoff,
552 unsigned long flags);
553
554 /*
555 * huegtlb page specific state flags. These flags are located in page.private
556 * of the hugetlb head page. Functions created via the below macros should be
557 * used to manipulate these flags.
558 *
559 * HPG_restore_reserve - Set when a hugetlb page consumes a reservation at
560 * allocation time. Cleared when page is fully instantiated. Free
561 * routine checks flag to restore a reservation on error paths.
562 * Synchronization: Examined or modified by code that knows it has
563 * the only reference to page. i.e. After allocation but before use
564 * or when the page is being freed.
565 * HPG_migratable - Set after a newly allocated page is added to the page
566 * cache and/or page tables. Indicates the page is a candidate for
567 * migration.
568 * Synchronization: Initially set after new page allocation with no
569 * locking. When examined and modified during migration processing
570 * (isolate, migrate, putback) the hugetlb_lock is held.
571 * HPG_temporary - Set on a page that is temporarily allocated from the buddy
572 * allocator. Typically used for migration target pages when no pages
573 * are available in the pool. The hugetlb free page path will
574 * immediately free pages with this flag set to the buddy allocator.
575 * Synchronization: Can be set after huge page allocation from buddy when
576 * code knows it has only reference. All other examinations and
577 * modifications require hugetlb_lock.
578 * HPG_freed - Set when page is on the free lists.
579 * Synchronization: hugetlb_lock held for examination and modification.
580 * HPG_vmemmap_optimized - Set when the vmemmap pages of the page are freed.
581 * HPG_raw_hwp_unreliable - Set when the hugetlb page has a hwpoison sub-page
582 * that is not tracked by raw_hwp_page list.
583 */
584 enum hugetlb_page_flags {
585 HPG_restore_reserve = 0,
586 HPG_migratable,
587 HPG_temporary,
588 HPG_freed,
589 HPG_vmemmap_optimized,
590 HPG_raw_hwp_unreliable,
591 __NR_HPAGEFLAGS,
592 };
593
594 /*
595 * Macros to create test, set and clear function definitions for
596 * hugetlb specific page flags.
597 */
598 #ifdef CONFIG_HUGETLB_PAGE
599 #define TESTHPAGEFLAG(uname, flname) \
600 static __always_inline \
601 bool folio_test_hugetlb_##flname(struct folio *folio) \
602 { void *private = &folio->private; \
603 return test_bit(HPG_##flname, private); \
604 }
605
606 #define SETHPAGEFLAG(uname, flname) \
607 static __always_inline \
608 void folio_set_hugetlb_##flname(struct folio *folio) \
609 { void *private = &folio->private; \
610 set_bit(HPG_##flname, private); \
611 }
612
613 #define CLEARHPAGEFLAG(uname, flname) \
614 static __always_inline \
615 void folio_clear_hugetlb_##flname(struct folio *folio) \
616 { void *private = &folio->private; \
617 clear_bit(HPG_##flname, private); \
618 }
619 #else
620 #define TESTHPAGEFLAG(uname, flname) \
621 static inline bool \
622 folio_test_hugetlb_##flname(struct folio *folio) \
623 { return 0; }
624
625 #define SETHPAGEFLAG(uname, flname) \
626 static inline void \
627 folio_set_hugetlb_##flname(struct folio *folio) \
628 { }
629
630 #define CLEARHPAGEFLAG(uname, flname) \
631 static inline void \
632 folio_clear_hugetlb_##flname(struct folio *folio) \
633 { }
634 #endif
635
636 #define HPAGEFLAG(uname, flname) \
637 TESTHPAGEFLAG(uname, flname) \
638 SETHPAGEFLAG(uname, flname) \
639 CLEARHPAGEFLAG(uname, flname) \
640
641 /*
642 * Create functions associated with hugetlb page flags
643 */
644 HPAGEFLAG(RestoreReserve, restore_reserve)
645 HPAGEFLAG(Migratable, migratable)
646 HPAGEFLAG(Temporary, temporary)
647 HPAGEFLAG(Freed, freed)
648 HPAGEFLAG(VmemmapOptimized, vmemmap_optimized)
649 HPAGEFLAG(RawHwpUnreliable, raw_hwp_unreliable)
650
651 #ifdef CONFIG_HUGETLB_PAGE
652
653 #define HSTATE_NAME_LEN 32
654 /* Defines one hugetlb page size */
655 struct hstate {
656 struct mutex resize_lock;
657 struct lock_class_key resize_key;
658 int next_nid_to_alloc;
659 int next_nid_to_free;
660 unsigned int order;
661 unsigned int demote_order;
662 unsigned long mask;
663 unsigned long max_huge_pages;
664 unsigned long nr_huge_pages;
665 unsigned long free_huge_pages;
666 unsigned long resv_huge_pages;
667 unsigned long surplus_huge_pages;
668 unsigned long nr_overcommit_huge_pages;
669 struct list_head hugepage_activelist;
670 struct list_head hugepage_freelists[MAX_NUMNODES];
671 unsigned int max_huge_pages_node[MAX_NUMNODES];
672 unsigned int nr_huge_pages_node[MAX_NUMNODES];
673 unsigned int free_huge_pages_node[MAX_NUMNODES];
674 unsigned int surplus_huge_pages_node[MAX_NUMNODES];
675 char name[HSTATE_NAME_LEN];
676 };
677
678 struct huge_bootmem_page {
679 struct list_head list;
680 struct hstate *hstate;
681 };
682
683 int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list);
684 int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn);
685 void wait_for_freed_hugetlb_folios(void);
686 struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
687 unsigned long addr, bool cow_from_owner);
688 struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid,
689 nodemask_t *nmask, gfp_t gfp_mask,
690 bool allow_alloc_fallback);
691 struct folio *alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid,
692 nodemask_t *nmask, gfp_t gfp_mask);
693
694 int hugetlb_add_to_page_cache(struct folio *folio, struct address_space *mapping,
695 pgoff_t idx);
696 void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
697 unsigned long address, struct folio *folio);
698
699 /* arch callback */
700 int __init __alloc_bootmem_huge_page(struct hstate *h, int nid);
701 int __init alloc_bootmem_huge_page(struct hstate *h, int nid);
702 bool __init hugetlb_node_alloc_supported(void);
703
704 void __init hugetlb_add_hstate(unsigned order);
705 bool __init arch_hugetlb_valid_size(unsigned long size);
706 struct hstate *size_to_hstate(unsigned long size);
707
708 #ifndef HUGE_MAX_HSTATE
709 #define HUGE_MAX_HSTATE 1
710 #endif
711
712 extern struct hstate hstates[HUGE_MAX_HSTATE];
713 extern unsigned int default_hstate_idx;
714
715 #define default_hstate (hstates[default_hstate_idx])
716
hugetlb_folio_subpool(struct folio * folio)717 static inline struct hugepage_subpool *hugetlb_folio_subpool(struct folio *folio)
718 {
719 return folio->_hugetlb_subpool;
720 }
721
hugetlb_set_folio_subpool(struct folio * folio,struct hugepage_subpool * subpool)722 static inline void hugetlb_set_folio_subpool(struct folio *folio,
723 struct hugepage_subpool *subpool)
724 {
725 folio->_hugetlb_subpool = subpool;
726 }
727
hstate_file(struct file * f)728 static inline struct hstate *hstate_file(struct file *f)
729 {
730 return hstate_inode(file_inode(f));
731 }
732
hstate_sizelog(int page_size_log)733 static inline struct hstate *hstate_sizelog(int page_size_log)
734 {
735 if (!page_size_log)
736 return &default_hstate;
737
738 if (page_size_log < BITS_PER_LONG)
739 return size_to_hstate(1UL << page_size_log);
740
741 return NULL;
742 }
743
hstate_vma(struct vm_area_struct * vma)744 static inline struct hstate *hstate_vma(struct vm_area_struct *vma)
745 {
746 return hstate_file(vma->vm_file);
747 }
748
huge_page_size(const struct hstate * h)749 static inline unsigned long huge_page_size(const struct hstate *h)
750 {
751 return (unsigned long)PAGE_SIZE << h->order;
752 }
753
754 extern unsigned long vma_kernel_pagesize(struct vm_area_struct *vma);
755
756 extern unsigned long vma_mmu_pagesize(struct vm_area_struct *vma);
757
huge_page_mask(struct hstate * h)758 static inline unsigned long huge_page_mask(struct hstate *h)
759 {
760 return h->mask;
761 }
762
huge_page_order(struct hstate * h)763 static inline unsigned int huge_page_order(struct hstate *h)
764 {
765 return h->order;
766 }
767
huge_page_shift(struct hstate * h)768 static inline unsigned huge_page_shift(struct hstate *h)
769 {
770 return h->order + PAGE_SHIFT;
771 }
772
hstate_is_gigantic(struct hstate * h)773 static inline bool hstate_is_gigantic(struct hstate *h)
774 {
775 return huge_page_order(h) > MAX_PAGE_ORDER;
776 }
777
pages_per_huge_page(const struct hstate * h)778 static inline unsigned int pages_per_huge_page(const struct hstate *h)
779 {
780 return 1 << h->order;
781 }
782
blocks_per_huge_page(struct hstate * h)783 static inline unsigned int blocks_per_huge_page(struct hstate *h)
784 {
785 return huge_page_size(h) / 512;
786 }
787
filemap_lock_hugetlb_folio(struct hstate * h,struct address_space * mapping,pgoff_t idx)788 static inline struct folio *filemap_lock_hugetlb_folio(struct hstate *h,
789 struct address_space *mapping, pgoff_t idx)
790 {
791 return filemap_lock_folio(mapping, idx << huge_page_order(h));
792 }
793
794 #include <asm/hugetlb.h>
795
796 #ifndef is_hugepage_only_range
is_hugepage_only_range(struct mm_struct * mm,unsigned long addr,unsigned long len)797 static inline int is_hugepage_only_range(struct mm_struct *mm,
798 unsigned long addr, unsigned long len)
799 {
800 return 0;
801 }
802 #define is_hugepage_only_range is_hugepage_only_range
803 #endif
804
805 #ifndef arch_clear_hugetlb_flags
arch_clear_hugetlb_flags(struct folio * folio)806 static inline void arch_clear_hugetlb_flags(struct folio *folio) { }
807 #define arch_clear_hugetlb_flags arch_clear_hugetlb_flags
808 #endif
809
810 #ifndef arch_make_huge_pte
arch_make_huge_pte(pte_t entry,unsigned int shift,vm_flags_t flags)811 static inline pte_t arch_make_huge_pte(pte_t entry, unsigned int shift,
812 vm_flags_t flags)
813 {
814 return pte_mkhuge(entry);
815 }
816 #endif
817
folio_hstate(struct folio * folio)818 static inline struct hstate *folio_hstate(struct folio *folio)
819 {
820 VM_BUG_ON_FOLIO(!folio_test_hugetlb(folio), folio);
821 return size_to_hstate(folio_size(folio));
822 }
823
hstate_index_to_shift(unsigned index)824 static inline unsigned hstate_index_to_shift(unsigned index)
825 {
826 return hstates[index].order + PAGE_SHIFT;
827 }
828
hstate_index(struct hstate * h)829 static inline int hstate_index(struct hstate *h)
830 {
831 return h - hstates;
832 }
833
834 int dissolve_free_hugetlb_folio(struct folio *folio);
835 int dissolve_free_hugetlb_folios(unsigned long start_pfn,
836 unsigned long end_pfn);
837
838 #ifdef CONFIG_MEMORY_FAILURE
839 extern void folio_clear_hugetlb_hwpoison(struct folio *folio);
840 #else
folio_clear_hugetlb_hwpoison(struct folio * folio)841 static inline void folio_clear_hugetlb_hwpoison(struct folio *folio)
842 {
843 }
844 #endif
845
846 #ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
847 #ifndef arch_hugetlb_migration_supported
arch_hugetlb_migration_supported(struct hstate * h)848 static inline bool arch_hugetlb_migration_supported(struct hstate *h)
849 {
850 if ((huge_page_shift(h) == PMD_SHIFT) ||
851 (huge_page_shift(h) == PUD_SHIFT) ||
852 (huge_page_shift(h) == PGDIR_SHIFT))
853 return true;
854 else
855 return false;
856 }
857 #endif
858 #else
arch_hugetlb_migration_supported(struct hstate * h)859 static inline bool arch_hugetlb_migration_supported(struct hstate *h)
860 {
861 return false;
862 }
863 #endif
864
hugepage_migration_supported(struct hstate * h)865 static inline bool hugepage_migration_supported(struct hstate *h)
866 {
867 return arch_hugetlb_migration_supported(h);
868 }
869
870 /*
871 * Movability check is different as compared to migration check.
872 * It determines whether or not a huge page should be placed on
873 * movable zone or not. Movability of any huge page should be
874 * required only if huge page size is supported for migration.
875 * There won't be any reason for the huge page to be movable if
876 * it is not migratable to start with. Also the size of the huge
877 * page should be large enough to be placed under a movable zone
878 * and still feasible enough to be migratable. Just the presence
879 * in movable zone does not make the migration feasible.
880 *
881 * So even though large huge page sizes like the gigantic ones
882 * are migratable they should not be movable because its not
883 * feasible to migrate them from movable zone.
884 */
hugepage_movable_supported(struct hstate * h)885 static inline bool hugepage_movable_supported(struct hstate *h)
886 {
887 if (!hugepage_migration_supported(h))
888 return false;
889
890 if (hstate_is_gigantic(h))
891 return false;
892 return true;
893 }
894
895 /* Movability of hugepages depends on migration support. */
htlb_alloc_mask(struct hstate * h)896 static inline gfp_t htlb_alloc_mask(struct hstate *h)
897 {
898 gfp_t gfp = __GFP_COMP | __GFP_NOWARN;
899
900 gfp |= hugepage_movable_supported(h) ? GFP_HIGHUSER_MOVABLE : GFP_HIGHUSER;
901
902 return gfp;
903 }
904
htlb_modify_alloc_mask(struct hstate * h,gfp_t gfp_mask)905 static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask)
906 {
907 gfp_t modified_mask = htlb_alloc_mask(h);
908
909 /* Some callers might want to enforce node */
910 modified_mask |= (gfp_mask & __GFP_THISNODE);
911
912 modified_mask |= (gfp_mask & __GFP_NOWARN);
913
914 return modified_mask;
915 }
916
htlb_allow_alloc_fallback(int reason)917 static inline bool htlb_allow_alloc_fallback(int reason)
918 {
919 bool allowed_fallback = false;
920
921 /*
922 * Note: the memory offline, memory failure and migration syscalls will
923 * be allowed to fallback to other nodes due to lack of a better chioce,
924 * that might break the per-node hugetlb pool. While other cases will
925 * set the __GFP_THISNODE to avoid breaking the per-node hugetlb pool.
926 */
927 switch (reason) {
928 case MR_MEMORY_HOTPLUG:
929 case MR_MEMORY_FAILURE:
930 case MR_SYSCALL:
931 case MR_MEMPOLICY_MBIND:
932 allowed_fallback = true;
933 break;
934 default:
935 break;
936 }
937
938 return allowed_fallback;
939 }
940
huge_pte_lockptr(struct hstate * h,struct mm_struct * mm,pte_t * pte)941 static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
942 struct mm_struct *mm, pte_t *pte)
943 {
944 const unsigned long size = huge_page_size(h);
945
946 VM_WARN_ON(size == PAGE_SIZE);
947
948 /*
949 * hugetlb must use the exact same PT locks as core-mm page table
950 * walkers would. When modifying a PTE table, hugetlb must take the
951 * PTE PT lock, when modifying a PMD table, hugetlb must take the PMD
952 * PT lock etc.
953 *
954 * The expectation is that any hugetlb folio smaller than a PMD is
955 * always mapped into a single PTE table and that any hugetlb folio
956 * smaller than a PUD (but at least as big as a PMD) is always mapped
957 * into a single PMD table.
958 *
959 * If that does not hold for an architecture, then that architecture
960 * must disable split PT locks such that all *_lockptr() functions
961 * will give us the same result: the per-MM PT lock.
962 *
963 * Note that with e.g., CONFIG_PGTABLE_LEVELS=2 where
964 * PGDIR_SIZE==P4D_SIZE==PUD_SIZE==PMD_SIZE, we'd use pud_lockptr()
965 * and core-mm would use pmd_lockptr(). However, in such configurations
966 * split PMD locks are disabled -- they don't make sense on a single
967 * PGDIR page table -- and the end result is the same.
968 */
969 if (size >= PUD_SIZE)
970 return pud_lockptr(mm, (pud_t *) pte);
971 else if (size >= PMD_SIZE || IS_ENABLED(CONFIG_HIGHPTE))
972 return pmd_lockptr(mm, (pmd_t *) pte);
973 /* pte_alloc_huge() only applies with !CONFIG_HIGHPTE */
974 return ptep_lockptr(mm, pte);
975 }
976
977 #ifndef hugepages_supported
978 /*
979 * Some platform decide whether they support huge pages at boot
980 * time. Some of them, such as powerpc, set HPAGE_SHIFT to 0
981 * when there is no such support
982 */
983 #define hugepages_supported() (HPAGE_SHIFT != 0)
984 #endif
985
986 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm);
987
hugetlb_count_init(struct mm_struct * mm)988 static inline void hugetlb_count_init(struct mm_struct *mm)
989 {
990 atomic_long_set(&mm->hugetlb_usage, 0);
991 }
992
hugetlb_count_add(long l,struct mm_struct * mm)993 static inline void hugetlb_count_add(long l, struct mm_struct *mm)
994 {
995 atomic_long_add(l, &mm->hugetlb_usage);
996 }
997
hugetlb_count_sub(long l,struct mm_struct * mm)998 static inline void hugetlb_count_sub(long l, struct mm_struct *mm)
999 {
1000 atomic_long_sub(l, &mm->hugetlb_usage);
1001 }
1002
1003 #ifndef huge_ptep_modify_prot_start
1004 #define huge_ptep_modify_prot_start huge_ptep_modify_prot_start
huge_ptep_modify_prot_start(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep)1005 static inline pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
1006 unsigned long addr, pte_t *ptep)
1007 {
1008 unsigned long psize = huge_page_size(hstate_vma(vma));
1009
1010 return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, psize);
1011 }
1012 #endif
1013
1014 #ifndef huge_ptep_modify_prot_commit
1015 #define huge_ptep_modify_prot_commit huge_ptep_modify_prot_commit
huge_ptep_modify_prot_commit(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep,pte_t old_pte,pte_t pte)1016 static inline void huge_ptep_modify_prot_commit(struct vm_area_struct *vma,
1017 unsigned long addr, pte_t *ptep,
1018 pte_t old_pte, pte_t pte)
1019 {
1020 unsigned long psize = huge_page_size(hstate_vma(vma));
1021
1022 set_huge_pte_at(vma->vm_mm, addr, ptep, pte, psize);
1023 }
1024 #endif
1025
1026 #ifdef CONFIG_NUMA
1027 void hugetlb_register_node(struct node *node);
1028 void hugetlb_unregister_node(struct node *node);
1029 #endif
1030
1031 /*
1032 * Check if a given raw @page in a hugepage is HWPOISON.
1033 */
1034 bool is_raw_hwpoison_page_in_hugepage(struct page *page);
1035
huge_page_mask_align(struct file * file)1036 static inline unsigned long huge_page_mask_align(struct file *file)
1037 {
1038 return PAGE_MASK & ~huge_page_mask(hstate_file(file));
1039 }
1040
1041 #else /* CONFIG_HUGETLB_PAGE */
1042 struct hstate {};
1043
1044 static inline unsigned long huge_page_mask_align(struct file *file)
1045 {
1046 return 0;
1047 }
1048
1049 static inline struct hugepage_subpool *hugetlb_folio_subpool(struct folio *folio)
1050 {
1051 return NULL;
1052 }
1053
1054 static inline struct folio *filemap_lock_hugetlb_folio(struct hstate *h,
1055 struct address_space *mapping, pgoff_t idx)
1056 {
1057 return NULL;
1058 }
1059
1060 static inline int isolate_or_dissolve_huge_page(struct page *page,
1061 struct list_head *list)
1062 {
1063 return -ENOMEM;
1064 }
1065
1066 static inline int replace_free_hugepage_folios(unsigned long start_pfn,
1067 unsigned long end_pfn)
1068 {
1069 return 0;
1070 }
1071
1072 static inline void wait_for_freed_hugetlb_folios(void)
1073 {
1074 }
1075
1076 static inline struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
1077 unsigned long addr,
1078 bool cow_from_owner)
1079 {
1080 return NULL;
1081 }
1082
1083 static inline struct folio *
1084 alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid,
1085 nodemask_t *nmask, gfp_t gfp_mask)
1086 {
1087 return NULL;
1088 }
1089
1090 static inline struct folio *
1091 alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid,
1092 nodemask_t *nmask, gfp_t gfp_mask,
1093 bool allow_alloc_fallback)
1094 {
1095 return NULL;
1096 }
1097
1098 static inline int __alloc_bootmem_huge_page(struct hstate *h)
1099 {
1100 return 0;
1101 }
1102
1103 static inline struct hstate *hstate_file(struct file *f)
1104 {
1105 return NULL;
1106 }
1107
1108 static inline struct hstate *hstate_sizelog(int page_size_log)
1109 {
1110 return NULL;
1111 }
1112
1113 static inline struct hstate *hstate_vma(struct vm_area_struct *vma)
1114 {
1115 return NULL;
1116 }
1117
1118 static inline struct hstate *folio_hstate(struct folio *folio)
1119 {
1120 return NULL;
1121 }
1122
1123 static inline struct hstate *size_to_hstate(unsigned long size)
1124 {
1125 return NULL;
1126 }
1127
1128 static inline unsigned long huge_page_size(struct hstate *h)
1129 {
1130 return PAGE_SIZE;
1131 }
1132
1133 static inline unsigned long huge_page_mask(struct hstate *h)
1134 {
1135 return PAGE_MASK;
1136 }
1137
1138 static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
1139 {
1140 return PAGE_SIZE;
1141 }
1142
1143 static inline unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
1144 {
1145 return PAGE_SIZE;
1146 }
1147
1148 static inline unsigned int huge_page_order(struct hstate *h)
1149 {
1150 return 0;
1151 }
1152
1153 static inline unsigned int huge_page_shift(struct hstate *h)
1154 {
1155 return PAGE_SHIFT;
1156 }
1157
1158 static inline bool hstate_is_gigantic(struct hstate *h)
1159 {
1160 return false;
1161 }
1162
1163 static inline unsigned int pages_per_huge_page(struct hstate *h)
1164 {
1165 return 1;
1166 }
1167
1168 static inline unsigned hstate_index_to_shift(unsigned index)
1169 {
1170 return 0;
1171 }
1172
1173 static inline int hstate_index(struct hstate *h)
1174 {
1175 return 0;
1176 }
1177
1178 static inline int dissolve_free_hugetlb_folio(struct folio *folio)
1179 {
1180 return 0;
1181 }
1182
1183 static inline int dissolve_free_hugetlb_folios(unsigned long start_pfn,
1184 unsigned long end_pfn)
1185 {
1186 return 0;
1187 }
1188
1189 static inline bool hugepage_migration_supported(struct hstate *h)
1190 {
1191 return false;
1192 }
1193
1194 static inline bool hugepage_movable_supported(struct hstate *h)
1195 {
1196 return false;
1197 }
1198
1199 static inline gfp_t htlb_alloc_mask(struct hstate *h)
1200 {
1201 return 0;
1202 }
1203
1204 static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask)
1205 {
1206 return 0;
1207 }
1208
1209 static inline bool htlb_allow_alloc_fallback(int reason)
1210 {
1211 return false;
1212 }
1213
1214 static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
1215 struct mm_struct *mm, pte_t *pte)
1216 {
1217 return &mm->page_table_lock;
1218 }
1219
1220 static inline void hugetlb_count_init(struct mm_struct *mm)
1221 {
1222 }
1223
1224 static inline void hugetlb_report_usage(struct seq_file *f, struct mm_struct *m)
1225 {
1226 }
1227
1228 static inline void hugetlb_count_sub(long l, struct mm_struct *mm)
1229 {
1230 }
1231
1232 static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
1233 unsigned long addr, pte_t *ptep)
1234 {
1235 #ifdef CONFIG_MMU
1236 return ptep_get(ptep);
1237 #else
1238 return *ptep;
1239 #endif
1240 }
1241
1242 static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
1243 pte_t *ptep, pte_t pte, unsigned long sz)
1244 {
1245 }
1246
1247 static inline void hugetlb_register_node(struct node *node)
1248 {
1249 }
1250
1251 static inline void hugetlb_unregister_node(struct node *node)
1252 {
1253 }
1254
1255 static inline bool hugetlbfs_pagecache_present(
1256 struct hstate *h, struct vm_area_struct *vma, unsigned long address)
1257 {
1258 return false;
1259 }
1260 #endif /* CONFIG_HUGETLB_PAGE */
1261
huge_pte_lock(struct hstate * h,struct mm_struct * mm,pte_t * pte)1262 static inline spinlock_t *huge_pte_lock(struct hstate *h,
1263 struct mm_struct *mm, pte_t *pte)
1264 {
1265 spinlock_t *ptl;
1266
1267 ptl = huge_pte_lockptr(h, mm, pte);
1268 spin_lock(ptl);
1269 return ptl;
1270 }
1271
1272 #if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_CMA)
1273 extern void __init hugetlb_cma_reserve(int order);
1274 #else
hugetlb_cma_reserve(int order)1275 static inline __init void hugetlb_cma_reserve(int order)
1276 {
1277 }
1278 #endif
1279
1280 #ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
hugetlb_pmd_shared(pte_t * pte)1281 static inline bool hugetlb_pmd_shared(pte_t *pte)
1282 {
1283 return page_count(virt_to_page(pte)) > 1;
1284 }
1285 #else
hugetlb_pmd_shared(pte_t * pte)1286 static inline bool hugetlb_pmd_shared(pte_t *pte)
1287 {
1288 return false;
1289 }
1290 #endif
1291
1292 bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr);
1293
1294 #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
1295 /*
1296 * ARCHes with special requirements for evicting HUGETLB backing TLB entries can
1297 * implement this.
1298 */
1299 #define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
1300 #endif
1301
__vma_shareable_lock(struct vm_area_struct * vma)1302 static inline bool __vma_shareable_lock(struct vm_area_struct *vma)
1303 {
1304 return (vma->vm_flags & VM_MAYSHARE) && vma->vm_private_data;
1305 }
1306
1307 bool __vma_private_lock(struct vm_area_struct *vma);
1308
1309 /*
1310 * Safe version of huge_pte_offset() to check the locks. See comments
1311 * above huge_pte_offset().
1312 */
1313 static inline pte_t *
hugetlb_walk(struct vm_area_struct * vma,unsigned long addr,unsigned long sz)1314 hugetlb_walk(struct vm_area_struct *vma, unsigned long addr, unsigned long sz)
1315 {
1316 #if defined(CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING) && defined(CONFIG_LOCKDEP)
1317 struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
1318
1319 /*
1320 * If pmd sharing possible, locking needed to safely walk the
1321 * hugetlb pgtables. More information can be found at the comment
1322 * above huge_pte_offset() in the same file.
1323 *
1324 * NOTE: lockdep_is_held() is only defined with CONFIG_LOCKDEP.
1325 */
1326 if (__vma_shareable_lock(vma))
1327 WARN_ON_ONCE(!lockdep_is_held(&vma_lock->rw_sema) &&
1328 !lockdep_is_held(
1329 &vma->vm_file->f_mapping->i_mmap_rwsem));
1330 #endif
1331 return huge_pte_offset(vma->vm_mm, addr, sz);
1332 }
1333
1334 #endif /* _LINUX_HUGETLB_H */
1335