1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */
2c6557e7fSMartin Schwidefsky /*
3c6557e7fSMartin Schwidefsky * IBM System z Huge TLB Page Support for Kernel.
4c6557e7fSMartin Schwidefsky *
5c6557e7fSMartin Schwidefsky * Copyright IBM Corp. 2008
6c6557e7fSMartin Schwidefsky * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
7c6557e7fSMartin Schwidefsky */
8c6557e7fSMartin Schwidefsky
9c6557e7fSMartin Schwidefsky #ifndef _ASM_S390_HUGETLB_H
10c6557e7fSMartin Schwidefsky #define _ASM_S390_HUGETLB_H
11c6557e7fSMartin Schwidefsky
12ca5999fdSMike Rapoport #include <linux/pgtable.h>
1365fddcfcSMike Rapoport #include <asm/page.h>
14c6557e7fSMartin Schwidefsky
15c6557e7fSMartin Schwidefsky #define hugetlb_free_pgd_range free_pgd_range
16466178fcSHeiko Carstens #define hugepages_supported() (MACHINE_HAS_EDAT1)
17c6557e7fSMartin Schwidefsky
18c6557e7fSMartin Schwidefsky void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
19935d4f0cSRyan Roberts pte_t *ptep, pte_t pte, unsigned long sz);
20935d4f0cSRyan Roberts void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
21c6557e7fSMartin Schwidefsky pte_t *ptep, pte_t pte);
22*e6c0c032SChristophe Leroy pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep);
23e5098611SMartin Schwidefsky pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
24e5098611SMartin Schwidefsky unsigned long addr, pte_t *ptep);
25c6557e7fSMartin Schwidefsky
26c6557e7fSMartin Schwidefsky /*
27c6557e7fSMartin Schwidefsky * If the arch doesn't supply something else, assume that hugepage
28c6557e7fSMartin Schwidefsky * size aligned regions are ok without further preparation.
29c6557e7fSMartin Schwidefsky */
prepare_hugepage_range(struct file * file,unsigned long addr,unsigned long len)30c6557e7fSMartin Schwidefsky static inline int prepare_hugepage_range(struct file *file,
31c6557e7fSMartin Schwidefsky unsigned long addr, unsigned long len)
32c6557e7fSMartin Schwidefsky {
337c8d42fdSGerald Schaefer struct hstate *h = hstate_file(file);
347c8d42fdSGerald Schaefer
357c8d42fdSGerald Schaefer if (len & ~huge_page_mask(h))
36c6557e7fSMartin Schwidefsky return -EINVAL;
377c8d42fdSGerald Schaefer if (addr & ~huge_page_mask(h))
38c6557e7fSMartin Schwidefsky return -EINVAL;
39c6557e7fSMartin Schwidefsky return 0;
40c6557e7fSMartin Schwidefsky }
41c6557e7fSMartin Schwidefsky
arch_clear_hugetlb_flags(struct folio * folio)4251718e25SMatthew Wilcox (Oracle) static inline void arch_clear_hugetlb_flags(struct folio *folio)
433afdfca6SJanosch Frank {
4451718e25SMatthew Wilcox (Oracle) clear_bit(PG_arch_1, &folio->flags);
453afdfca6SJanosch Frank }
4651718e25SMatthew Wilcox (Oracle) #define arch_clear_hugetlb_flags arch_clear_hugetlb_flags
47c6557e7fSMartin Schwidefsky
huge_pte_clear(struct mm_struct * mm,unsigned long addr,pte_t * ptep,unsigned long sz)48e5098611SMartin Schwidefsky static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
499386fac3SPunit Agrawal pte_t *ptep, unsigned long sz)
50c6557e7fSMartin Schwidefsky {
51d08de8e2SGerald Schaefer if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
52b8e3b379SHeiko Carstens set_pte(ptep, __pte(_REGION3_ENTRY_EMPTY));
53d08de8e2SGerald Schaefer else
54b8e3b379SHeiko Carstens set_pte(ptep, __pte(_SEGMENT_ENTRY_EMPTY));
55c6557e7fSMartin Schwidefsky }
56c6557e7fSMartin Schwidefsky
huge_ptep_clear_flush(struct vm_area_struct * vma,unsigned long address,pte_t * ptep)57ae075629SBaolin Wang static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
58c6557e7fSMartin Schwidefsky unsigned long address, pte_t *ptep)
59c6557e7fSMartin Schwidefsky {
60ae075629SBaolin Wang return huge_ptep_get_and_clear(vma->vm_mm, address, ptep);
61e5098611SMartin Schwidefsky }
62e5098611SMartin Schwidefsky
huge_ptep_set_access_flags(struct vm_area_struct * vma,unsigned long addr,pte_t * ptep,pte_t pte,int dirty)63e5098611SMartin Schwidefsky static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma,
64e5098611SMartin Schwidefsky unsigned long addr, pte_t *ptep,
65e5098611SMartin Schwidefsky pte_t pte, int dirty)
66e5098611SMartin Schwidefsky {
67*e6c0c032SChristophe Leroy int changed = !pte_same(huge_ptep_get(vma->vm_mm, addr, ptep), pte);
68e5098611SMartin Schwidefsky if (changed) {
69e5098611SMartin Schwidefsky huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
70935d4f0cSRyan Roberts __set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
71e5098611SMartin Schwidefsky }
72e5098611SMartin Schwidefsky return changed;
73e5098611SMartin Schwidefsky }
74e5098611SMartin Schwidefsky
huge_ptep_set_wrprotect(struct mm_struct * mm,unsigned long addr,pte_t * ptep)75e5098611SMartin Schwidefsky static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
76e5098611SMartin Schwidefsky unsigned long addr, pte_t *ptep)
77e5098611SMartin Schwidefsky {
78e5098611SMartin Schwidefsky pte_t pte = huge_ptep_get_and_clear(mm, addr, ptep);
79935d4f0cSRyan Roberts __set_huge_pte_at(mm, addr, ptep, pte_wrprotect(pte));
80c6557e7fSMartin Schwidefsky }
81c6557e7fSMartin Schwidefsky
mk_huge_pte(struct page * page,pgprot_t pgprot)82106c992aSGerald Schaefer static inline pte_t mk_huge_pte(struct page *page, pgprot_t pgprot)
83106c992aSGerald Schaefer {
84e5098611SMartin Schwidefsky return mk_pte(page, pgprot);
85e5098611SMartin Schwidefsky }
86106c992aSGerald Schaefer
huge_pte_none(pte_t pte)87e5098611SMartin Schwidefsky static inline int huge_pte_none(pte_t pte)
88e5098611SMartin Schwidefsky {
89e5098611SMartin Schwidefsky return pte_none(pte);
90106c992aSGerald Schaefer }
91106c992aSGerald Schaefer
huge_pte_none_mostly(pte_t pte)92679d1033SPeter Xu static inline int huge_pte_none_mostly(pte_t pte)
93679d1033SPeter Xu {
94679d1033SPeter Xu return huge_pte_none(pte);
95679d1033SPeter Xu }
96679d1033SPeter Xu
huge_pte_write(pte_t pte)97106c992aSGerald Schaefer static inline int huge_pte_write(pte_t pte)
98106c992aSGerald Schaefer {
99e5098611SMartin Schwidefsky return pte_write(pte);
100106c992aSGerald Schaefer }
101106c992aSGerald Schaefer
huge_pte_dirty(pte_t pte)102106c992aSGerald Schaefer static inline int huge_pte_dirty(pte_t pte)
103106c992aSGerald Schaefer {
104e5098611SMartin Schwidefsky return pte_dirty(pte);
105106c992aSGerald Schaefer }
106106c992aSGerald Schaefer
huge_pte_mkwrite(pte_t pte)107106c992aSGerald Schaefer static inline pte_t huge_pte_mkwrite(pte_t pte)
108106c992aSGerald Schaefer {
1092f0584f3SRick Edgecombe return pte_mkwrite_novma(pte);
110106c992aSGerald Schaefer }
111106c992aSGerald Schaefer
huge_pte_mkdirty(pte_t pte)112106c992aSGerald Schaefer static inline pte_t huge_pte_mkdirty(pte_t pte)
113106c992aSGerald Schaefer {
114e5098611SMartin Schwidefsky return pte_mkdirty(pte);
115e5098611SMartin Schwidefsky }
116e5098611SMartin Schwidefsky
huge_pte_wrprotect(pte_t pte)117e5098611SMartin Schwidefsky static inline pte_t huge_pte_wrprotect(pte_t pte)
118e5098611SMartin Schwidefsky {
119e5098611SMartin Schwidefsky return pte_wrprotect(pte);
120106c992aSGerald Schaefer }
121106c992aSGerald Schaefer
huge_pte_modify(pte_t pte,pgprot_t newprot)122106c992aSGerald Schaefer static inline pte_t huge_pte_modify(pte_t pte, pgprot_t newprot)
123106c992aSGerald Schaefer {
124e5098611SMartin Schwidefsky return pte_modify(pte, newprot);
125106c992aSGerald Schaefer }
126106c992aSGerald Schaefer
huge_pte_mkuffd_wp(pte_t pte)127229f3fa7SPeter Xu static inline pte_t huge_pte_mkuffd_wp(pte_t pte)
128229f3fa7SPeter Xu {
129229f3fa7SPeter Xu return pte;
130229f3fa7SPeter Xu }
131229f3fa7SPeter Xu
huge_pte_clear_uffd_wp(pte_t pte)132229f3fa7SPeter Xu static inline pte_t huge_pte_clear_uffd_wp(pte_t pte)
133229f3fa7SPeter Xu {
134229f3fa7SPeter Xu return pte;
135229f3fa7SPeter Xu }
136229f3fa7SPeter Xu
huge_pte_uffd_wp(pte_t pte)137229f3fa7SPeter Xu static inline int huge_pte_uffd_wp(pte_t pte)
138229f3fa7SPeter Xu {
139229f3fa7SPeter Xu return 0;
140229f3fa7SPeter Xu }
141229f3fa7SPeter Xu
gigantic_page_runtime_supported(void)1424eb0716eSAlexandre Ghiti static inline bool gigantic_page_runtime_supported(void)
1434eb0716eSAlexandre Ghiti {
1444eb0716eSAlexandre Ghiti return true;
1454eb0716eSAlexandre Ghiti }
1464eb0716eSAlexandre Ghiti
147c6557e7fSMartin Schwidefsky #endif /* _ASM_S390_HUGETLB_H */
148