Home
last modified time | relevance | path

Searched refs:p4d (Results 1 – 25 of 136) sorted by relevance

123456

/linux/include/asm-generic/
H A Dpgtable-nopud.h16 typedef struct { p4d_t p4d; } pud_t; member
28 static inline int p4d_none(p4d_t p4d) { return 0; } in p4d_none() argument
29 static inline int p4d_bad(p4d_t p4d) { return 0; } in p4d_bad() argument
30 static inline int p4d_present(p4d_t p4d) { return 1; } in p4d_present() argument
31 static inline void p4d_clear(p4d_t *p4d) { } in p4d_clear() argument
32 #define pud_ERROR(pud) (p4d_ERROR((pud).p4d))
34 #define p4d_populate(mm, p4d, pud) do { } while (0) argument
35 #define p4d_populate_safe(mm, p4d, pud) do { } while (0) argument
42 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) in pud_offset() argument
44 return (pud_t *)p4d; in pud_offset()
[all …]
/linux/mm/kasan/
H A Dinit.c45 static inline bool kasan_pud_table(p4d_t p4d) in kasan_pud_table() argument
47 return p4d_page(p4d) == virt_to_page(lm_alias(kasan_early_shadow_pud)); in kasan_pud_table()
50 static inline bool kasan_pud_table(p4d_t p4d) in kasan_pud_table() argument
144 static int __ref zero_pud_populate(p4d_t *p4d, unsigned long addr, in zero_pud_populate() argument
147 pud_t *pud = pud_offset(p4d, addr); in zero_pud_populate()
185 p4d_t *p4d = p4d_offset(pgd, addr); in zero_p4d_populate() local
194 p4d_populate_kernel(addr, p4d, in zero_p4d_populate()
196 pud = pud_offset(p4d, addr); in zero_p4d_populate()
205 if (p4d_none(*p4d)) { in zero_p4d_populate()
209 p = pud_alloc(&init_mm, p4d, addr); in zero_p4d_populate()
[all …]
/linux/arch/riscv/include/asm/
H A Dpgtable-64.h46 unsigned long p4d; member
49 #define p4d_val(x) ((x).p4d)
276 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) in set_p4d() argument
279 WRITE_ONCE(*p4dp, p4d); in set_p4d()
281 set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) }); in set_p4d()
284 static inline int p4d_none(p4d_t p4d) in p4d_none() argument
287 return (p4d_val(p4d) == 0); in p4d_none()
292 static inline int p4d_present(p4d_t p4d) in p4d_present() argument
295 return (p4d_val(p4d) & _PAGE_PRESENT); in p4d_present()
300 static inline int p4d_bad(p4d_t p4d) in p4d_bad() argument
[all …]
H A Dpgalloc.h42 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) in p4d_populate() argument
47 set_p4d(p4d, __p4d((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); in p4d_populate()
51 static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, in p4d_populate_safe() argument
57 set_p4d_safe(p4d, in p4d_populate_safe()
62 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) in pgd_populate() argument
65 unsigned long pfn = virt_to_pfn(p4d); in pgd_populate()
72 p4d_t *p4d) in pgd_populate_safe() argument
75 unsigned long pfn = virt_to_pfn(p4d); in pgd_populate_safe()
96 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, in __p4d_free_tlb() argument
100 tlb_remove_ptdesc(tlb, virt_to_ptdesc(p4d)); in __p4d_free_tlb()
/linux/arch/powerpc/include/asm/nohash/64/
H A Dpgtable-4k.h56 #define p4d_none(p4d) (!p4d_val(p4d)) argument
57 #define p4d_bad(p4d) (p4d_val(p4d) == 0) argument
58 #define p4d_present(p4d) (p4d_val(p4d) != 0) argument
62 static inline pud_t *p4d_pgtable(p4d_t p4d) in p4d_pgtable() argument
64 return (pud_t *) (p4d_val(p4d) & ~P4D_MASKED_BITS); in p4d_pgtable()
72 static inline pte_t p4d_pte(p4d_t p4d) in p4d_pte() argument
74 return __pte(p4d_val(p4d)); in p4d_pte()
81 extern struct page *p4d_page(p4d_t p4d);
/linux/arch/x86/mm/
H A Dinit_64.c73 DEFINE_POPULATE(p4d_populate, p4d, pud, init) in DEFINE_POPULATE() argument
74 DEFINE_POPULATE(pgd_populate, pgd, p4d, init) in DEFINE_POPULATE()
88 DEFINE_ENTRY(p4d, p4d, init)
192 p4d_t *p4d; in sync_global_pgds_l4() local
196 p4d = p4d_offset(pgd, addr); in sync_global_pgds_l4()
201 if (!p4d_none(*p4d_ref) && !p4d_none(*p4d)) in sync_global_pgds_l4()
202 BUG_ON(p4d_pgtable(*p4d) in sync_global_pgds_l4()
205 if (p4d_none(*p4d)) in sync_global_pgds_l4()
206 set_p4d(p4d, *p4d_ref); in sync_global_pgds_l4()
270 p4d_t *p4d = (p4d_t *)spp_getpage(); in fill_p4d() local
[all …]
H A Dkasan_init_64.c102 static void __init kasan_populate_p4d(p4d_t *p4d, unsigned long addr, in kasan_populate_p4d() argument
108 if (p4d_none(*p4d)) { in kasan_populate_p4d()
111 p4d_populate(&init_mm, p4d, p); in kasan_populate_p4d()
114 pud = pud_offset(p4d, addr); in kasan_populate_p4d()
126 p4d_t *p4d; in kasan_populate_pgd() local
134 p4d = p4d_offset(pgd, addr); in kasan_populate_pgd()
137 kasan_populate_p4d(p4d, addr, next, nid); in kasan_populate_pgd()
138 } while (p4d++, addr = next, addr != end); in kasan_populate_pgd()
193 unsigned long p4d; in early_p4d_offset() local
198 p4d = pgd_val(*pgd) & PTE_PFN_MASK; in early_p4d_offset()
[all …]
H A Dpti.c209 p4d_t *p4d; in pti_user_pagetable_walk_pmd() local
212 p4d = pti_user_pagetable_walk_p4d(address); in pti_user_pagetable_walk_pmd()
213 if (!p4d) in pti_user_pagetable_walk_pmd()
216 BUILD_BUG_ON(p4d_leaf(*p4d)); in pti_user_pagetable_walk_pmd()
217 if (p4d_none(*p4d)) { in pti_user_pagetable_walk_pmd()
222 set_p4d(p4d, __p4d(_KERNPG_TABLE | __pa(new_pud_page))); in pti_user_pagetable_walk_pmd()
225 pud = pud_offset(p4d, address); in pti_user_pagetable_walk_pmd()
328 p4d_t *p4d; in pti_clone_pgtable() local
338 p4d = p4d_offset(pgd, addr); in pti_clone_pgtable()
339 if (WARN_ON(p4d_none(*p4d))) in pti_clone_pgtable()
[all …]
/linux/arch/x86/include/asm/
H A Dpgalloc.h111 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) in p4d_populate() argument
114 set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud))); in p4d_populate()
117 static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) in p4d_populate_safe() argument
120 set_p4d_safe(p4d, __p4d(_PAGE_TABLE | __pa(pud))); in p4d_populate_safe()
132 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) in pgd_populate() argument
136 paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); in pgd_populate()
137 set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); in pgd_populate()
140 static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) in pgd_populate_safe() argument
144 paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT); in pgd_populate_safe()
145 set_pgd_safe(pgd, __pgd(_PAGE_TABLE | __pa(p4d))); in pgd_populate_safe()
[all …]
/linux/arch/s390/mm/
H A Dvmem.c315 static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end, in modify_pud_table() argument
324 pud = pud_offset(p4d, addr); in modify_pud_table()
371 static void try_free_pud_table(p4d_t *p4d, unsigned long start) in try_free_pud_table() argument
376 pud = pud_offset(p4d, start); in try_free_pud_table()
381 vmem_free_pages(p4d_deref(*p4d), CRST_ALLOC_ORDER, NULL); in try_free_pud_table()
382 p4d_clear(p4d); in try_free_pud_table()
390 p4d_t *p4d; in modify_p4d_table() local
393 p4d = p4d_offset(pgd, addr); in modify_p4d_table()
394 for (; addr < end; addr = next, p4d++) { in modify_p4d_table()
397 if (p4d_none(*p4d)) in modify_p4d_table()
[all …]
/linux/arch/loongarch/mm/
H A Dhugetlbpage.c20 p4d_t *p4d; in huge_pte_alloc() local
25 p4d = p4d_alloc(mm, pgd, addr); in huge_pte_alloc()
26 pud = pud_alloc(mm, p4d, addr); in huge_pte_alloc()
37 p4d_t *p4d; in huge_pte_offset() local
43 p4d = p4d_offset(pgd, addr); in huge_pte_offset()
44 if (p4d_present(p4dp_get(p4d))) { in huge_pte_offset()
45 pud = pud_offset(p4d, addr); in huge_pte_offset()
/linux/arch/um/include/asm/
H A Dpgtable-4level.h69 #define p4d_populate(mm, p4d, pud) \ argument
70 set_p4d(p4d, __p4d(_PAGE_TABLE + __pa(pud)))
89 static inline void p4d_clear (p4d_t *p4d) in p4d_clear() argument
91 set_p4d(p4d, __p4d(_PAGE_NEEDSYNC)); in p4d_clear()
97 #define p4d_page(p4d) phys_to_page(p4d_val(p4d) & PAGE_MASK) argument
98 #define p4d_pgtable(p4d) ((pud_t *) __va(p4d_val(p4d) & PAGE_MASK)) argument
/linux/arch/x86/boot/startup/
H A Dmap_kernel.c96 p4dval_t *p4d; in __startup_64() local
130 p4d = (p4dval_t *)rip_rel_ptr(level4_kernel_pgt); in __startup_64()
131 p4d[MAX_PTRS_PER_P4D - 1] += load_delta; in __startup_64()
133 pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE; in __startup_64()
156 p4d = &early_pgts[next_early_pgt++]->pmd; in __startup_64()
159 pgd[i + 0] = (pgdval_t)p4d + pgtable_flags; in __startup_64()
160 pgd[i + 1] = (pgdval_t)p4d + pgtable_flags; in __startup_64()
163 p4d[(i + 0) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags; in __startup_64()
164 p4d[(i + 1) % PTRS_PER_P4D] = (pgdval_t)pud + pgtable_flags; in __startup_64()
/linux/arch/x86/hyperv/
H A Dhv_crash.c120 p4d_t *p4d; in hv_crash_clear_kernpt() local
124 p4d = p4d_offset(pgd, trampoline_pa); in hv_crash_clear_kernpt()
125 native_p4d_clear(p4d); in hv_crash_clear_kernpt()
225 p4d_t *p4d; in hv_crash_fixup_kernpt() local
228 p4d = p4d_offset(pgd, trampoline_pa); in hv_crash_fixup_kernpt()
231 p4d_populate(&init_mm, p4d, (pud_t *)hv_crash_ptpgs[1]); in hv_crash_fixup_kernpt()
232 p4d->p4d = p4d->p4d & ~(_PAGE_NX); /* enable execute */ in hv_crash_fixup_kernpt()
506 p4d_t *p4d; in hv_crash_build_tramp_pt() local
512 p4d = hv_crash_ptpgs[0] + pgd_index(addr) * sizeof(p4d); in hv_crash_build_tramp_pt()
514 set_p4d(p4d, __p4d(_PAGE_TABLE | pa)); in hv_crash_build_tramp_pt()
[all …]
/linux/arch/parisc/mm/
H A Dhugetlbpage.c30 p4d_t *p4d; in huge_pte_alloc() local
43 p4d = p4d_offset(pgd, addr); in huge_pte_alloc()
44 pud = pud_alloc(mm, p4d, addr); in huge_pte_alloc()
57 p4d_t *p4d; in huge_pte_offset() local
66 p4d = p4d_offset(pgd, addr); in huge_pte_offset()
67 if (!p4d_none(*p4d)) { in huge_pte_offset()
68 pud = pud_offset(p4d, addr); in huge_pte_offset()
/linux/arch/x86/power/
H A Dhibernate_64.c32 p4d_t *p4d = NULL; in set_up_temporary_text_mapping() local
55 p4d = (p4d_t *)get_safe_page(GFP_ATOMIC); in set_up_temporary_text_mapping()
56 if (!p4d) in set_up_temporary_text_mapping()
72 if (p4d) { in set_up_temporary_text_mapping()
74 pgd_t new_pgd = __pgd(__pa(p4d) | pgprot_val(pgtable_prot)); in set_up_temporary_text_mapping()
76 set_p4d(p4d + p4d_index(restore_jump_address), new_p4d); in set_up_temporary_text_mapping()
H A Dhibernate.c157 p4d_t *p4d; in relocate_restore_code() local
171 p4d = p4d_offset(pgd, relocated_restore_code); in relocate_restore_code()
172 if (p4d_leaf(*p4d)) { in relocate_restore_code()
173 set_p4d(p4d, __p4d(p4d_val(*p4d) & ~_PAGE_NX)); in relocate_restore_code()
176 pud = pud_offset(p4d, relocated_restore_code); in relocate_restore_code()
/linux/arch/um/kernel/
H A Dtlb.c115 static inline int update_pud_range(p4d_t *p4d, unsigned long addr, in update_pud_range() argument
123 pud = pud_offset(p4d, addr); in update_pud_range()
142 p4d_t *p4d; in update_p4d_range() local
146 p4d = p4d_offset(pgd, addr); in update_p4d_range()
149 if (!p4d_present(*p4d)) { in update_p4d_range()
150 if (p4d_needsync(*p4d)) { in update_p4d_range()
153 p4d_mkuptodate(*p4d); in update_p4d_range()
156 ret = update_pud_range(p4d, addr, next, ops); in update_p4d_range()
157 } while (p4d++, addr = next, ((addr < end) && !ret)); in update_p4d_range()
/linux/include/linux/
H A Dpgalloc.h15 #define pgd_populate_kernel(addr, pgd, p4d) \ argument
17 pgd_populate(&init_mm, pgd, p4d); \
22 #define p4d_populate_kernel(addr, p4d, pud) \ argument
24 p4d_populate(&init_mm, p4d, pud); \
/linux/mm/
H A Dsparse-vmemmap.c211 pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node) in vmemmap_pud_populate() argument
213 pud_t *pud = pud_offset(p4d, addr); in vmemmap_pud_populate()
226 p4d_t *p4d = p4d_offset(pgd, addr); in vmemmap_p4d_populate() local
227 if (p4d_none(*p4d)) { in vmemmap_p4d_populate()
232 p4d_populate_kernel(addr, p4d, p); in vmemmap_p4d_populate()
234 return p4d; in vmemmap_p4d_populate()
255 p4d_t *p4d; in vmemmap_populate_address() local
263 p4d = vmemmap_p4d_populate(pgd, addr, node); in vmemmap_populate_address()
264 if (!p4d) in vmemmap_populate_address()
266 pud = vmemmap_pud_populate(p4d, addr, node); in vmemmap_populate_address()
[all …]
/linux/arch/sh/mm/
H A Dfault.c56 p4d_t *p4d; in show_pte() local
69 p4d = p4d_offset(pgd, addr); in show_pte()
71 pr_cont(", *p4d=%0*Lx", (u32)(sizeof(*p4d) * 2), in show_pte()
72 (u64)p4d_val(*p4d)); in show_pte()
74 if (p4d_none(*p4d)) in show_pte()
77 if (p4d_bad(*p4d)) { in show_pte()
82 pud = pud_offset(p4d, addr); in show_pte()
124 p4d_t *p4d, *p4d_k; in vmalloc_sync_one() local
134 p4d = p4d_offset(pgd, address); in vmalloc_sync_one()
139 pud = pud_offset(p4d, address); in vmalloc_sync_one()
/linux/arch/s390/include/asm/
H A Dpgalloc.h68 static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d) in p4d_free() argument
73 pagetable_dtor(virt_to_ptdesc(p4d)); in p4d_free()
74 crst_table_free(mm, (unsigned long *) p4d); in p4d_free()
122 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) in pgd_populate() argument
124 set_pgd(pgd, __pgd(_REGION1_ENTRY | __pa(p4d))); in pgd_populate()
127 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) in p4d_populate() argument
129 set_p4d(p4d, __p4d(_REGION2_ENTRY | __pa(pud))); in p4d_populate()
/linux/arch/s390/boot/
H A Dvmem.c148 static bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr, in kasan_p4d_populate_zero_shadow() argument
153 p4d_populate(&init_mm, p4d, kasan_early_shadow_pud); in kasan_p4d_populate_zero_shadow()
201 static inline bool kasan_p4d_populate_zero_shadow(p4d_t *p4d, unsigned long addr, in kasan_p4d_populate_zero_shadow() argument
386 static void pgtable_pud_populate(p4d_t *p4d, unsigned long addr, unsigned long end, in pgtable_pud_populate() argument
393 pud = pud_offset(p4d, addr); in pgtable_pud_populate()
422 p4d_t *p4d; in pgtable_p4d_populate() local
425 p4d = p4d_offset(pgd, addr); in pgtable_p4d_populate()
426 for (; addr < end; addr = next, p4d++) { in pgtable_p4d_populate()
428 if (p4d_none(*p4d)) { in pgtable_p4d_populate()
429 if (kasan_p4d_populate_zero_shadow(p4d, addr, next, mode)) in pgtable_p4d_populate()
[all …]
/linux/arch/loongarch/include/asm/
H A Dpgtable.h153 static inline int p4d_none(p4d_t p4d) in p4d_none() argument
155 return p4d_val(p4d) == (unsigned long)invalid_pud_table; in p4d_none()
158 static inline int p4d_bad(p4d_t p4d) in p4d_bad() argument
160 return p4d_val(p4d) & ~PAGE_MASK; in p4d_bad()
163 static inline int p4d_present(p4d_t p4d) in p4d_present() argument
165 return p4d_val(p4d) != (unsigned long)invalid_pud_table; in p4d_present()
168 static inline pud_t *p4d_pgtable(p4d_t p4d) in p4d_pgtable() argument
170 return (pud_t *)p4d_val(p4d); in p4d_pgtable()
173 static inline void set_p4d(p4d_t *p4d, p4d_t p4dval) in set_p4d() argument
175 WRITE_ONCE(*p4d, p4dval); in set_p4d()
[all …]
/linux/arch/arc/include/asm/
H A Dpgtable-levels.h111 #define p4d_pgtable(p4d) ((pud_t *)(p4d_val(p4d) & PAGE_MASK)) argument
112 #define p4d_page(p4d) virt_to_page(p4d_pgtable(p4d)) argument
113 #define set_p4d(p4dp, p4d) (*(p4dp) = p4d) argument

123456