Home
last modified time | relevance | path

Searched refs:p4dp (Results 1 – 25 of 40) sorted by relevance

12

/linux/arch/riscv/mm/
H A Dkasan_init.c115 p4d_t *p4dp, *p; in kasan_populate_p4d() local
123 p4dp = p4d_offset(pgd, vaddr); in kasan_populate_p4d()
128 if (p4d_none(p4dp_get(p4dp)) && IS_ALIGNED(vaddr, P4D_SIZE) && in kasan_populate_p4d()
132 set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_KERNEL)); in kasan_populate_p4d()
138 kasan_populate_pud(p4dp, vaddr, next); in kasan_populate_p4d()
139 } while (p4dp++, vaddr = next, vaddr != end); in kasan_populate_p4d()
165 static void __init kasan_early_clear_pud(p4d_t *p4dp, in kasan_early_clear_pud() argument
172 pudp = (pud_t *)p4dp; in kasan_early_clear_pud()
174 base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(p4dp_get(p4dp)))); in kasan_early_clear_pud()
193 p4d_t *p4dp, *base_p4d; in kasan_early_clear_p4d() local
[all …]
H A Dfault.c31 p4d_t *p4dp, p4d; in show_pte() local
50 p4dp = p4d_offset(pgdp, addr); in show_pte()
51 p4d = p4dp_get(p4dp); in show_pte()
56 pudp = pud_offset(p4dp, addr); in show_pte()
H A Dinit.c665 static void __meminit create_p4d_mapping(p4d_t *p4dp, uintptr_t va, phys_addr_t pa, phys_addr_t sz, in create_p4d_mapping() argument
673 if (p4d_val(p4dp[p4d_index]) == 0) in create_p4d_mapping()
674 p4dp[p4d_index] = pfn_p4d(PFN_DOWN(pa), prot); in create_p4d_mapping()
678 if (p4d_val(p4dp[p4d_index]) == 0) { in create_p4d_mapping()
680 p4dp[p4d_index] = pfn_p4d(PFN_DOWN(next_phys), PAGE_TABLE); in create_p4d_mapping()
684 next_phys = PFN_PHYS(_p4d_pfn(p4dp[p4d_index])); in create_p4d_mapping()
1742 p4d_t *p4dp, p4d; in remove_p4d_mapping() local
1747 p4dp = p4d_base + p4d_index(addr); in remove_p4d_mapping()
1748 p4d = p4dp_get(p4dp); in remove_p4d_mapping()
1754 p4d_clear(p4dp); in remove_p4d_mapping()
[all …]
/linux/arch/arm64/mm/
H A Dkasan_init.c89 static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node, in kasan_pud_offset() argument
92 if (p4d_none(READ_ONCE(*p4dp))) { in kasan_pud_offset()
96 __p4d_populate(p4dp, pud_phys, P4D_TYPE_TABLE); in kasan_pud_offset()
99 return early ? pud_offset_kimg(p4dp, addr) : pud_offset(p4dp, addr); in kasan_pud_offset()
144 static void __init kasan_pud_populate(p4d_t *p4dp, unsigned long addr, in kasan_pud_populate() argument
148 pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early); in kasan_pud_populate()
160 p4d_t *p4dp = kasan_p4d_offset(pgdp, addr, node, early); in kasan_p4d_populate() local
164 kasan_pud_populate(p4dp, addr, next, node, early); in kasan_p4d_populate()
165 } while (p4dp++, addr = next, addr != end && p4d_none(READ_ONCE(*p4dp))); in kasan_p4d_populate()
H A Dhugetlbpage.c86 p4d_t *p4dp; in find_num_contig() local
91 p4dp = p4d_offset(pgdp, addr); in find_num_contig()
92 pudp = pud_offset(p4dp, addr); in find_num_contig()
244 p4d_t *p4dp; in huge_pte_alloc() local
250 p4dp = p4d_alloc(mm, pgdp, addr); in huge_pte_alloc()
251 if (!p4dp) in huge_pte_alloc()
254 pudp = pud_alloc(mm, p4dp, addr); in huge_pte_alloc()
285 p4d_t *p4dp; in huge_pte_offset() local
293 p4dp = p4d_offset(pgdp, addr); in huge_pte_offset()
294 if (!p4d_present(READ_ONCE(*p4dp))) in huge_pte_offset()
[all …]
/linux/arch/loongarch/mm/
H A Dkasan_init.c142 static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node, bool early) in kasan_pud_offset() argument
144 if (__p4d_none(early, p4dp_get(p4dp))) { in kasan_pud_offset()
149 p4d_populate(&init_mm, p4dp, (pud_t *)__va(pud_phys)); in kasan_pud_offset()
152 return pud_offset(p4dp, addr); in kasan_pud_offset()
195 static void __init kasan_pud_populate(p4d_t *p4dp, unsigned long addr, in kasan_pud_populate() argument
199 pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early); in kasan_pud_populate()
211 p4d_t *p4dp = kasan_p4d_offset(pgdp, addr, node, early); in kasan_p4d_populate() local
215 kasan_pud_populate(p4dp, addr, next, node, early); in kasan_p4d_populate()
216 } while (p4dp++, addr = next, addr != end && __p4d_none(early, READ_ONCE(*p4dp))); in kasan_p4d_populate()
/linux/arch/powerpc/mm/kasan/
H A Dinit_book3e_64.c35 p4d_t *p4dp; in kasan_map_kernel_page() local
41 p4dp = p4d_offset(pgdp, ea); in kasan_map_kernel_page()
42 if (kasan_pud_table(*p4dp)) { in kasan_map_kernel_page()
45 p4d_populate(&init_mm, p4dp, pudp); in kasan_map_kernel_page()
47 pudp = pud_offset(p4dp, ea); in kasan_map_kernel_page()
/linux/arch/arm64/include/asm/
H A Dpgtable.h1005 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) in set_p4d() argument
1007 if (in_swapper_pgdir(p4dp)) { in set_p4d()
1008 set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d))); in set_p4d()
1012 WRITE_ONCE(*p4dp, p4d); in set_p4d()
1016 static inline void p4d_clear(p4d_t *p4dp) in p4d_clear() argument
1019 set_p4d(p4dp, __p4d(0)); in p4d_clear()
1029 static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr) in p4d_to_folded_pud() argument
1032 VM_BUG_ON(((addr >> P4D_SHIFT) ^ ((u64)p4dp >> 3)) % PTRS_PER_P4D); in p4d_to_folded_pud()
1034 return (pud_t *)PTR_ALIGN_DOWN(p4dp, PAGE_SIZE) + pud_index(addr); in p4d_to_folded_pud()
1042 static inline phys_addr_t pud_offset_phys(p4d_t *p4dp, unsigned long addr) in pud_offset_phys() argument
[all …]
/linux/arch/powerpc/mm/book3s64/
H A Dradix_pgtable.c79 p4d_t *p4dp; in early_map_kernel_page() local
85 p4dp = p4d_offset(pgdp, ea); in early_map_kernel_page()
86 if (p4d_none(*p4dp)) { in early_map_kernel_page()
89 p4d_populate(&init_mm, p4dp, pudp); in early_map_kernel_page()
91 pudp = pud_offset(p4dp, ea); in early_map_kernel_page()
131 p4d_t *p4dp; in __map_kernel_page() local
154 p4dp = p4d_offset(pgdp, ea); in __map_kernel_page()
155 pudp = pud_alloc(&init_mm, p4dp, ea); in __map_kernel_page()
192 p4d_t *p4dp; in radix__change_memory_range() local
205 p4dp = p4d_offset(pgdp, idx); in radix__change_memory_range()
[all …]
/linux/arch/sparc/mm/
H A Dsrmmu.c490 p4d_t *p4dp; in srmmu_mapioaddr() local
498 p4dp = p4d_offset(pgdp, virt_addr); in srmmu_mapioaddr()
499 pudp = pud_offset(p4dp, virt_addr); in srmmu_mapioaddr()
529 p4d_t *p4dp; in srmmu_unmapioaddr() local
536 p4dp = p4d_offset(pgdp, virt_addr); in srmmu_unmapioaddr()
537 pudp = pud_offset(p4dp, virt_addr); in srmmu_unmapioaddr()
676 p4d_t *p4dp; in srmmu_early_allocate_ptable_skeleton() local
683 p4dp = p4d_offset(pgdp, start); in srmmu_early_allocate_ptable_skeleton()
684 pudp = pud_offset(p4dp, start); in srmmu_early_allocate_ptable_skeleton()
711 p4d_t *p4dp; in srmmu_allocate_ptable_skeleton() local
[all …]
H A Dfault_64.c82 p4d_t *p4dp; in get_user_insn() local
91 p4dp = p4d_offset(pgdp, tpc); in get_user_insn()
92 if (p4d_none(*p4dp) || unlikely(p4d_bad(*p4dp))) in get_user_insn()
94 pudp = pud_offset(p4dp, tpc); in get_user_insn()
/linux/arch/powerpc/include/asm/nohash/64/
H A Dpgtable-4k.h67 static inline void p4d_clear(p4d_t *p4dp) in p4d_clear() argument
69 *p4dp = __p4d(0); in p4d_clear()
H A Dpgtable.h143 static inline void p4d_set(p4d_t *p4dp, unsigned long val) in p4d_set() argument
145 *p4dp = __p4d(val); in p4d_set()
/linux/include/trace/events/
H A Dxen.h224 TP_PROTO(p4d_t *p4dp, p4d_t *user_p4dp, p4d_t p4dval),
225 TP_ARGS(p4dp, user_p4dp, p4dval),
227 __field(p4d_t *, p4dp)
231 TP_fast_assign(__entry->p4dp = p4dp;
235 __entry->p4dp, __entry->user_p4dp,
/linux/mm/
H A Ddebug_vm_pgtable.c50 p4d_t *p4dp; member
536 p4d_t p4d = p4dp_get(args->p4dp); in p4d_clear_tests()
543 p4d_clear(args->p4dp); in p4d_clear_tests()
544 p4d = p4dp_get(args->p4dp); in p4d_clear_tests()
561 p4d_clear(args->p4dp); in p4d_populate_tests()
562 p4d_populate(args->mm, args->p4dp, args->start_pudp); in p4d_populate_tests()
563 p4d = p4dp_get(args->p4dp); in p4d_populate_tests()
593 p4d_clear(args->p4dp); in pgd_populate_tests()
1030 p4d_clear(args->p4dp); in destroy_args()
1194 args->p4dp = p4d_alloc(args->mm, args->pgdp, args->vaddr); in init_args()
[all …]
H A Dgup.c943 unsigned long address, p4d_t *p4dp, in follow_pud_mask() argument
952 pudp = pud_offset(p4dp, address); in follow_pud_mask()
975 p4d_t *p4dp, p4d; in follow_p4d_mask() local
977 p4dp = p4d_offset(pgdp, address); in follow_p4d_mask()
978 p4d = p4dp_get(p4dp); in follow_p4d_mask()
984 return follow_pud_mask(vma, address, p4dp, flags, page_mask); in follow_p4d_mask()
3054 static int gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, in gup_fast_pud_range() argument
3061 pudp = pud_offset_lockless(p4dp, p4d, addr); in gup_fast_pud_range()
3085 p4d_t *p4dp; in gup_fast_p4d_range() local
3087 p4dp = p4d_offset_lockless(pgdp, pgd, addr); in gup_fast_p4d_range()
[all …]
/linux/arch/arm/mm/
H A Dkasan_init.c128 p4d_t *p4dp; in kasan_pgd_populate() local
155 p4dp = p4d_offset(pgdp, addr); in kasan_pgd_populate()
156 pudp = pud_offset(p4dp, addr); in kasan_pgd_populate()
/linux/arch/s390/mm/
H A Dpageattr.c296 p4d_t *p4dp; in walk_p4d_level() local
299 p4dp = p4d_offset(pgd, addr); in walk_p4d_level()
301 if (p4d_none(*p4dp)) in walk_p4d_level()
304 rc = walk_pud_level(p4dp, addr, next, flags); in walk_p4d_level()
305 p4dp++; in walk_p4d_level()
/linux/arch/x86/boot/compressed/
H A Dident_map_64.c260 p4d_t *p4dp; in set_clr_page_flags() local
283 p4dp = p4d_offset(pgdp, address); in set_clr_page_flags()
284 pudp = pud_offset(p4dp, address); in set_clr_page_flags()
/linux/arch/powerpc/mm/
H A Dpgtable.c433 p4d_t p4d, *p4dp; in __find_linux_pte() local
458 p4dp = p4d_offset(pgdp, ea); in __find_linux_pte()
459 p4d = READ_ONCE(*p4dp); in __find_linux_pte()
466 ret_pte = (pte_t *)p4dp; in __find_linux_pte()
/linux/arch/arc/include/asm/
H A Dpgtable-levels.h113 #define set_p4d(p4dp, p4d) (*(p4dp) = p4d) argument
/linux/arch/riscv/include/asm/
H A Dpgtable-64.h276 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) in set_p4d() argument
279 WRITE_ONCE(*p4dp, p4d); in set_p4d()
281 set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) }); in set_p4d()
/linux/arch/x86/include/asm/
H A Dpgtable.h81 # define set_p4d(p4dp, p4d) native_set_p4d(p4dp, p4d) argument
1444 static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp) in kernel_to_user_p4dp() argument
1446 return ptr_set_bit(p4dp, PTI_PGTABLE_SWITCH_BIT); in kernel_to_user_p4dp()
1449 static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp) in user_to_kernel_p4dp() argument
1451 return ptr_clear_bit(p4dp, PTI_PGTABLE_SWITCH_BIT); in user_to_kernel_p4dp()
1732 #define set_p4d_safe(p4dp, p4d) \ argument
1734 WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \
1735 set_p4d(p4dp, p4d); \
/linux/arch/powerpc/lib/
H A Dcode-patching.c251 p4d_t *p4dp; in unmap_patch_area() local
258 p4dp = p4d_offset(pgdp, addr); in unmap_patch_area()
259 if (WARN_ON(p4d_none(*p4dp))) in unmap_patch_area()
262 pudp = pud_offset(p4dp, addr); in unmap_patch_area()
/linux/arch/mips/mm/
H A Dtlb-r4k.c299 p4d_t *p4dp; in __update_tlb() local
325 p4dp = p4d_offset(pgdp, address); in __update_tlb()
326 pudp = pud_offset(p4dp, address); in __update_tlb()

12