| /linux/arch/arm64/mm/ |
| H A D | mmu.c | 363 static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, in alloc_init_pud() argument
370 p4d_t p4d = READ_ONCE(*p4dp); in alloc_init_pud()
386 __p4d_populate(p4dp, pud_phys, p4dval); in alloc_init_pud()
389 pudp = pud_set_fixmap_offset(p4dp, addr); in alloc_init_pud()
438 p4d_t *p4dp; in alloc_init_p4d() local
450 p4dp = p4d_set_fixmap(p4d_phys); in alloc_init_p4d()
451 init_clear_pgtable(p4dp); in alloc_init_p4d()
452 p4dp += p4d_index(addr); in alloc_init_p4d()
456 p4dp = p4d_set_fixmap_offset(pgdp, addr); in alloc_init_p4d()
460 p4d_t old_p4d = READ_ONCE(*p4dp); in alloc_init_p4d()
[all …]
|
| H A D | hugetlbpage.c | 82 p4d_t *p4dp; in find_num_contig() local
87 p4dp = p4d_offset(pgdp, addr); in find_num_contig()
88 pudp = pud_offset(p4dp, addr); in find_num_contig()
241 p4d_t *p4dp; in huge_pte_alloc() local
247 p4dp = p4d_alloc(mm, pgdp, addr); in huge_pte_alloc()
248 if (!p4dp) in huge_pte_alloc()
251 pudp = pud_alloc(mm, p4dp, addr); in huge_pte_alloc()
282 p4d_t *p4dp; in huge_pte_offset() local
290 p4dp = p4d_offset(pgdp, addr); in huge_pte_offset()
291 if (!p4d_present(READ_ONCE(*p4dp))) in huge_pte_offset()
[all …]
|
| H A D | fault.c | 162 p4d_t *p4dp, p4d; in show_pte() local
170 p4dp = p4d_offset(pgdp, addr); in show_pte()
171 p4d = READ_ONCE(*p4dp); in show_pte()
176 pudp = pud_offset(p4dp, addr); in show_pte()
|
| /linux/arch/x86/include/asm/ |
| H A D | pgtable_64.h | 138 static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d) in native_set_p4d() argument
144 WRITE_ONCE(*p4dp, p4d); in native_set_p4d()
149 pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd); in native_set_p4d()
150 WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd))); in native_set_p4d()
|
| /linux/arch/sparc/mm/ |
| H A D | srmmu.c | 490 p4d_t *p4dp; in srmmu_mapioaddr() local
498 p4dp = p4d_offset(pgdp, virt_addr); in srmmu_mapioaddr()
499 pudp = pud_offset(p4dp, virt_addr); in srmmu_mapioaddr()
529 p4d_t *p4dp; in srmmu_unmapioaddr() local
536 p4dp = p4d_offset(pgdp, virt_addr); in srmmu_unmapioaddr()
537 pudp = pud_offset(p4dp, virt_addr); in srmmu_unmapioaddr()
647 p4d_t *p4dp; in srmmu_early_allocate_ptable_skeleton() local
654 p4dp = p4d_offset(pgdp, start); in srmmu_early_allocate_ptable_skeleton()
655 pudp = pud_offset(p4dp, start); in srmmu_early_allocate_ptable_skeleton()
682 p4d_t *p4dp; in srmmu_allocate_ptable_skeleton() local
[all …]
|
| H A D | fault_64.c | 82 p4d_t *p4dp; in get_user_insn() local
91 p4dp = p4d_offset(pgdp, tpc); in get_user_insn()
92 if (p4d_none(*p4dp) || unlikely(p4d_bad(*p4dp))) in get_user_insn()
94 pudp = pud_offset(p4dp, tpc); in get_user_insn()
|
| /linux/arch/powerpc/include/asm/nohash/64/ |
| H A D | pgtable-4k.h | 67 static inline void p4d_clear(p4d_t *p4dp) in p4d_clear() argument
69 *p4dp = __p4d(0); in p4d_clear()
|
| /linux/arch/arm64/include/asm/ |
| H A D | tlb.h | 108 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4dp, in __p4d_free_tlb() argument
111 struct ptdesc *ptdesc = virt_to_ptdesc(p4dp); in __p4d_free_tlb()
|
| /linux/arch/riscv/mm/ |
| H A D | fault.c | 31 p4d_t *p4dp, p4d; in show_pte() local
50 p4dp = p4d_offset(pgdp, addr); in show_pte()
51 p4d = p4dp_get(p4dp); in show_pte()
56 pudp = pud_offset(p4dp, addr); in show_pte()
|
| H A D | init.c | 628 static void __meminit create_p4d_mapping(p4d_t *p4dp, uintptr_t va, phys_addr_t pa, phys_addr_t sz, in create_p4d_mapping() argument
636 if (p4d_val(p4dp[p4d_index]) == 0) in create_p4d_mapping()
637 p4dp[p4d_index] = pfn_p4d(PFN_DOWN(pa), prot); in create_p4d_mapping()
641 if (p4d_val(p4dp[p4d_index]) == 0) { in create_p4d_mapping()
643 p4dp[p4d_index] = pfn_p4d(PFN_DOWN(next_phys), PAGE_TABLE); in create_p4d_mapping()
647 next_phys = PFN_PHYS(_p4d_pfn(p4dp[p4d_index])); in create_p4d_mapping()
1653 p4d_t *p4dp, p4d; in remove_p4d_mapping() local
1658 p4dp = pgtable_l5_enabled ? p4d_base + p4d_index(addr) : p4d_base; in remove_p4d_mapping()
1659 p4d = p4dp_get(p4dp); in remove_p4d_mapping()
1665 p4d_clear(p4dp); in remove_p4d_mapping()
[all …]
|
| /linux/arch/x86/boot/compressed/ |
| H A D | ident_map_64.c | 260 p4d_t *p4dp; in set_clr_page_flags() local
283 p4dp = p4d_offset(pgdp, address); in set_clr_page_flags()
284 pudp = pud_offset(p4dp, address); in set_clr_page_flags()
|
| /linux/arch/arc/include/asm/ |
| H A D | pgtable-levels.h | 113 #define set_p4d(p4dp, p4d) (*(p4dp) = p4d) argument
|
| /linux/arch/riscv/include/asm/ |
| H A D | pgtable-64.h | 276 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) in set_p4d() argument
279 WRITE_ONCE(*p4dp, p4d); in set_p4d()
281 set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) }); in set_p4d()
|
| /linux/mm/ |
| H A D | pagewalk.c | 913 p4d_t *p4dp; in folio_walk_start()
925 p4dp = p4d_offset(pgdp, addr); in folio_walk_start()
926 if (p4d_none_or_clear_bad(p4dp)) in folio_walk_start()
929 pudp = pud_offset(p4dp, addr); in folio_walk_start()
915 p4d_t *p4dp; folio_walk_start() local
|
| H A D | memory.c | 525 p4d_t p4d, *p4dp; in __print_bad_page_map_pgtable() local
542 p4dp = p4d_offset(pgdp, addr); in __print_bad_page_map_pgtable()
543 p4d = p4dp_get(p4dp); in __print_bad_page_map_pgtable()
551 pudp = pud_offset(p4dp, addr); in __print_bad_page_map_pgtable()
6984 p4d_t *p4dp, p4d; in generic_access_phys()
7001 p4dp = p4d_offset(pgdp, address); in generic_access_phys()
7002 p4d = p4dp_get(p4dp); in generic_access_phys()
7006 pudp = pud_offset(p4dp, address); in generic_access_phys()
6856 p4d_t *p4dp, p4d; follow_pfnmap_start() local
|
| /linux/arch/mips/mm/ |
| H A D | tlb-r4k.c | 301 p4d_t *p4dp; in __update_tlb() local
327 p4dp = p4d_offset(pgdp, address); in __update_tlb()
328 pudp = pud_offset(p4dp, address); in __update_tlb()
|
| /linux/arch/sparc/kernel/ |
| H A D | signal32.c | 300 p4d_t *p4dp; in flush_signal_insns() local
320 p4dp = p4d_offset(pgdp, address); in flush_signal_insns()
321 if (p4d_none(*p4dp)) in flush_signal_insns()
323 pudp = pud_offset(p4dp, address); in flush_signal_insns()
|
| /linux/arch/loongarch/include/asm/ |
| H A D | pgtable.h | 189 static inline void p4d_clear(p4d_t *p4dp) in p4d_clear() argument
191 set_p4d(p4dp, __p4d((unsigned long)invalid_pud_table)); in p4d_clear()
|
| /linux/arch/powerpc/kvm/ |
| H A D | book3s_64_mmu_radix.c | 1291 p4d_t p4d, *p4dp; in debugfs_radix_read() local
1364 p4dp = p4d_offset(pgdp, gpa); in debugfs_radix_read()
1365 p4d = READ_ONCE(*p4dp); in debugfs_radix_read()
|
| /linux/kernel/events/ |
| H A D | core.c | 8517 p4d_t *p4dp, p4d; in perf_get_pgtable_size() local
8530 p4dp = p4d_offset_lockless(pgdp, pgd, addr); in perf_get_pgtable_size()
8531 p4d = p4dp_get(p4dp); in perf_get_pgtable_size()
8538 pudp = pud_offset_lockless(p4dp, p4d, addr); in perf_get_pgtable_size()
|