| /linux/arch/arm64/mm/ |
| H A D | mmu.c | 363 static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end, in alloc_init_pud() argument
370 p4d_t p4d = READ_ONCE(*p4dp); in alloc_init_pud()
386 __p4d_populate(p4dp, pud_phys, p4dval); in alloc_init_pud()
389 pudp = pud_set_fixmap_offset(p4dp, addr); in alloc_init_pud()
438 p4d_t *p4dp; in alloc_init_p4d() local
450 p4dp = p4d_set_fixmap(p4d_phys); in alloc_init_p4d()
451 init_clear_pgtable(p4dp); in alloc_init_p4d()
452 p4dp += p4d_index(addr); in alloc_init_p4d()
456 p4dp = p4d_set_fixmap_offset(pgdp, addr); in alloc_init_p4d()
460 p4d_t old_p4d = READ_ONCE(*p4dp); in alloc_init_p4d()
691 p4d_t *p4dp, p4d; split_kernel_leaf_mapping_locked() local
1584 unmap_hotplug_pud_range(p4d_t * p4dp,unsigned long addr,unsigned long end,bool free_mapped,struct vmem_altmap * altmap) unmap_hotplug_pud_range() argument
1620 p4d_t *p4dp, p4d; unmap_hotplug_p4d_range() local
1741 free_empty_pud_table(p4d_t * p4dp,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling) free_empty_pud_table() argument
1785 p4d_t *p4dp, p4d; free_empty_p4d_table() local
1894 p4d_clear_huge(p4d_t * p4dp) p4d_clear_huge() argument
2056 p4d_t *p4dp, p4d; addr_splits_kernel_leaf() local
[all...] |
| H A D | hugetlbpage.c | 82 p4d_t *p4dp; in find_num_contig() local
87 p4dp = p4d_offset(pgdp, addr); in find_num_contig()
88 pudp = pud_offset(p4dp, addr); in find_num_contig()
241 p4d_t *p4dp; in huge_pte_alloc() local
247 p4dp = p4d_alloc(mm, pgdp, addr); in huge_pte_alloc()
248 if (!p4dp) in huge_pte_alloc()
251 pudp = pud_alloc(mm, p4dp, addr); in huge_pte_alloc()
282 p4d_t *p4dp; in huge_pte_offset() local
290 p4dp = p4d_offset(pgdp, addr); in huge_pte_offset()
291 if (!p4d_present(READ_ONCE(*p4dp))) in huge_pte_offset()
[all …]
|
| H A D | fault.c | 166 p4d_t *p4dp, p4d; in show_pte() local
174 p4dp = p4d_offset_lockless(pgdp, pgd, addr); in show_pte()
175 p4d = READ_ONCE(*p4dp); in show_pte()
180 pudp = pud_offset_lockless(p4dp, p4d, addr); in show_pte()
|
| /linux/arch/s390/mm/ |
| H A D | gmap_helpers.c | 47 p4d_t *p4dp, p4d; in try_get_locked_pte() local
55 p4dp = p4d_offset_lockless(pgdp, pgd, vmaddr); in try_get_locked_pte()
56 p4d = p4dp_get(p4dp); in try_get_locked_pte()
59 pudp = pud_offset_lockless(p4dp, p4d, vmaddr); in try_get_locked_pte()
|
| /linux/arch/x86/include/asm/ |
| H A D | pgtable_64.h | 138 static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d) in native_set_p4d() argument
144 WRITE_ONCE(*p4dp, p4d); in native_set_p4d()
149 pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd); in native_set_p4d()
150 WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd))); in native_set_p4d()
|
| H A D | pgtable.h | 73 # define set_p4d(p4dp, p4d) native_set_p4d(p4dp, p4d) argument
1445 static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp) in kernel_to_user_p4dp() argument
1447 return ptr_set_bit(p4dp, PTI_PGTABLE_SWITCH_BIT); in kernel_to_user_p4dp()
1450 static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp) in user_to_kernel_p4dp() argument
1452 return ptr_clear_bit(p4dp, PTI_PGTABLE_SWITCH_BIT); in user_to_kernel_p4dp()
1733 #define set_p4d_safe(p4dp, p4d) \ argument
1735 WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \
1736 set_p4d(p4dp, p4d); \
|
| /linux/arch/sparc/mm/ |
| H A D | srmmu.c | 490 p4d_t *p4dp; in srmmu_mapioaddr() local
498 p4dp = p4d_offset(pgdp, virt_addr); in srmmu_mapioaddr()
499 pudp = pud_offset(p4dp, virt_addr); in srmmu_mapioaddr()
529 p4d_t *p4dp; in srmmu_unmapioaddr() local
536 p4dp = p4d_offset(pgdp, virt_addr); in srmmu_unmapioaddr()
537 pudp = pud_offset(p4dp, virt_addr); in srmmu_unmapioaddr()
647 p4d_t *p4dp; in srmmu_early_allocate_ptable_skeleton() local
654 p4dp = p4d_offset(pgdp, start); in srmmu_early_allocate_ptable_skeleton()
655 pudp = pud_offset(p4dp, start); in srmmu_early_allocate_ptable_skeleton()
682 p4d_t *p4dp; in srmmu_allocate_ptable_skeleton() local
[all …]
|
| H A D | fault_64.c | 82 p4d_t *p4dp; in get_user_insn() local
91 p4dp = p4d_offset(pgdp, tpc); in get_user_insn()
92 if (p4d_none(*p4dp) || unlikely(p4d_bad(*p4dp))) in get_user_insn()
94 pudp = pud_offset(p4dp, tpc); in get_user_insn()
|
| /linux/arch/powerpc/include/asm/nohash/64/ |
| H A D | pgtable-4k.h | 67 static inline void p4d_clear(p4d_t *p4dp) in p4d_clear() argument
69 *p4dp = __p4d(0); in p4d_clear()
|
| /linux/arch/arm64/include/asm/ |
| H A D | tlb.h | 108 static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4dp, in __p4d_free_tlb() argument
111 struct ptdesc *ptdesc = virt_to_ptdesc(p4dp); in __p4d_free_tlb()
|
| /linux/arch/riscv/mm/ |
| H A D | fault.c | 31 p4d_t *p4dp, p4d; in show_pte() local
50 p4dp = p4d_offset(pgdp, addr); in show_pte()
51 p4d = p4dp_get(p4dp); in show_pte()
56 pudp = pud_offset(p4dp, addr); in show_pte()
|
| H A D | init.c | 628 static void __meminit create_p4d_mapping(p4d_t *p4dp, uintptr_t va, phys_addr_t pa, phys_addr_t sz, in create_p4d_mapping() argument
636 if (p4d_val(p4dp[p4d_index]) == 0) in create_p4d_mapping()
637 p4dp[p4d_index] = pfn_p4d(PFN_DOWN(pa), prot); in create_p4d_mapping()
641 if (p4d_val(p4dp[p4d_index]) == 0) { in create_p4d_mapping()
643 p4dp[p4d_index] = pfn_p4d(PFN_DOWN(next_phys), PAGE_TABLE); in create_p4d_mapping()
647 next_phys = PFN_PHYS(_p4d_pfn(p4dp[p4d_index])); in create_p4d_mapping()
1655 p4d_t *p4dp, p4d; in remove_p4d_mapping() local
1660 p4dp = pgtable_l5_enabled ? p4d_base + p4d_index(addr) : p4d_base; in remove_p4d_mapping()
1661 p4d = p4dp_get(p4dp); in remove_p4d_mapping()
1667 p4d_clear(p4dp); in remove_p4d_mapping()
[all …]
|
| /linux/arch/x86/boot/compressed/ |
| H A D | ident_map_64.c | 260 p4d_t *p4dp; in set_clr_page_flags() local
283 p4dp = p4d_offset(pgdp, address); in set_clr_page_flags()
284 pudp = pud_offset(p4dp, address); in set_clr_page_flags()
|
| /linux/arch/arc/include/asm/ |
| H A D | pgtable-levels.h | 113 #define set_p4d(p4dp, p4d) (*(p4dp) = p4d) argument
|
| /linux/arch/riscv/include/asm/ |
| H A D | pgtable-64.h | 276 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) in set_p4d() argument
279 WRITE_ONCE(*p4dp, p4d); in set_p4d()
281 set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) }); in set_p4d()
|
| /linux/mm/ |
| H A D | gup.c | 943 unsigned long address, p4d_t *p4dp, in follow_pud_mask() argument
952 pudp = pud_offset(p4dp, address); in follow_pud_mask()
975 p4d_t *p4dp, p4d; in follow_p4d_mask() local
977 p4dp = p4d_offset(pgdp, address); in follow_p4d_mask()
978 p4d = p4dp_get(p4dp); in follow_p4d_mask()
984 return follow_pud_mask(vma, address, p4dp, flags, page_mask); in follow_p4d_mask()
3043 static int gup_fast_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, in gup_fast_pud_range() argument
3050 pudp = pud_offset_lockless(p4dp, p4d, addr); in gup_fast_pud_range()
3074 p4d_t *p4dp; in gup_fast_p4d_range() local
3076 p4dp = p4d_offset_lockless(pgdp, pgd, addr); in gup_fast_p4d_range()
[all …]
|
| H A D | migrate_device.c | 989 p4d_t *p4dp; in migrate_vma_insert_page() local
1000 p4dp = p4d_alloc(mm, pgdp, addr); in migrate_vma_insert_page()
1001 if (!p4dp) in migrate_vma_insert_page()
1003 pudp = pud_alloc(mm, p4dp, addr); in migrate_vma_insert_page()
|
| H A D | khugepaged.c | 1825 p4d_t *p4dp; in set_huge_pmd() local
1832 p4dp = p4d_alloc(mm, pgdp, addr); in set_huge_pmd()
1833 if (!p4dp) in set_huge_pmd()
1835 pudp = pud_alloc(mm, p4dp, addr); in set_huge_pmd()
|
| H A D | memory.c | 568 p4d_t p4d, *p4dp; in __print_bad_page_map_pgtable() local
585 p4dp = p4d_offset(pgdp, addr); in __print_bad_page_map_pgtable()
586 p4d = p4dp_get(p4dp); in __print_bad_page_map_pgtable()
594 pudp = pud_offset(p4dp, addr); in __print_bad_page_map_pgtable()
7046 p4d_t *p4dp, p4d; in follow_pfnmap_start() local
7063 p4dp = p4d_offset(pgdp, address); in follow_pfnmap_start()
7064 p4d = p4dp_get(p4dp); in follow_pfnmap_start()
7068 pudp = pud_offset(p4dp, address); in follow_pfnmap_start()
|
| /linux/arch/mips/mm/ |
| H A D | tlb-r4k.c | 301 p4d_t *p4dp; in __update_tlb() local
327 p4dp = p4d_offset(pgdp, address); in __update_tlb()
328 pudp = pud_offset(p4dp, address); in __update_tlb()
|
| /linux/arch/sparc/kernel/ |
| H A D | signal32.c | 300 p4d_t *p4dp; in flush_signal_insns() local
320 p4dp = p4d_offset(pgdp, address); in flush_signal_insns()
321 if (p4d_none(*p4dp)) in flush_signal_insns()
323 pudp = pud_offset(p4dp, address); in flush_signal_insns()
|
| /linux/arch/loongarch/include/asm/ |
| H A D | pgtable.h | 189 static inline void p4d_clear(p4d_t *p4dp) in p4d_clear() argument
191 set_p4d(p4dp, __p4d((unsigned long)invalid_pud_table)); in p4d_clear()
|
| /linux/arch/powerpc/kvm/ |
| H A D | book3s_64_mmu_radix.c | 1291 p4d_t p4d, *p4dp; in debugfs_radix_read() local
1364 p4dp = p4d_offset(pgdp, gpa); in debugfs_radix_read()
1365 p4d = READ_ONCE(*p4dp); in debugfs_radix_read()
|
| /linux/kernel/events/ |
| H A D | core.c | 8534 p4d_t *p4dp, p4d; in perf_get_pgtable_size()
8547 p4dp = p4d_offset_lockless(pgdp, pgd, addr); in perf_get_pgtable_size()
8548 p4d = p4dp_get(p4dp); in perf_get_pgtable_size()
8555 pudp = pud_offset_lockless(p4dp, p4d, addr); in perf_get_pgtable_size()
8520 p4d_t *p4dp, p4d; perf_get_pgtable_size() local
|