| /linux/arch/arm64/mm/ |
| H A D | mmu.c | 69 void noinstr set_swapper_pgd(pgd_t *pgdp, pgd_t pgd) in set_swapper_pgd() argument
78 WRITE_ONCE(*pgdp, pgd); in set_swapper_pgd()
85 fixmap_pgdp = pgd_set_fixmap(__pa_symbol(pgdp)); in set_swapper_pgd()
430 static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end, in alloc_init_p4d() argument
437 pgd_t pgd = READ_ONCE(*pgdp); in alloc_init_p4d()
453 __pgd_populate(pgdp, p4d_phys, pgdval); in alloc_init_p4d()
456 p4dp = p4d_set_fixmap_offset(pgdp, addr); in alloc_init_p4d()
489 pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); in __create_pgd_mapping_locked() local
504 ret = alloc_init_p4d(pgdp, addr, next, phys, prot, pgtable_alloc, in __create_pgd_mapping_locked()
509 } while (pgdp in __create_pgd_mapping_locked()
690 pgd_t *pgdp, pgd; split_kernel_leaf_mapping_locked() local
1615 unmap_hotplug_p4d_range(pgd_t * pgdp,unsigned long addr,unsigned long end,bool free_mapped,struct vmem_altmap * altmap) unmap_hotplug_p4d_range() argument
1639 pgd_t *pgdp, pgd; unmap_hotplug_range() local
1781 free_empty_p4d_table(pgd_t * pgdp,unsigned long addr,unsigned long end,unsigned long floor,unsigned long ceiling) free_empty_p4d_table() argument
1825 pgd_t *pgdp, pgd; free_empty_tables() local
2055 pgd_t *pgdp, pgd; addr_splits_kernel_leaf() local
2327 __cpu_replace_ttbr1(pgd_t * pgdp,bool cnp) __cpu_replace_ttbr1() argument
[all...] |
| H A D | hugetlbpage.c | 81 pgd_t *pgdp = pgd_offset(mm, addr); in find_num_contig() local
87 p4dp = p4d_offset(pgdp, addr); in find_num_contig()
240 pgd_t *pgdp; in huge_pte_alloc() local
246 pgdp = pgd_offset(mm, addr); in huge_pte_alloc()
247 p4dp = p4d_alloc(mm, pgdp, addr); in huge_pte_alloc()
281 pgd_t *pgdp; in huge_pte_offset() local
286 pgdp = pgd_offset(mm, addr); in huge_pte_offset()
287 if (!pgd_present(READ_ONCE(*pgdp))) in huge_pte_offset()
290 p4dp = p4d_offset(pgdp, addr); in huge_pte_offset()
|
| H A D | fault.c | 135 pgd_t *pgdp; in show_pte() local
161 pgdp = pgd_offset(mm, addr); in show_pte()
162 pgd = READ_ONCE(*pgdp); in show_pte()
174 p4dp = p4d_offset_lockless(pgdp, pgd, addr); in show_pte()
|
| /linux/arch/sparc/mm/ |
| H A D | srmmu.c | 110 static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp) in srmmu_ctxd_set() argument
114 pte = __pte((SRMMU_ET_PTD | (__nocache_pa(pgdp) >> 4))); in srmmu_ctxd_set()
489 pgd_t *pgdp; in srmmu_mapioaddr() local
497 pgdp = pgd_offset_k(virt_addr); in srmmu_mapioaddr()
498 p4dp = p4d_offset(pgdp, virt_addr); in srmmu_mapioaddr()
528 pgd_t *pgdp; in srmmu_unmapioaddr() local
535 pgdp = pgd_offset_k(virt_addr); in srmmu_unmapioaddr()
536 p4dp = p4d_offset(pgdp, virt_addr); in srmmu_unmapioaddr()
646 pgd_t *pgdp; in srmmu_early_allocate_ptable_skeleton() local
653 pgdp = pgd_offset_k(start); in srmmu_early_allocate_ptable_skeleton()
[all …]
|
| H A D | fault_64.c | 81 pgd_t *pgdp = pgd_offset(current->mm, tpc); in get_user_insn() local
89 if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) in get_user_insn()
91 p4dp = p4d_offset(pgdp, tpc); in get_user_insn()
|
| /linux/arch/s390/mm/ |
| H A D | gmap_helpers.c | 48 pgd_t *pgdp, pgd; in try_get_locked_pte() local
51 pgdp = pgd_offset(mm, vmaddr); in try_get_locked_pte()
52 pgd = pgdp_get(pgdp); in try_get_locked_pte()
55 p4dp = p4d_offset_lockless(pgdp, pgd, vmaddr); in try_get_locked_pte()
|
| /linux/arch/x86/include/asm/ |
| H A D | pgtable.h | 68 #define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd) argument
905 pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd);
912 static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) in pti_set_user_pgtbl() argument
916 return __pti_set_user_pgtbl(pgdp, pgd); in pti_set_user_pgtbl()
919 static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) in pti_set_user_pgtbl() argument
1435 static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp) in kernel_to_user_pgdp() argument
1437 return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT); in kernel_to_user_pgdp()
1440 static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp) in user_to_kernel_pgdp() argument
1442 return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT); in user_to_kernel_pgdp()
1739 #define set_pgd_safe(pgdp, pgd) \ argument
[all …]
|
| H A D | pgtable_64.h | 158 static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) in native_set_pgd() argument
160 WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd)); in native_set_pgd()
|
| /linux/arch/riscv/mm/ |
| H A D | fault.c | 30 pgd_t *pgdp, pgd; in show_pte() local
44 pgdp = pgd_offset(mm, addr); in show_pte()
45 pgd = pgdp_get(pgdp); in show_pte()
50 p4dp = p4d_offset(pgdp, addr); in show_pte()
|
| H A D | init.c | 685 void __meminit create_pgd_mapping(pgd_t *pgdp, uintptr_t va, phys_addr_t pa, phys_addr_t sz, in create_pgd_mapping() argument
693 if (pgd_val(pgdp[pgd_idx]) == 0) in create_pgd_mapping()
694 pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(pa), prot); in create_pgd_mapping()
698 if (pgd_val(pgdp[pgd_idx]) == 0) { in create_pgd_mapping()
700 pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); in create_pgd_mapping()
704 next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_idx])); in create_pgd_mapping()
|
| /linux/arch/riscv/include/asm/ |
| H A D | pgtable-64.h | 343 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) in set_pgd() argument
346 WRITE_ONCE(*pgdp, pgd); in set_pgd()
348 set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) }); in set_pgd()
|
| H A D | mmu.h | 41 void __meminit create_pgd_mapping(pgd_t *pgdp, uintptr_t va, phys_addr_t pa, phys_addr_t sz,
|
| /linux/arch/arm/include/asm/ |
| H A D | pgtable-nommu.h | 23 #define pgd_clear(pgdp) argument
|
| /linux/arch/m68k/include/asm/ |
| H A D | mcf_pgtable.h | 107 static inline void pgd_set(pgd_t *pgdp, pmd_t *pmdp) in pgd_set() argument
109 pgd_val(*pgdp) = virt_to_phys(pmdp); in pgd_set()
|
| /linux/arch/x86/boot/compressed/ |
| H A D | ident_map_64.c | 259 pgd_t *pgdp = (pgd_t *)top_level_pgt; in set_clr_page_flags() local
283 p4dp = p4d_offset(pgdp, address); in set_clr_page_flags()
|
| /linux/arch/riscv/kernel/ |
| H A D | hibernate.c | 312 static int temp_pgtable_mapping(pgd_t *pgdp, unsigned long start, unsigned long end, pgprot_t prot) in temp_pgtable_mapping() argument
314 pgd_t *dst_pgdp = pgd_offset_pgd(pgdp, start); in temp_pgtable_mapping()
|
| /linux/arch/mips/mm/ |
| H A D | tlb-r4k.c | 300 pgd_t *pgdp; in __update_tlb() local
323 pgdp = pgd_offset(vma->vm_mm, address); in __update_tlb()
327 p4dp = p4d_offset(pgdp, address); in __update_tlb()
|
| /linux/arch/sparc/kernel/ |
| H A D | signal32.c | 299 pgd_t *pgdp; in flush_signal_insns() local
317 pgdp = pgd_offset(current->mm, address); in flush_signal_insns()
318 if (pgd_none(*pgdp)) in flush_signal_insns()
320 p4dp = p4d_offset(pgdp, address); in flush_signal_insns()
|
| /linux/mm/ |
| H A D | gup.c | 971 unsigned long address, pgd_t *pgdp, in follow_p4d_mask() argument
977 p4dp = p4d_offset(pgdp, address); in follow_p4d_mask()
3069 static int gup_fast_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, in gup_fast_p4d_range() argument
3076 p4dp = p4d_offset_lockless(pgdp, pgd, addr); in gup_fast_p4d_range()
3096 pgd_t *pgdp; in gup_fast_pgd_range() local
3098 pgdp = pgd_offset(current->mm, addr); in gup_fast_pgd_range()
3100 pgd_t pgd = pgdp_get(pgdp); in gup_fast_pgd_range()
3106 if (!gup_fast_p4d_range(pgdp, pgd, addr, next, flags, in gup_fast_pgd_range()
3109 } while (pgdp++, addr = next, addr != end); in gup_fast_pgd_range()
|
| H A D | migrate_device.c | 988 pgd_t *pgdp; in migrate_vma_insert_page() local
999 pgdp = pgd_offset(mm, addr); in migrate_vma_insert_page()
1000 p4dp = p4d_alloc(mm, pgdp, addr); in migrate_vma_insert_page()
|
| H A D | memory.c | 571 pgd_t *pgdp; in __print_bad_page_map_pgtable() local
577 pgdp = pgd_offset(mm, addr); in __print_bad_page_map_pgtable()
578 ptval_to_str(pgd_str, pgd_val(*pgdp)); in __print_bad_page_map_pgtable()
580 if (!pgd_present(*pgdp) || pgd_leaf(*pgdp)) { in __print_bad_page_map_pgtable()
585 p4dp = p4d_offset(pgdp, addr); in __print_bad_page_map_pgtable()
7045 pgd_t *pgdp; in follow_pfnmap_start() local
7059 pgdp = pgd_offset(mm, address); in follow_pfnmap_start()
7060 if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) in follow_pfnmap_start()
7063 p4dp = p4d_offset(pgdp, address); in follow_pfnmap_start()
|
| H A D | khugepaged.c | 1824 pgd_t *pgdp; in set_huge_pmd() local
1831 pgdp = pgd_offset(mm, addr); in set_huge_pmd()
1832 p4dp = p4d_alloc(mm, pgdp, addr); in set_huge_pmd()
|
| /linux/arch/powerpc/kvm/ |
| H A D | book3s_64_mmu_radix.c | 1290 pgd_t *pgdp; in debugfs_radix_read() local
1363 pgdp = pgt + pgd_index(gpa); in debugfs_radix_read()
1364 p4dp = p4d_offset(pgdp, gpa); in debugfs_radix_read()
|
| /linux/kernel/events/ |
| H A D | core.c | 8533 pgd_t *pgdp, pgd; in perf_get_pgtable_size()
8539 pgdp = pgd_offset(mm, addr); in perf_get_pgtable_size()
8540 pgd = pgdp_get(pgdp); in perf_get_pgtable_size()
8547 p4dp = p4d_offset_lockless(pgdp, pgd, addr); in perf_get_pgtable_size()
8519 pgd_t *pgdp, pgd; perf_get_pgtable_size() local
|