/linux/arch/arm64/mm/ |
H A D | mmu.c | 277 static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr, in alloc_init_cont_pmd() argument 283 pud_t pud = READ_ONCE(*pudp); in alloc_init_cont_pmd() 301 __pud_populate(pudp, pmd_phys, pudval); in alloc_init_cont_pmd() 304 pmdp = pmd_set_fixmap_offset(pudp, addr); in alloc_init_cont_pmd() 333 pud_t *pudp; in alloc_init_pud() local 343 pudp = pud_set_fixmap(pud_phys); in alloc_init_pud() 344 init_clear_pgtable(pudp); in alloc_init_pud() 345 pudp += pud_index(addr); in alloc_init_pud() 349 pudp = pud_set_fixmap_offset(p4dp, addr); in alloc_init_pud() 353 pud_t old_pud = READ_ONCE(*pudp); in alloc_init_pud() [all …]
|
H A D | kasan_init.c | 76 static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node, in kasan_pmd_offset() argument 79 if (pud_none(READ_ONCE(*pudp))) { in kasan_pmd_offset() 83 __pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE); in kasan_pmd_offset() 86 return early ? pmd_offset_kimg(pudp, addr) : pmd_offset(pudp, addr); in kasan_pmd_offset() 132 static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr, in kasan_pmd_populate() argument 136 pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early); in kasan_pmd_populate() 148 pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early); in kasan_pud_populate() local 152 kasan_pmd_populate(pudp, addr, next, node, early); in kasan_pud_populate() 153 } while (pudp++, addr = next, addr != end && pud_none(READ_ONCE(*pudp))); in kasan_pud_populate() 260 pud_t *pudp = (pud_t *)__phys_to_kimg(__pgd_to_phys(pgd)); in clone_next_level() local [all …]
|
/linux/arch/powerpc/mm/kasan/ |
H A D | init_book3e_64.c | 36 pud_t *pudp; in kasan_map_kernel_page() local 43 pudp = memblock_alloc(PUD_TABLE_SIZE, PUD_TABLE_SIZE); in kasan_map_kernel_page() 44 memcpy(pudp, kasan_early_shadow_pud, PUD_TABLE_SIZE); in kasan_map_kernel_page() 45 p4d_populate(&init_mm, p4dp, pudp); in kasan_map_kernel_page() 47 pudp = pud_offset(p4dp, ea); in kasan_map_kernel_page() 48 if (kasan_pmd_table(*pudp)) { in kasan_map_kernel_page() 51 pud_populate(&init_mm, pudp, pmdp); in kasan_map_kernel_page() 53 pmdp = pmd_offset(pudp, ea); in kasan_map_kernel_page()
|
/linux/arch/x86/include/asm/ |
H A D | pgtable-3level.h | 53 static inline void native_set_pud(pud_t *pudp, pud_t pud) in native_set_pud() argument 56 pud.p4d.pgd = pti_set_user_pgtbl(&pudp->p4d.pgd, pud.p4d.pgd); in native_set_pud() 58 pxx_xchg64(pud, pudp, native_pud_val(pud)); in native_set_pud() 81 static inline void native_pud_clear(pud_t *pudp) in native_pud_clear() argument 85 static inline void pud_clear(pud_t *pudp) in pud_clear() argument 87 set_pud(pudp, __pud(0)); in pud_clear() 113 static inline pud_t native_pudp_get_and_clear(pud_t *pudp) in native_pudp_get_and_clear() argument 115 return pxx_xchg64(pud, pudp, 0ULL); in native_pudp_get_and_clear()
|
/linux/arch/riscv/mm/ |
H A D | kasan_init.c | 84 pud_t *pudp, *p; in kasan_populate_pud() local 92 pudp = pud_offset(p4d, vaddr); in kasan_populate_pud() 97 if (pud_none(pudp_get(pudp)) && IS_ALIGNED(vaddr, PUD_SIZE) && in kasan_populate_pud() 101 set_pud(pudp, pfn_pud(PFN_DOWN(phys_addr), PAGE_KERNEL)); in kasan_populate_pud() 107 kasan_populate_pmd(pudp, vaddr, next); in kasan_populate_pud() 108 } while (pudp++, vaddr = next, vaddr != end); in kasan_populate_pud() 168 pud_t *pudp, *base_pud; in kasan_early_clear_pud() local 172 pudp = (pud_t *)p4dp; in kasan_early_clear_pud() 175 pudp = base_pud + pud_index(vaddr); in kasan_early_clear_pud() 182 pud_clear(pudp); in kasan_early_clear_pud() [all …]
|
/linux/arch/loongarch/mm/ |
H A D | kasan_init.c | 131 static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node, bool early) in kasan_pmd_offset() argument 133 if (__pud_none(early, pudp_get(pudp))) { in kasan_pmd_offset() 138 pud_populate(&init_mm, pudp, (pmd_t *)__va(pmd_phys)); in kasan_pmd_offset() 141 return pmd_offset(pudp, addr); in kasan_pmd_offset() 185 static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr, in kasan_pmd_populate() argument 189 pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early); in kasan_pmd_populate() 201 pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early); in kasan_pud_populate() local 205 kasan_pmd_populate(pudp, addr, next, node, early); in kasan_pud_populate() 206 } while (pudp++, addr = next, addr != end && __pud_none(early, READ_ONCE(*pudp))); in kasan_pud_populate()
|
/linux/mm/ |
H A D | debug_vm_pgtable.c | 52 pud_t *pudp; member 357 set_pud_at(args->mm, vaddr, args->pudp, pud); in pud_advanced_tests() 359 pudp_set_wrprotect(args->mm, vaddr, args->pudp); in pud_advanced_tests() 360 pud = pudp_get(args->pudp); in pud_advanced_tests() 364 pudp_huge_get_and_clear(args->mm, vaddr, args->pudp); in pud_advanced_tests() 365 pud = pudp_get(args->pudp); in pud_advanced_tests() 372 set_pud_at(args->mm, vaddr, args->pudp, pud); in pud_advanced_tests() 376 pudp_set_access_flags(args->vma, vaddr, args->pudp, pud, 1); in pud_advanced_tests() 377 pud = pudp_get(args->pudp); in pud_advanced_tests() 381 pudp_huge_get_and_clear_full(args->vma, vaddr, args->pudp, 1); in pud_advanced_tests() [all …]
|
H A D | gup.c | 626 unsigned long addr, pud_t *pudp, in follow_huge_pud() argument 631 pud_t pud = *pudp; in follow_huge_pud() 635 assert_spin_locked(pud_lockptr(mm, pudp)); in follow_huge_pud() 658 touch_pud(vma, addr, pudp, flags & FOLL_WRITE); in follow_huge_pud() 765 unsigned long addr, pud_t *pudp, in follow_huge_pud() argument 954 unsigned long address, pud_t *pudp, in follow_pmd_mask() argument 963 pmd = pmd_offset(pudp, address); in follow_pmd_mask() 1010 pud_t *pudp, pud; in follow_pud_mask() local 1015 pudp = pud_offset(p4dp, address); in follow_pud_mask() 1016 pud = READ_ONCE(*pudp); in follow_pud_mask() [all …]
|
H A D | pagewalk.c | 836 pud_t *pudp, pud; in folio_walk_start() 857 pudp = pud_offset(p4dp, addr); in folio_walk_start() 858 pud = pudp_get(pudp); in folio_walk_start() 863 ptl = pud_lock(vma->vm_mm, pudp); in folio_walk_start() 864 pud = pudp_get(pudp); in folio_walk_start() 868 fw->pudp = pudp; 892 pmdp = pmd_offset(pudp, addr); 722 pud_t *pudp, pud; folio_walk_start() local
|
H A D | mprotect.c | 436 pud_t *pudp, pud; in change_pud_range() 442 pudp = pud_offset(p4d, addr); in change_pud_range() 446 ret = change_prepare(vma, pudp, pmd, addr, cp_flags); in change_pud_range() 452 pud = READ_ONCE(*pudp); in change_pud_range() 466 __split_huge_pud(vma, pudp, addr); in change_pud_range() 469 ret = change_huge_pud(tlb, vma, pudp, in change_pud_range() 480 pages += change_pmd_range(tlb, vma, pudp, addr, next, newprot, in change_pud_range() 482 } while (pudp++, addr = next, addr != end); in change_pud_range() 434 pud_t *pudp, pud; change_pud_range() local
|
H A D | pgtable-generic.c | 151 pud_t *pudp) in pudp_huge_clear_flush() argument 156 VM_BUG_ON(!pud_trans_huge(*pudp) && !pud_devmap(*pudp)); in pudp_huge_clear_flush() 157 pud = pudp_huge_get_and_clear(vma->vm_mm, address, pudp); in pudp_huge_clear_flush()
|
/linux/arch/arc/include/asm/ |
H A D | pgalloc.h | 75 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) in p4d_populate() argument 77 set_p4d(p4dp, __p4d((unsigned long)pudp)); in p4d_populate() 86 static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) in pud_populate() argument 88 set_pud(pudp, __pud((unsigned long)pmdp)); in pud_populate()
|
/linux/include/trace/events/ |
H A D | xen.h | 207 TP_PROTO(pud_t *pudp, pud_t pudval), 208 TP_ARGS(pudp, pudval), 210 __field(pud_t *, pudp) 213 TP_fast_assign(__entry->pudp = pudp; 216 __entry->pudp, 242 TP_PROTO(pud_t *pudp, pud_t pudval), 243 TP_ARGS(pudp, pudval), 245 __field(pud_t *, pudp) 248 TP_fast_assign(__entry->pudp = pudp; 251 __entry->pudp,
|
/linux/arch/x86/mm/ |
H A D | pgtable.c | 191 void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd) in pud_populate() argument 197 set_pud(pudp, __pud(__pa(pmd) | _PAGE_PRESENT)); in pud_populate() 537 pud_t *pudp, pud_t entry, int dirty) in pudp_set_access_flags() argument 539 int changed = !pud_same(*pudp, entry); in pudp_set_access_flags() 544 set_pud(pudp, entry); in pudp_set_access_flags() 585 unsigned long addr, pud_t *pudp) in pudp_test_and_clear_young() argument 589 if (pud_young(*pudp)) in pudp_test_and_clear_young() 591 (unsigned long *)pudp); in pudp_test_and_clear_young() 647 pud_t *pudp) in pudp_invalidate() argument 649 VM_WARN_ON_ONCE(!pud_present(*pudp)); in pudp_invalidate() [all …]
|
/linux/arch/powerpc/include/asm/book3s/64/ |
H A D | pgtable.h | 862 static inline void pud_clear(pud_t *pudp) in pud_clear() argument 869 WARN_ON((pud_val(*pudp) & (H_PAGE_HASHPTE | _PAGE_PTE)) == (H_PAGE_HASHPTE | _PAGE_PTE)); in pud_clear() 871 *pudp = __pud(0); in pud_clear() 1105 pud_t *pudp, pud_t pud); 1144 pud_hugepage_update(struct mm_struct *mm, unsigned long addr, pud_t *pudp, in pud_hugepage_update() argument 1148 return radix__pud_hugepage_update(mm, addr, pudp, clr, set); in pud_hugepage_update() 1150 return pud_val(*pudp); in pud_hugepage_update() 1169 unsigned long addr, pud_t *pudp) in __pudp_test_and_clear_young() argument 1173 if ((pud_raw(*pudp) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0) in __pudp_test_and_clear_young() 1175 old = pud_hugepage_update(mm, addr, pudp, _PAGE_ACCESSED, 0); in __pudp_test_and_clear_young() [all …]
|
/linux/include/asm-generic/ |
H A D | tlb.h | 674 #define __tlb_remove_pud_tlb_entry(tlb, pudp, address) do {} while (0) argument 677 #define tlb_remove_pud_tlb_entry(tlb, pudp, address) \ argument 680 __tlb_remove_pud_tlb_entry(tlb, pudp, address); \ 720 #define pud_free_tlb(tlb, pudp, address) \ argument 724 __pud_free_tlb(tlb, pudp, address); \ 729 #define p4d_free_tlb(tlb, pudp, address) \ argument 733 __p4d_free_tlb(tlb, pudp, address); \
|
/linux/include/linux/ |
H A D | page_table_check.h | 23 void __page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, pud_t pud); 86 static inline void page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, in page_table_check_pud_set() argument 92 __page_table_check_pud_set(mm, pudp, pud); in page_table_check_pud_set() 137 static inline void page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, in page_table_check_pud_set() argument
|
/linux/arch/arm/mm/ |
H A D | kasan_init.c | 93 static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr, in kasan_pmd_populate() argument 97 pmd_t *pmdp = pmd_offset(pudp, addr); in kasan_pmd_populate() 129 pud_t *pudp; in kasan_pgd_populate() local 156 pudp = pud_offset(p4dp, addr); in kasan_pgd_populate() 158 kasan_pmd_populate(pudp, addr, next, early); in kasan_pgd_populate()
|
/linux/arch/powerpc/include/asm/nohash/64/ |
H A D | pgtable.h | 109 static inline void pud_set(pud_t *pudp, unsigned long val) in pud_set() argument 111 *pudp = __pud(val); in pud_set() 114 static inline void pud_clear(pud_t *pudp) in pud_clear() argument 116 *pudp = __pud(0); in pud_clear()
|
/linux/arch/m68k/include/asm/ |
H A D | motorola_pgtable.h | 101 static inline void pud_set(pud_t *pudp, pmd_t *pmdp) in pud_set() argument 103 pud_val(*pudp) = _PAGE_TABLE | _PAGE_ACCESSED | __pa(pmdp); in pud_set() 137 #define pud_clear(pudp) ({ pud_val(*pudp) = 0; }) argument
|
/linux/arch/sparc/mm/ |
H A D | fault_64.c | 83 pud_t *pudp; in get_user_insn() local 94 pudp = pud_offset(p4dp, tpc); in get_user_insn() 95 if (pud_none(*pudp) || unlikely(pud_bad(*pudp))) in get_user_insn() 101 pmdp = pmd_offset(pudp, tpc); in get_user_insn()
|
/linux/arch/sparc/include/asm/ |
H A D | pgalloc_32.h | 29 static inline void pud_set(pud_t * pudp, pmd_t * pmdp) in pud_set() argument 33 set_pte((pte_t *)pudp, __pte((SRMMU_ET_PTD | (pa >> 4)))); in pud_set()
|
/linux/arch/alpha/include/asm/ |
H A D | pgtable.h | 212 extern inline void pud_set(pud_t * pudp, pmd_t * pmdp) in pud_set() argument 213 { pud_val(*pudp) = _PAGE_TABLE | ((((unsigned long) pmdp) - PAGE_OFFSET) << (32-PAGE_SHIFT)); } in pud_set() 246 extern inline void pud_clear(pud_t * pudp) { pud_val(*pudp) = 0; } in pud_clear() argument
|
/linux/arch/riscv/include/asm/ |
H A D | pgtable-64.h | 201 static inline void set_pud(pud_t *pudp, pud_t pud) in set_pud() argument 203 WRITE_ONCE(*pudp, pud); in set_pud() 206 static inline void pud_clear(pud_t *pudp) in pud_clear() argument 208 set_pud(pudp, __pud(0)); in pud_clear()
|
/linux/arch/arm64/include/asm/ |
H A D | tlb.h | 101 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp, in __pud_free_tlb() argument 104 struct ptdesc *ptdesc = virt_to_ptdesc(pudp); in __pud_free_tlb()
|