Home
last modified time | relevance | path

Searched refs:pgd (Results 1 – 25 of 189) sorted by relevance

12345678

/linux/arch/x86/mm/
H A Dkasan_init_64.c122 static void __init kasan_populate_pgd(pgd_t *pgd, unsigned long addr, in kasan_populate_pgd() argument
129 if (pgd_none(*pgd)) { in kasan_populate_pgd()
131 pgd_populate(&init_mm, pgd, p); in kasan_populate_pgd()
134 p4d = p4d_offset(pgd, addr); in kasan_populate_pgd()
144 pgd_t *pgd; in kasan_populate_shadow() local
149 pgd = pgd_offset_k(addr); in kasan_populate_shadow()
152 kasan_populate_pgd(pgd, addr, next, nid); in kasan_populate_shadow()
153 } while (pgd++, addr = next, addr != end); in kasan_populate_shadow()
170 pgd_t *pgd; in clear_pgds() local
175 pgd = pgd_offset_k(start); in clear_pgds()
[all …]
/linux/arch/powerpc/include/asm/book3s/64/
H A Dpgalloc.h39 static inline void radix__pgd_free(struct mm_struct *mm, pgd_t *pgd) in radix__pgd_free() argument
42 free_page((unsigned long)pgd); in radix__pgd_free()
44 free_pages((unsigned long)pgd, 4); in radix__pgd_free()
50 pgd_t *pgd; in pgd_alloc() local
55 pgd = kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), in pgd_alloc()
57 if (unlikely(!pgd)) in pgd_alloc()
58 return pgd; in pgd_alloc()
65 kmemleak_no_scan(pgd); in pgd_alloc()
76 memset(pgd, 0, PGD_TABLE_SIZE); in pgd_alloc()
78 return pgd; in pgd_alloc()
[all …]
/linux/arch/arm/mm/
H A Dpgd.c21 #define _pgd_free(mm, pgd) kfree(pgd) argument
24 #define _pgd_free(mm, pgd) __pgd_free(mm, pgd) argument
142 pgd_t *pgd; in pgd_free() local
151 pgd = pgd_base + pgd_index(0); in pgd_free()
152 if (pgd_none_or_clear_bad(pgd)) in pgd_free()
155 p4d = p4d_offset(pgd, 0); in pgd_free()
179 pgd_clear(pgd); in pgd_free()
186 for (pgd = pgd_base; pgd < pgd_base + PTRS_PER_PGD; pgd++) { in pgd_free()
187 if (pgd_none_or_clear_bad(pgd)) in pgd_free()
189 if (pgd_val(*pgd) & L_PGD_SWAPPER) in pgd_free()
[all …]
/linux/arch/riscv/include/asm/
H A Dpgtable-64.h343 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) in set_pgd() argument
346 WRITE_ONCE(*pgdp, pgd); in set_pgd()
348 set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) }); in set_pgd()
351 static inline int pgd_none(pgd_t pgd) in pgd_none() argument
354 return (pgd_val(pgd) == 0); in pgd_none()
359 static inline int pgd_present(pgd_t pgd) in pgd_present() argument
362 return (pgd_val(pgd) & _PAGE_PRESENT); in pgd_present()
367 static inline int pgd_bad(pgd_t pgd) in pgd_bad() argument
370 return !pgd_present(pgd); in pgd_bad()
375 static inline void pgd_clear(pgd_t *pgd) in pgd_clear() argument
[all …]
H A Dpgalloc.h62 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) in pgd_populate() argument
67 set_pgd(pgd, __pgd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); in pgd_populate()
71 static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, in pgd_populate_safe() argument
77 set_pgd_safe(pgd, in pgd_populate_safe()
104 static inline void sync_kernel_mappings(pgd_t *pgd) in sync_kernel_mappings() argument
106 memcpy(pgd + USER_PTRS_PER_PGD, in sync_kernel_mappings()
107 init_mm.pgd + USER_PTRS_PER_PGD, in sync_kernel_mappings()
113 pgd_t *pgd; in pgd_alloc() local
115 pgd = __pgd_alloc(mm, 0); in pgd_alloc()
116 if (likely(pgd != NULL)) { in pgd_alloc()
[all …]
/linux/arch/x86/power/
H A Dhibernate_32.c30 static pmd_t *resume_one_md_table_init(pgd_t *pgd) in resume_one_md_table_init() argument
41 set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT)); in resume_one_md_table_init()
42 p4d = p4d_offset(pgd, 0); in resume_one_md_table_init()
47 p4d = p4d_offset(pgd, 0); in resume_one_md_table_init()
84 pgd_t *pgd; in resume_physical_mapping_init() local
90 pgd = pgd_base + pgd_idx; in resume_physical_mapping_init()
93 for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) { in resume_physical_mapping_init()
94 pmd = resume_one_md_table_init(pgd); in resume_physical_mapping_init()
147 pgd_t *pgd; in set_up_temporary_text_mapping() local
151 pgd = pgd_base + pgd_index(restore_jump_address); in set_up_temporary_text_mapping()
[all …]
/linux/arch/sh/mm/
H A Dfault.c39 pgd_t *pgd; in show_pte() local
42 pgd = mm->pgd; in show_pte()
44 pgd = get_TTB(); in show_pte()
46 if (unlikely(!pgd)) in show_pte()
47 pgd = swapper_pg_dir; in show_pte()
50 pr_alert("pgd = %p\n", pgd); in show_pte()
51 pgd += pgd_index(addr); in show_pte()
52 pr_alert("[%08lx] *pgd=%0*llx", addr, (u32)(sizeof(*pgd) * 2), in show_pte()
53 (u64)pgd_val(*pgd)); in show_pte()
61 if (pgd_none(*pgd)) in show_pte()
[all …]
H A Dpgtable.c14 pgd_t *pgd = x; in pgd_ctor() local
16 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); in pgd_ctor()
17 memcpy(pgd + USER_PTRS_PER_PGD, in pgd_ctor()
39 void pgd_free(struct mm_struct *mm, pgd_t *pgd) in pgd_free() argument
41 kmem_cache_free(pgd_cachep, pgd); in pgd_free()
/linux/arch/x86/kernel/
H A Dmachine_kexec_32.c45 free_pages((unsigned long)image->arch.pgd, pgd_allocation_order()); in machine_kexec_free_page_tables()
46 image->arch.pgd = NULL; in machine_kexec_free_page_tables()
61 image->arch.pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, in machine_kexec_alloc_page_tables()
69 if (!image->arch.pgd || in machine_kexec_alloc_page_tables()
80 pgd_t *pgd, pmd_t *pmd, pte_t *pte, in machine_kexec_page_table_set_one() argument
86 pgd += pgd_index(vaddr); in machine_kexec_page_table_set_one()
88 if (!(pgd_val(*pgd) & _PAGE_PRESENT)) in machine_kexec_page_table_set_one()
89 set_pgd(pgd, __pgd(__pa(pmd) | _PAGE_PRESENT)); in machine_kexec_page_table_set_one()
91 p4d = p4d_offset(pgd, vaddr); in machine_kexec_page_table_set_one()
110 image->arch.pgd, pmd, image->arch.pte0, in machine_kexec_prepare_page_tables()
[all …]
/linux/arch/hexagon/include/asm/
H A Dpgalloc.h23 pgd_t *pgd; in pgd_alloc() local
25 pgd = __pgd_alloc(mm, 0); in pgd_alloc()
35 memcpy(pgd, swapper_pg_dir, PTRS_PER_PGD*sizeof(pgd_t)); in pgd_alloc()
39 mm->context.ptbase = __pa(pgd); in pgd_alloc()
41 return pgd; in pgd_alloc()
83 pmdindex = (pgd_t *)pmd - mm->pgd; in pmd_populate_kernel()
84 ppmd = (pmd_t *)current->active_mm->pgd + pmdindex; in pmd_populate_kernel()
/linux/arch/loongarch/mm/
H A Dhugetlbpage.c19 pgd_t *pgd; in huge_pte_alloc() local
24 pgd = pgd_offset(mm, addr); in huge_pte_alloc()
25 p4d = p4d_alloc(mm, pgd, addr); in huge_pte_alloc()
36 pgd_t *pgd; in huge_pte_offset() local
41 pgd = pgd_offset(mm, addr); in huge_pte_offset()
42 if (pgd_present(pgdp_get(pgd))) { in huge_pte_offset()
43 p4d = p4d_offset(pgd, addr); in huge_pte_offset()
H A Dpageattr.c28 static int pageattr_pgd_entry(pgd_t *pgd, unsigned long addr, in pageattr_pgd_entry() argument
31 pgd_t val = pgdp_get(pgd); in pageattr_pgd_entry()
35 set_pgd(pgd, val); in pageattr_pgd_entry()
163 pgd_t *pgd; in kernel_page_present() local
173 pgd = pgd_offset_k(addr); in kernel_page_present()
174 if (pgd_none(pgdp_get(pgd))) in kernel_page_present()
176 if (pgd_leaf(pgdp_get(pgd))) in kernel_page_present()
179 p4d = p4d_offset(pgd, addr); in kernel_page_present()
/linux/arch/x86/xen/
H A Dmmu_pv.c100 pgdval_t xen_pgd_val(pgd_t pgd);
105 pgd_t xen_make_pgd(pgdval_t pgd);
437 __visible pgdval_t xen_pgd_val(pgd_t pgd) in xen_pgd_val() argument
439 return pte_mfn_to_pfn(pgd.pgd); in xen_pgd_val()
451 __visible pgd_t xen_make_pgd(pgdval_t pgd) in xen_make_pgd() argument
453 pgd = pte_pfn_to_mfn(pgd); in xen_make_pgd()
454 return native_make_pgd(pgd); in xen_make_pgd()
517 static pgd_t *xen_get_user_pgd(pgd_t *pgd) in xen_get_user_pgd() argument
519 pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK); in xen_get_user_pgd()
520 unsigned offset = pgd - pgd_page; in xen_get_user_pgd()
[all …]
/linux/arch/x86/boot/startup/
H A Dmap_kernel.c95 pgdval_t *pgd; in __startup_64() local
126 pgd = rip_rel_ptr(early_top_pgt); in __startup_64()
127 pgd[pgd_index(__START_KERNEL_map)] += load_delta; in __startup_64()
133 pgd[pgd_index(__START_KERNEL_map)] = (pgdval_t)p4d | _PAGE_TABLE; in __startup_64()
159 pgd[i + 0] = (pgdval_t)p4d + pgtable_flags; in __startup_64()
160 pgd[i + 1] = (pgdval_t)p4d + pgtable_flags; in __startup_64()
167 pgd[i + 0] = (pgdval_t)pud + pgtable_flags; in __startup_64()
168 pgd[i + 1] = (pgdval_t)pud + pgtable_flags; in __startup_64()
/linux/arch/parisc/mm/
H A Dhugetlbpage.c29 pgd_t *pgd; in huge_pte_alloc() local
42 pgd = pgd_offset(mm, addr); in huge_pte_alloc()
43 p4d = p4d_offset(pgd, addr); in huge_pte_alloc()
56 pgd_t *pgd; in huge_pte_offset() local
64 pgd = pgd_offset(mm, addr); in huge_pte_offset()
65 if (!pgd_none(*pgd)) { in huge_pte_offset()
66 p4d = p4d_offset(pgd, addr); in huge_pte_offset()
/linux/arch/x86/include/asm/
H A Dpgtable_64.h140 pgd_t pgd; in native_set_p4d() local
148 pgd = native_make_pgd(native_p4d_val(p4d)); in native_set_p4d()
149 pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd); in native_set_p4d()
150 WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd))); in native_set_p4d()
158 static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) in native_set_pgd() argument
160 WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd)); in native_set_pgd()
163 static inline void native_pgd_clear(pgd_t *pgd) in native_pgd_clear() argument
165 native_set_pgd(pgd, native_make_pgd(0)); in native_pgd_clear()
/linux/arch/riscv/mm/
H A Dfault.c30 pgd_t *pgdp, pgd; in show_pte() local
42 mm == &init_mm ? (u64)__pa_symbol(mm->pgd) : virt_to_phys(mm->pgd)); in show_pte()
45 pgd = pgdp_get(pgdp); in show_pte()
46 pr_alert("[%016lx] pgd=%016lx", addr, pgd_val(pgd)); in show_pte()
47 if (pgd_none(pgd) || pgd_bad(pgd) || pgd_leaf(pgd)) in show_pte()
170 pgd_t *pgd, *pgd_k; in vmalloc_fault() local
192 pgd = (pgd_t *)pfn_to_virt(pfn) + index; in vmalloc_fault()
193 pgd_k = init_mm.pgd + index; in vmalloc_fault()
199 set_pgd(pgd, pgdp_get(pgd_k)); in vmalloc_fault()
/linux/arch/powerpc/include/asm/nohash/
H A Dpgalloc.h22 pgd_t *pgd = kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), in pgd_alloc() local
26 memcpy(pgd + USER_PTRS_PER_PGD, swapper_pg_dir + USER_PTRS_PER_PGD, in pgd_alloc()
29 return pgd; in pgd_alloc()
32 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) in pgd_free() argument
34 kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd); in pgd_free()
/linux/arch/m68k/include/asm/
H A Dmmu_context.h67 static inline void set_context(mm_context_t context, pgd_t *pgd) in set_context() argument
76 set_context(tsk->mm->context, next->pgd); in switch_mm()
88 set_context(mm->context, mm->pgd); in activate_mm()
98 pgd_t *pgd; in load_ksp_mmu() local
124 pgd = pgd_offset(mm, mmuar); in load_ksp_mmu()
125 if (pgd_none(*pgd)) in load_ksp_mmu()
128 p4d = p4d_offset(pgd, mmuar); in load_ksp_mmu()
230 mm->context = virt_to_phys(mm->pgd); in init_new_context()
305 next_mm->context = virt_to_phys(next_mm->pgd); in activate_mm()
/linux/arch/powerpc/include/asm/book3s/32/
H A Dpgalloc.h10 pgd_t *pgd = kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), in pgd_alloc() local
14 memcpy(pgd + USER_PTRS_PER_PGD, swapper_pg_dir + USER_PTRS_PER_PGD, in pgd_alloc()
17 return pgd; in pgd_alloc()
20 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) in pgd_free() argument
22 kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd); in pgd_free()
/linux/arch/microblaze/include/asm/
H A Dmmu_context_mm.h50 extern void set_context(mm_context_t context, pgd_t *pgd);
117 tsk->thread.pgdir = next->pgd; in switch_mm()
119 set_context(next->context, next->pgd); in switch_mm()
130 current->thread.pgdir = mm->pgd; in activate_mm()
132 set_context(mm->context, mm->pgd); in activate_mm()
/linux/arch/alpha/mm/
H A Dfault.c52 pcb->ptbr = ((unsigned long) next_mm->pgd - IDENT_ADDR) >> PAGE_SHIFT; in __load_new_mm_context()
237 pgd_t *pgd, *pgd_k; in do_page_fault() local
239 pgd = current->active_mm->pgd + index; in do_page_fault()
241 if (!pgd_present(*pgd) && pgd_present(*pgd_k)) { in do_page_fault()
242 pgd_val(*pgd) = pgd_val(*pgd_k); in do_page_fault()
/linux/mm/
H A Dpagewalk.c16 * a missing entry found at level 1 (p4d) is actually at level 0 (pgd).
248 static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, in walk_pud_range()
259 p4d = p4d_offset(pgd, addr); in walk_p4d_range()
289 pgd_t *pgd; in walk_pgd_range()
297 if (walk->pgd) in walk_pgd_range()
298 pgd = walk->pgd + pgd_index(addr); in walk_pgd_range()
300 pgd = pgd_offset(walk->mm, addr); in walk_pgd_range()
303 if (pgd_none_or_clear_bad(pgd)) { in walk_pgd_range()
305 err = __p4d_alloc(walk->mm, pgd, add in walk_pgd_range()
250 walk_p4d_range(pgd_t * pgd,unsigned long addr,unsigned long end,struct mm_walk * walk) walk_p4d_range() argument
291 pgd_t *pgd; walk_pgd_range() local
626 walk_kernel_page_table_range(unsigned long start,unsigned long end,const struct mm_walk_ops * ops,pgd_t * pgd,void * private) walk_kernel_page_table_range() argument
649 walk_kernel_page_table_range_lockless(unsigned long start,unsigned long end,const struct mm_walk_ops * ops,pgd_t * pgd,void * private) walk_kernel_page_table_range_lockless() argument
686 walk_page_range_debug(struct mm_struct * mm,unsigned long start,unsigned long end,const struct mm_walk_ops * ops,pgd_t * pgd,void * private) walk_page_range_debug() argument
[all...]
H A Dpgalloc-track.h6 static inline p4d_t *p4d_alloc_track(struct mm_struct *mm, pgd_t *pgd, in p4d_alloc_track() argument
10 if (unlikely(pgd_none(*pgd))) { in p4d_alloc_track()
11 if (__p4d_alloc(mm, pgd, address)) in p4d_alloc_track()
16 return p4d_offset(pgd, address); in p4d_alloc_track()
/linux/include/linux/
H A Dptdump.h18 void (*note_page_pgd)(struct ptdump_state *st, unsigned long addr, pgd_t pgd);
24 void (*effective_prot_pgd)(struct ptdump_state *st, pgd_t pgd);
29 struct mm_struct *mm, pgd_t *pgd,
31 void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd);

12345678