/linux/include/asm-generic/ |
H A D | pgtable-nop4d.h | 9 typedef struct { pgd_t pgd; } p4d_t; member 21 static inline int pgd_none(pgd_t pgd) { return 0; } in pgd_none() argument 22 static inline int pgd_bad(pgd_t pgd) { return 0; } in pgd_bad() argument 23 static inline int pgd_present(pgd_t pgd) { return 1; } in pgd_present() argument 24 static inline void pgd_clear(pgd_t *pgd) { } in pgd_clear() argument 25 #define p4d_ERROR(p4d) (pgd_ERROR((p4d).pgd)) 27 #define pgd_populate(mm, pgd, p4d) do { } while (0) argument 28 #define pgd_populate_safe(mm, pgd, p4d) do { } while (0) argument 35 static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) in p4d_offset() argument 37 return (p4d_t *)pgd; in p4d_offset() [all …]
|
/linux/arch/x86/mm/ |
H A D | kasan_init_64.c | 123 static void __init kasan_populate_pgd(pgd_t *pgd, unsigned long addr, in kasan_populate_pgd() argument 130 if (pgd_none(*pgd)) { in kasan_populate_pgd() 132 pgd_populate(&init_mm, pgd, p); in kasan_populate_pgd() 135 p4d = p4d_offset(pgd, addr); in kasan_populate_pgd() 145 pgd_t *pgd; in kasan_populate_shadow() local 150 pgd = pgd_offset_k(addr); in kasan_populate_shadow() 153 kasan_populate_pgd(pgd, addr, next, nid); in kasan_populate_shadow() 154 } while (pgd++, addr = next, addr != end); in kasan_populate_shadow() 171 pgd_t *pgd; in clear_pgds() local 176 pgd = pgd_offset_k(start); in clear_pgds() [all …]
|
H A D | pgtable.c | 101 static inline void pgd_list_add(pgd_t *pgd) in pgd_list_add() 103 struct ptdesc *ptdesc = virt_to_ptdesc(pgd); in pgd_list_del() argument 108 static inline void pgd_list_del(pgd_t *pgd) in pgd_list_del() 110 struct ptdesc *ptdesc = virt_to_ptdesc(pgd); 121 static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm) in pgd_page_get_mm() 123 virt_to_ptdesc(pgd)->pt_mm = mm; in pgd_page_get_mm() 131 static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd) in pgd_ctor() 133 /* If the pgd points to a shared pagetable level (either the in pgd_ctor() 139 clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY, in pgd_ctor() 146 pgd_set_mm(pgd, m argument 96 pgd_list_add(pgd_t * pgd) pgd_list_add() argument 116 pgd_set_mm(pgd_t * pgd,struct mm_struct * mm) pgd_set_mm() argument 126 pgd_ctor(struct mm_struct * mm,pgd_t * pgd) pgd_ctor() argument 276 pgd_t pgd = *pgdp; mop_up_one_pmd() local 308 pgd_prepopulate_pmd(struct mm_struct * mm,pgd_t * pgd,pmd_t * pmds[]) pgd_prepopulate_pmd() argument 412 _pgd_free(pgd_t * pgd) _pgd_free() argument 427 _pgd_free(pgd_t * pgd) _pgd_free() argument 435 pgd_t *pgd; pgd_alloc() local 487 pgd_free(struct mm_struct * mm,pgd_t * pgd) pgd_free() argument [all...] |
H A D | pti.c | 124 pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd) in __pti_set_user_pgtbl() argument 135 if (!pgdp_maps_userspace(pgdp) || (pgd.pgd & _PAGE_NOPTISHADOW)) in __pti_set_user_pgtbl() 136 return pgd; in __pti_set_user_pgtbl() 142 kernel_to_user_pgdp(pgdp)->pgd = pgd.pgd; in __pti_set_user_pgtbl() 157 if ((pgd.pgd & (_PAGE_USER|_PAGE_PRESENT)) == (_PAGE_USER|_PAGE_PRESENT) && in __pti_set_user_pgtbl() 159 pgd.pgd |= _PAGE_NX; in __pti_set_user_pgtbl() 162 return pgd; in __pti_set_user_pgtbl() 173 pgd_t *pgd = kernel_to_user_pgdp(pgd_offset_k(address)); in pti_user_pagetable_walk_p4d() local 181 if (pgd_none(*pgd)) { in pti_user_pagetable_walk_p4d() 186 set_pgd(pgd, __pgd(_KERNPG_TABLE | __pa(new_p4d_page))); in pti_user_pagetable_walk_p4d() [all …]
|
H A D | ident_map.c | 50 static void free_p4d(struct x86_mapping_info *info, pgd_t *pgd) in free_p4d() argument 52 p4d_t *p4d = p4d_offset(pgd, 0); in free_p4d() 66 void kernel_ident_mapping_free(struct x86_mapping_info *info, pgd_t *pgd) in kernel_ident_mapping_free() argument 71 if (!pgd_present(pgd[i])) in kernel_ident_mapping_free() 74 free_p4d(info, &pgd[i]); in kernel_ident_mapping_free() 77 info->free_pgt_page(pgd, info->context); in kernel_ident_mapping_free() 199 pgd_t *pgd = pgd_page + pgd_index(addr); in kernel_ident_mapping_init() local 206 if (pgd_present(*pgd)) { in kernel_ident_mapping_init() 207 p4d = p4d_offset(pgd, 0); in kernel_ident_mapping_init() 221 set_pgd(pgd, __pgd(__pa(p4d) | info->kernpg_flag | _PAGE_NOPTISHADOW)); in kernel_ident_mapping_init() [all …]
|
H A D | init_32.c | 67 static pmd_t * __init one_md_table_init(pgd_t *pgd) in one_md_table_init() argument 74 if (!(pgd_val(*pgd) & _PAGE_PRESENT)) { in one_md_table_init() 76 set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT)); in one_md_table_init() 77 p4d = p4d_offset(pgd, 0); in one_md_table_init() 84 p4d = p4d_offset(pgd, 0); in one_md_table_init() 208 pgd_t *pgd; in page_table_range_init() local 220 pgd = pgd_base + pgd_idx; in page_table_range_init() 222 for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) { in page_table_range_init() 223 pmd = one_md_table_init(pgd); in page_table_range_init() 260 pgd_t *pgd; in kernel_physical_mapping_init() local [all …]
|
/linux/arch/powerpc/include/asm/book3s/64/ |
H A D | pgalloc.h | 39 static inline void radix__pgd_free(struct mm_struct *mm, pgd_t *pgd) in radix__pgd_free() argument 42 free_page((unsigned long)pgd); in radix__pgd_free() 44 free_pages((unsigned long)pgd, 4); in radix__pgd_free() 50 pgd_t *pgd; in pgd_alloc() local 55 pgd = kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), in pgd_alloc() 57 if (unlikely(!pgd)) in pgd_alloc() 58 return pgd; in pgd_alloc() 65 kmemleak_no_scan(pgd); in pgd_alloc() 76 memset(pgd, 0, PGD_TABLE_SIZE); in pgd_alloc() 78 return pgd; in pgd_alloc() [all …]
|
/linux/arch/arm/mm/ |
H A D | pgd.c | 3 * linux/arch/arm/mm/pgd.c 21 #define _pgd_free(mm, pgd) kfree(pgd) argument 24 #define _pgd_free(mm, pgd) __pgd_free(mm, pgd) argument 142 pgd_t *pgd; in pgd_free() 151 pgd = pgd_base + pgd_index(0); in pgd_free() 152 if (pgd_none_or_clear_bad(pgd)) in pgd_free() 155 p4d = p4d_offset(pgd, 0); in pgd_free() 179 pgd_clear(pgd); in pgd_free() 144 pgd_t *pgd; pgd_free() local [all...] |
/linux/arch/riscv/include/asm/ |
H A D | pgtable-64.h | 343 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) in set_pgd() argument 346 WRITE_ONCE(*pgdp, pgd); in set_pgd() 348 set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) }); in set_pgd() 351 static inline int pgd_none(pgd_t pgd) in pgd_none() argument 354 return (pgd_val(pgd) == 0); in pgd_none() 359 static inline int pgd_present(pgd_t pgd) in pgd_present() argument 362 return (pgd_val(pgd) & _PAGE_PRESENT); in pgd_present() 367 static inline int pgd_bad(pgd_t pgd) in pgd_bad() argument 370 return !pgd_present(pgd); in pgd_bad() 375 static inline void pgd_clear(pgd_t *pgd) in pgd_clear() argument [all …]
|
/linux/arch/x86/kernel/acpi/ |
H A D | madt_wakeup.c | 79 static int __init init_transition_pgtable(pgd_t *pgd) in init_transition_pgtable() argument 89 pgd += pgd_index(vaddr); in init_transition_pgtable() 90 if (!pgd_present(*pgd)) { in init_transition_pgtable() 94 set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE)); in init_transition_pgtable() 96 p4d = p4d_offset(pgd, vaddr); in init_transition_pgtable() 133 pgd_t *pgd; in acpi_mp_setup_reset() local 135 pgd = alloc_pgt_page(NULL); in acpi_mp_setup_reset() 136 if (!pgd) in acpi_mp_setup_reset() 144 if (kernel_ident_mapping_init(&info, pgd, mstart, mend)) { in acpi_mp_setup_reset() 145 kernel_ident_mapping_free(&info, pgd); in acpi_mp_setup_reset() [all …]
|
/linux/arch/x86/power/ |
H A D | hibernate_32.c | 30 static pmd_t *resume_one_md_table_init(pgd_t *pgd) in resume_one_md_table_init() argument 41 set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT)); in resume_one_md_table_init() 42 p4d = p4d_offset(pgd, 0); in resume_one_md_table_init() 47 p4d = p4d_offset(pgd, 0); in resume_one_md_table_init() 84 pgd_t *pgd; in resume_physical_mapping_init() local 90 pgd = pgd_base + pgd_idx; in resume_physical_mapping_init() 93 for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) { in resume_physical_mapping_init() 94 pmd = resume_one_md_table_init(pgd); in resume_physical_mapping_init() 147 pgd_t *pgd; in set_up_temporary_text_mapping() local 151 pgd = pgd_base + pgd_index(restore_jump_address); in set_up_temporary_text_mapping() [all …]
|
H A D | hibernate_64.c | 28 static int set_up_temporary_text_mapping(pgd_t *pgd) in set_up_temporary_text_mapping() argument 77 set_pgd(pgd + pgd_index(restore_jump_address), new_pgd); in set_up_temporary_text_mapping() 81 set_pgd(pgd + pgd_index(restore_jump_address), new_pgd); in set_up_temporary_text_mapping() 100 pgd_t *pgd; in set_up_temporary_mappings() local 104 pgd = (pgd_t *)get_safe_page(GFP_ATOMIC); in set_up_temporary_mappings() 105 if (!pgd) in set_up_temporary_mappings() 109 result = set_up_temporary_text_mapping(pgd); in set_up_temporary_mappings() 118 result = kernel_ident_mapping_init(&info, pgd, mstart, mend); in set_up_temporary_mappings() 123 temp_pgt = __pa(pgd); in set_up_temporary_mappings()
|
/linux/arch/sh/mm/ |
H A D | fault.c | 39 pgd_t *pgd; in show_pte() local 42 pgd = mm->pgd; in show_pte() 44 pgd = get_TTB(); in show_pte() 46 if (unlikely(!pgd)) in show_pte() 47 pgd = swapper_pg_dir; in show_pte() 50 pr_alert("pgd = %p\n", pgd); in show_pte() 51 pgd += pgd_index(addr); in show_pte() 52 pr_alert("[%08lx] *pgd=%0*llx", addr, (u32)(sizeof(*pgd) * 2), in show_pte() 53 (u64)pgd_val(*pgd)); in show_pte() 61 if (pgd_none(*pgd)) in show_pte() [all …]
|
H A D | pgtable.c | 14 pgd_t *pgd = x; in pgd_ctor() local 16 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); in pgd_ctor() 17 memcpy(pgd + USER_PTRS_PER_PGD, in pgd_ctor() 39 void pgd_free(struct mm_struct *mm, pgd_t *pgd) in pgd_free() argument 41 kmem_cache_free(pgd_cachep, pgd); in pgd_free()
|
/linux/arch/parisc/include/asm/ |
H A D | pgalloc.h | 16 /* Allocate the top level pgd (page directory) */ 21 pgd_t *pgd; pgd_alloc() local 32 pgd_free(struct mm_struct * mm,pgd_t * pgd) pgd_free() argument
|
/linux/arch/loongarch/mm/ |
H A D | hugetlbpage.c | 19 pgd_t *pgd; in huge_pte_alloc() local 24 pgd = pgd_offset(mm, addr); in huge_pte_alloc() 25 p4d = p4d_alloc(mm, pgd, addr); in huge_pte_alloc() 36 pgd_t *pgd; in huge_pte_offset() local 41 pgd = pgd_offset(mm, addr); in huge_pte_offset() 42 if (pgd_present(pgdp_get(pgd))) { in huge_pte_offset() 43 p4d = p4d_offset(pgd, addr); in huge_pte_offset()
|
/linux/arch/hexagon/include/asm/ |
H A D | pgalloc.h | 23 pgd_t *pgd; in pgd_alloc() local 25 pgd = __pgd_alloc(mm, 0); in pgd_alloc() 35 memcpy(pgd, swapper_pg_dir, PTRS_PER_PGD*sizeof(pgd_t)); in pgd_alloc() 39 mm->context.ptbase = __pa(pgd); in pgd_alloc() 41 return pgd; in pgd_alloc() 83 pmdindex = (pgd_t *)pmd - mm->pgd; in pmd_populate_kernel() 84 ppmd = (pmd_t *)current->active_mm->pgd + pmdindex; in pmd_populate_kernel()
|
/linux/arch/x86/xen/ |
H A D | mmu_pv.c | 92 pgdval_t xen_pgd_val(pgd_t pgd); 97 pgd_t xen_make_pgd(pgdval_t pgd); 426 __visible pgdval_t xen_pgd_val(pgd_t pgd) in xen_pgd_val() argument 428 return pte_mfn_to_pfn(pgd.pgd); in xen_pgd_val() 440 __visible pgd_t xen_make_pgd(pgdval_t pgd) in xen_make_pgd() argument 442 pgd = pte_pfn_to_mfn(pgd); in xen_make_pgd() 443 return native_make_pgd(pgd); in xen_make_pgd() 506 static pgd_t *xen_get_user_pgd(pgd_t *pgd) in xen_get_user_pgd() argument 508 pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK); in xen_get_user_pgd() 509 unsigned offset = pgd - pgd_page; in xen_get_user_pgd() [all …]
|
/linux/arch/x86/include/asm/ |
H A D | pgtable_64.h | 144 pgd_t pgd; in native_set_p4d() local 152 pgd = native_make_pgd(native_p4d_val(p4d)); in native_set_p4d() 153 pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd); in native_set_p4d() 154 WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd))); in native_set_p4d() 162 static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) in native_set_pgd() argument 164 WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd)); in native_set_pgd() 167 static inline void native_pgd_clear(pgd_t *pgd) in native_pgd_clear() argument 169 native_set_pgd(pgd, native_make_pgd(0)); in native_pgd_clear()
|
H A D | pgtable_types.h | 299 typedef struct { pgdval_t pgd; } pgd_t; member 335 static inline pgdval_t native_pgd_val(pgd_t pgd) in native_pgd_val() argument 337 return pgd.pgd & PGD_ALLOWED_BITS; in native_pgd_val() 340 static inline pgdval_t pgd_flags(pgd_t pgd) in pgd_flags() argument 342 return native_pgd_val(pgd) & PTE_FLAGS_MASK; in pgd_flags() 362 return (p4d_t) { .pgd = native_make_pgd((pgdval_t)val) }; in native_make_p4d() 367 return native_pgd_val(p4d.pgd); in native_p4d_val() 388 return (pud_t) { .p4d.pgd = native_make_pgd(val) }; in native_make_pud() 393 return native_pgd_val(pud.p4d.pgd); in native_pud_val() 412 return (pmd_t) { .pud.p4d.pgd = native_make_pgd(val) }; in native_make_pmd() [all …]
|
/linux/arch/um/kernel/ |
H A D | tlb.c | 138 static inline int update_p4d_range(pgd_t *pgd, unsigned long addr, in update_p4d_range() argument 146 p4d = p4d_offset(pgd, addr); in update_p4d_range() 163 pgd_t *pgd; in um_tlb_sync() local 180 pgd = pgd_offset(mm, addr); in um_tlb_sync() 183 if (!pgd_present(*pgd)) { in um_tlb_sync() 184 if (pgd_needsync(*pgd)) { in um_tlb_sync() 187 pgd_mkuptodate(*pgd); in um_tlb_sync() 190 ret = update_p4d_range(pgd, addr, next, &ops); in um_tlb_sync() 191 } while (pgd++, addr = next, in um_tlb_sync()
|
/linux/arch/parisc/mm/ |
H A D | hugetlbpage.c | 29 pgd_t *pgd; in huge_pte_alloc() local 42 pgd = pgd_offset(mm, addr); in huge_pte_alloc() 43 p4d = p4d_offset(pgd, addr); in huge_pte_alloc() 56 pgd_t *pgd; in huge_pte_offset() local 64 pgd = pgd_offset(mm, addr); in huge_pte_offset() 65 if (!pgd_none(*pgd)) { in huge_pte_offset() 66 p4d = p4d_offset(pgd, addr); in huge_pte_offset()
|
/linux/arch/s390/mm/ |
H A D | vmem.c | 378 static int modify_p4d_table(pgd_t *pgd, unsigned long addr, unsigned long end, in modify_p4d_table() argument 386 p4d = p4d_offset(pgd, addr); in modify_p4d_table() 409 static void try_free_p4d_table(pgd_t *pgd, unsigned long start) in try_free_p4d_table() argument 414 p4d = p4d_offset(pgd, start); in try_free_p4d_table() 419 vmem_free_pages(pgd_deref(*pgd), CRST_ALLOC_ORDER, NULL); in try_free_p4d_table() 420 pgd_clear(pgd); in try_free_p4d_table() 428 pgd_t *pgd; in modify_pagetable() local 438 pgd = pgd_offset_k(addr); in modify_pagetable() 441 if (pgd_none(*pgd)) in modify_pagetable() 443 } else if (pgd_none(*pgd)) { in modify_pagetable() [all …]
|
H A D | pgalloc.c | 77 unsigned long *pgd = NULL, *p4d = NULL, *__pgd; in crst_table_upgrade() local 94 pgd = crst_table_alloc(mm); in crst_table_upgrade() 95 if (unlikely(!pgd)) in crst_table_upgrade() 97 crst_table_init(pgd, _REGION1_ENTRY_EMPTY); in crst_table_upgrade() 98 pagetable_pgd_ctor(virt_to_ptdesc(pgd)); in crst_table_upgrade() 111 __pgd = (unsigned long *) mm->pgd; in crst_table_upgrade() 113 mm->pgd = (pgd_t *) p4d; in crst_table_upgrade() 115 mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | in crst_table_upgrade() 119 if (pgd) { in crst_table_upgrade() 120 __pgd = (unsigned long *) mm->pgd; in crst_table_upgrade() [all …]
|
/linux/arch/m68k/include/asm/ |
H A D | mmu_context.h | 67 static inline void set_context(mm_context_t context, pgd_t *pgd) in set_context() argument 76 set_context(tsk->mm->context, next->pgd); in switch_mm() 88 set_context(mm->context, mm->pgd); in activate_mm() 98 pgd_t *pgd; in load_ksp_mmu() local 124 pgd = pgd_offset(mm, mmuar); in load_ksp_mmu() 125 if (pgd_none(*pgd)) in load_ksp_mmu() 128 p4d = p4d_offset(pgd, mmuar); in load_ksp_mmu() 230 mm->context = virt_to_phys(mm->pgd); in init_new_context() 305 next_mm->context = virt_to_phys(next_mm->pgd); in activate_mm()
|