| /linux/arch/m68k/include/asm/ |
| H A D | mcf_pgalloc.h | 17 struct ptdesc *ptdesc = pagetable_alloc((GFP_DMA | __GFP_ZERO) & in pte_alloc_one_kernel() local
20 if (!ptdesc) in pte_alloc_one_kernel()
22 if (!pagetable_pte_ctor(mm, ptdesc)) { in pte_alloc_one_kernel()
23 pagetable_free(ptdesc); in pte_alloc_one_kernel()
27 return ptdesc_address(ptdesc); in pte_alloc_one_kernel()
42 struct ptdesc *ptdesc = virt_to_ptdesc(pgtable); in __pte_free_tlb() local
44 pagetable_dtor(ptdesc); in __pte_free_tlb()
45 pagetable_free(ptdesc); in __pte_free_tlb()
50 struct ptdesc *ptdesc = pagetable_alloc(GFP_DMA | __GFP_ZERO, 0); in pte_alloc_one() local
53 if (!ptdesc) in pte_alloc_one()
[all …]
|
| /linux/arch/s390/mm/ |
| H A D | pgalloc.c | 20 struct ptdesc *ptdesc; in crst_table_alloc_noprof() local
25 ptdesc = pagetable_alloc_noprof(gfp, CRST_ALLOC_ORDER); in crst_table_alloc_noprof()
26 if (!ptdesc) in crst_table_alloc_noprof()
28 table = ptdesc_address(ptdesc); in crst_table_alloc_noprof()
103 struct ptdesc *ptdesc; in page_table_alloc_noprof() local
108 ptdesc = pagetable_alloc_noprof(gfp, 0); in page_table_alloc_noprof()
109 if (!ptdesc) in page_table_alloc_noprof()
111 if (!pagetable_pte_ctor(mm, ptdesc)) { in page_table_alloc_noprof()
112 pagetable_free(ptdesc); in page_table_alloc_noprof()
115 table = ptdesc_address(ptdesc); in page_table_alloc_noprof()
[all …]
|
| /linux/arch/arm64/include/asm/ |
| H A D | tlb.h | 79 struct ptdesc *ptdesc = page_ptdesc(pte); in __pte_free_tlb() local
81 tlb_remove_ptdesc(tlb, ptdesc); in __pte_free_tlb()
88 struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); in __pmd_free_tlb() local
90 tlb_remove_ptdesc(tlb, ptdesc); in __pmd_free_tlb()
98 struct ptdesc *ptdesc = virt_to_ptdesc(pudp); in __pud_free_tlb() local
103 tlb_remove_ptdesc(tlb, ptdesc); in __pud_free_tlb()
111 struct ptdesc *ptdesc = virt_to_ptdesc(p4dp); in __p4d_free_tlb() local
116 tlb_remove_ptdesc(tlb, ptdesc); in __p4d_free_tlb()
|
| /linux/arch/parisc/include/asm/ |
| H A D | pgalloc.h | 34 struct ptdesc *ptdesc; in pmd_alloc_one() local
39 ptdesc = pagetable_alloc(gfp, PMD_TABLE_ORDER); in pmd_alloc_one()
40 if (!ptdesc) in pmd_alloc_one()
42 if (!pagetable_pmd_ctor(mm, ptdesc)) { in pmd_alloc_one()
43 pagetable_free(ptdesc); in pmd_alloc_one()
46 return ptdesc_address(ptdesc); in pmd_alloc_one()
|
| /linux/include/linux/ |
| H A D | mm_types.h | 554 * struct ptdesc - Memory descriptor for page tables.
561 * @pmd_huge_pte: Protected by ptdesc->ptl, used for THPs.
576 struct ptdesc { struct
614 static_assert(offsetof(struct page, pg) == offsetof(struct ptdesc, pt)) argument
627 static_assert(sizeof(struct ptdesc) <= sizeof(struct page));
630 const struct ptdesc *: (const struct page *)(pt), \
631 struct ptdesc *: (struct page *)(pt)))
634 const struct ptdesc *: (const struct folio *)(pt), \
635 struct ptdesc *: (struct folio *)(pt)))
638 const struct page *: (const struct ptdesc *)(
577 pt_flagsptdesc global() argument
579 __anona82e97cc1a0aptdesc global() argument
587 __page_mappingptdesc global() argument
589 __anona82e97cc1c0aptdesc global() argument
642 ptdesc_pmd_pts_init(struct ptdesc * ptdesc) ptdesc_pmd_pts_init() argument
647 ptdesc_pmd_pts_inc(struct ptdesc * ptdesc) ptdesc_pmd_pts_inc() argument
652 ptdesc_pmd_pts_dec(struct ptdesc * ptdesc) ptdesc_pmd_pts_dec() argument
657 ptdesc_pmd_pts_count(const struct ptdesc * ptdesc) ptdesc_pmd_pts_count() argument
662 ptdesc_pmd_is_shared(struct ptdesc * ptdesc) ptdesc_pmd_is_shared() argument
667 ptdesc_pmd_pts_init(struct ptdesc * ptdesc) ptdesc_pmd_pts_init() argument
[all...] |
| H A D | mm.h | 3721 static inline struct ptdesc *virt_to_ptdesc(const void *x) in virt_to_ptdesc()
3732 static inline void *ptdesc_address(const struct ptdesc *pt) in ptdesc_address()
3737 static inline bool pagetable_is_reserved(struct ptdesc *pt) in pagetable_is_reserved()
3749 static inline void ptdesc_set_kernel(struct ptdesc *ptdesc) in ptdesc_set_kernel() argument
3751 set_bit(PT_kernel, &ptdesc->pt_flags.f); in ptdesc_set_kernel()
3761 static inline void ptdesc_clear_kernel(struct ptdesc *ptdesc) in ptdesc_clear_kernel() argument
3768 clear_bit(PT_kernel, &ptdesc->pt_flags.f); in ptdesc_clear_kernel()
3777 static inline bool ptdesc_test_kernel(const struct ptdesc *ptdesc) in ptdesc_test_kernel() argument
3779 return test_bit(PT_kernel, &ptdesc->pt_flags.f); in ptdesc_test_kernel()
3792 static inline struct ptdesc *pagetable_alloc_noprof(gfp_t gfp, unsigned int order) in pagetable_alloc_noprof()
[all …]
|
| /linux/arch/powerpc/mm/book3s64/ |
| H A D | mmu_context.c | 245 struct ptdesc *ptdesc; in pmd_frag_destroy() local
247 ptdesc = virt_to_ptdesc(pmd_frag); in pmd_frag_destroy()
251 if (atomic_sub_and_test(PMD_FRAG_NR - count, &ptdesc->pt_frag_refcount)) { in pmd_frag_destroy()
252 pagetable_dtor(ptdesc); in pmd_frag_destroy()
253 pagetable_free(ptdesc); in pmd_frag_destroy()
|
| /linux/arch/riscv/mm/ |
| H A D | init.c | 425 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL, 0); in alloc_pte_late() local
432 BUG_ON(!ptdesc || !pagetable_pte_ctor(NULL, ptdesc)); in alloc_pte_late()
433 return __pa((pte_t *)ptdesc_address(ptdesc)); in alloc_pte_late()
492 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL, 0); in alloc_pmd_late() local
495 BUG_ON(!ptdesc || !pagetable_pmd_ctor(NULL, ptdesc)); in alloc_pmd_late()
496 return __pa((pmd_t *)ptdesc_address(ptdesc)); in alloc_pmd_late()
557 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL, 0); in alloc_pud_late() local
559 BUG_ON(!ptdesc); in alloc_pud_late()
560 pagetable_pud_ctor(ptdesc); in alloc_pud_late()
561 return __pa((pud_t *)ptdesc_address(ptdesc)); in alloc_pud_late()
[all …]
|
| /linux/arch/x86/mm/pat/ |
| H A D | set_memory.c | 417 struct ptdesc *ptdesc, *tmp; in __cpa_collapse_large_pages() local
442 list_for_each_entry_safe(ptdesc, tmp, &pgtables, pt_list) { in __cpa_collapse_large_pages()
443 list_del(&ptdesc->pt_list); in __cpa_collapse_large_pages()
449 if (PageTable((ptdesc_page(ptdesc)))) in __cpa_collapse_large_pages()
450 pagetable_dtor_free(ptdesc); in __cpa_collapse_large_pages()
452 pagetable_free(ptdesc); in __cpa_collapse_large_pages()
916 struct ptdesc *ptdesc; in __set_pmd_pte() local
918 list_for_each_entry(ptdesc, &pgd_list, pt_list) { in __set_pmd_pte()
924 pgd = (pgd_t *)ptdesc_address(ptdesc) + pgd_index(address); in __set_pmd_pte()
1315 struct ptdesc *ptdesc; in collapse_pmd_page() local
[all …]
|
| /linux/arch/x86/xen/ |
| H A D | mmu_pv.c | 839 struct ptdesc *ptdesc; in xen_mm_pin_all() local
844 list_for_each_entry(ptdesc, &pgd_list, pt_list) { in xen_mm_pin_all()
845 if (!PagePinned(ptdesc_page(ptdesc))) { in xen_mm_pin_all()
846 __xen_pgd_pin(&init_mm, (pgd_t *)ptdesc_address(ptdesc)); in xen_mm_pin_all()
847 SetPageSavePinned(ptdesc_page(ptdesc)); in xen_mm_pin_all()
950 struct ptdesc *ptdesc; in xen_mm_unpin_all() local
955 list_for_each_entry(ptdesc, &pgd_list, pt_list) { in xen_mm_unpin_all()
956 if (PageSavePinned(ptdesc_page(ptdesc))) { in xen_mm_unpin_all()
957 BUG_ON(!PagePinned(ptdesc_page(ptdesc))); in xen_mm_unpin_all()
958 __xen_pgd_unpin(&init_mm, (pgd_t *)ptdesc_address(ptdesc)); in xen_mm_unpin_all()
[all …]
|
| /linux/arch/arm64/mm/ |
| H A D | mmu.c | 548 struct ptdesc *ptdesc = pagetable_alloc(gfp & ~__GFP_ZERO, 0); in __pgd_pgtable_alloc() local
551 if (!ptdesc) in __pgd_pgtable_alloc()
554 pa = page_to_phys(ptdesc_page(ptdesc)); in __pgd_pgtable_alloc()
558 BUG_ON(!pagetable_pte_ctor(mm, ptdesc)); in __pgd_pgtable_alloc()
561 BUG_ON(!pagetable_pmd_ctor(mm, ptdesc)); in __pgd_pgtable_alloc()
564 pagetable_pud_ctor(ptdesc); in __pgd_pgtable_alloc()
567 pagetable_p4d_ctor(ptdesc); in __pgd_pgtable_alloc()
|
| /linux/mm/ |
| H A D | memory.c | 7717 bool ptlock_alloc(struct ptdesc *ptdesc) in ptlock_alloc() argument
7724 ptdesc->ptl = ptl; in ptlock_alloc()
7728 void ptlock_free(struct ptdesc *ptdesc) in ptlock_free() argument
7730 if (ptdesc->ptl) in ptlock_free()
7731 kmem_cache_free(page_ptl_cachep, ptdesc->ptl); in ptlock_free()
|
| /linux/arch/x86/include/asm/ |
| H A D | pgtable.h | 53 struct mm_struct *pgd_page_get_mm(struct ptdesc *pt);
|