28 bool __init __early_make_pgtable(unsigned long address, pmdval_t pmd);
73 #define set_pmd(pmdp, pmd)		native_set_pmd(pmdp, pmd)  argument
97 #define pmd_clear(pmd)			native_pmd_clear(pmd)  argument
123 static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)  in pmd_set_flags()  argument
125 	pmdval_t v = native_pmd_val(pmd);  in pmd_set_flags()
130 static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear)  in pmd_clear_flags()  argument
132 	pmdval_t v = native_pmd_val(pmd);  in pmd_clear_flags()
177 static inline bool pmd_dirty(pmd_t pmd)  in pmd_dirty()  argument
179 	return pmd_flags(pmd) & _PAGE_DIRTY_BITS;  in pmd_dirty()
182 static inline bool pmd_shstk(pmd_t pmd)  in pmd_shstk()  argument
185 	       (pmd_flags(pmd) & (_PAGE_RW | _PAGE_DIRTY | _PAGE_PSE)) ==  in pmd_shstk()
190 static inline int pmd_young(pmd_t pmd)  in pmd_young()  argument
192 	return pmd_flags(pmd) & _PAGE_ACCESSED;  in pmd_young()
222 static inline int pmd_write(pmd_t pmd)  in pmd_write()  argument
228 	return (pmd_flags(pmd) & _PAGE_RW) || pmd_shstk(pmd);  in pmd_write()
270 static inline unsigned long pmd_pfn(pmd_t pmd)  in pmd_pfn()  argument
272 	phys_addr_t pfn = pmd_val(pmd);  in pmd_pfn()
274 	return (pfn & pmd_pfn_mask(pmd)) >> PAGE_SHIFT;  in pmd_pfn()
304 static inline int pmd_trans_huge(pmd_t pmd)  in pmd_trans_huge()  argument
306 	return (pmd_val(pmd) & _PAGE_PSE) == _PAGE_PSE;  in pmd_trans_huge()
323 static inline bool pmd_special(pmd_t pmd)  in pmd_special()  argument
325 	return pmd_flags(pmd) & _PAGE_SPECIAL;  in pmd_special()
328 static inline pmd_t pmd_mkspecial(pmd_t pmd)  in pmd_mkspecial()  argument
330 	return pmd_set_flags(pmd, _PAGE_SPECIAL);  in pmd_mkspecial()
506 static inline pmd_t pmd_mksaveddirty(pmd_t pmd)  in pmd_mksaveddirty()  argument
508 	pmdval_t v = native_pmd_val(pmd);  in pmd_mksaveddirty()
515 static inline pmd_t pmd_clear_saveddirty(pmd_t pmd)  in pmd_clear_saveddirty()  argument
517 	pmdval_t v = native_pmd_val(pmd);  in pmd_clear_saveddirty()
523 static inline pmd_t pmd_wrprotect(pmd_t pmd)  in pmd_wrprotect()  argument
525 	pmd = pmd_clear_flags(pmd, _PAGE_RW);  in pmd_wrprotect()
529 	 * a shadow stack PMD (RW=0, Dirty=1). Move the hardware  in pmd_wrprotect()
532 	return pmd_mksaveddirty(pmd);  in pmd_wrprotect()
536 static inline int pmd_uffd_wp(pmd_t pmd)  in pmd_uffd_wp()  argument
538 	return pmd_flags(pmd) & _PAGE_UFFD_WP;  in pmd_uffd_wp()
541 static inline pmd_t pmd_mkuffd_wp(pmd_t pmd)  in pmd_mkuffd_wp()  argument
543 	return pmd_wrprotect(pmd_set_flags(pmd, _PAGE_UFFD_WP));  in pmd_mkuffd_wp()
546 static inline pmd_t pmd_clear_uffd_wp(pmd_t pmd)  in pmd_clear_uffd_wp()  argument
548 	return pmd_clear_flags(pmd, _PAGE_UFFD_WP);  in pmd_clear_uffd_wp()
552 static inline pmd_t pmd_mkold(pmd_t pmd)  in pmd_mkold()  argument
554 	return pmd_clear_flags(pmd, _PAGE_ACCESSED);  in pmd_mkold()
557 static inline pmd_t pmd_mkclean(pmd_t pmd)  in pmd_mkclean()  argument
559 	return pmd_clear_flags(pmd, _PAGE_DIRTY_BITS);  in pmd_mkclean()
562 static inline pmd_t pmd_mkdirty(pmd_t pmd)  in pmd_mkdirty()  argument
564 	pmd = pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);  in pmd_mkdirty()
566 	return pmd_mksaveddirty(pmd);  in pmd_mkdirty()
569 static inline pmd_t pmd_mkwrite_shstk(pmd_t pmd)  in pmd_mkwrite_shstk()  argument
571 	pmd = pmd_clear_flags(pmd, _PAGE_RW);  in pmd_mkwrite_shstk()
573 	return pmd_set_flags(pmd, _PAGE_DIRTY);  in pmd_mkwrite_shstk()
576 static inline pmd_t pmd_mkhuge(pmd_t pmd)  in pmd_mkhuge()  argument
578 	return pmd_set_flags(pmd, _PAGE_PSE);  in pmd_mkhuge()
581 static inline pmd_t pmd_mkyoung(pmd_t pmd)  in pmd_mkyoung()  argument
583 	return pmd_set_flags(pmd, _PAGE_ACCESSED);  in pmd_mkyoung()
586 static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)  in pmd_mkwrite_novma()  argument
588 	return pmd_set_flags(pmd, _PAGE_RW);  in pmd_mkwrite_novma()
591 pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
664 static inline int pmd_soft_dirty(pmd_t pmd)  in pmd_soft_dirty()  argument
666 	return pmd_flags(pmd) & _PAGE_SOFT_DIRTY;  in pmd_soft_dirty()
679 static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)  in pmd_mksoft_dirty()  argument
681 	return pmd_set_flags(pmd, _PAGE_SOFT_DIRTY);  in pmd_mksoft_dirty()
694 static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)  in pmd_clear_soft_dirty()  argument
696 	return pmd_clear_flags(pmd, _PAGE_SOFT_DIRTY);  in pmd_clear_soft_dirty()
764 static inline pmd_t pmd_mkinvalid(pmd_t pmd)  in pmd_mkinvalid()  argument
766 	return pfn_pmd(pmd_pfn(pmd),  in pmd_mkinvalid()
767 		      __pgprot(pmd_flags(pmd) & ~(_PAGE_PRESENT|_PAGE_PROTNONE)));  in pmd_mkinvalid()
811 static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)  in pmd_modify()  argument
813 	pmdval_t val = pmd_val(pmd), oldval = val;  in pmd_modify()
823 	 * Avoid creating shadow stack PMD by accident.  See comment in  in pmd_modify()
984 static inline int pmd_present(pmd_t pmd)  in pmd_present()  argument
992 	return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE);  in pmd_present()
1006 static inline int pmd_protnone(pmd_t pmd)  in pmd_protnone()  argument
1008 	return (pmd_flags(pmd) & (_PAGE_PROTNONE | _PAGE_PRESENT))  in pmd_protnone()
1013 static inline int pmd_none(pmd_t pmd)  in pmd_none()  argument
1017 	unsigned long val = native_pmd_val(pmd);  in pmd_none()
1021 static inline unsigned long pmd_page_vaddr(pmd_t pmd)  in pmd_page_vaddr()  argument
1023 	return (unsigned long)__va(pmd_val(pmd) & pmd_pfn_mask(pmd));  in pmd_page_vaddr()
1030 #define pmd_page(pmd)	pfn_to_page(pmd_pfn(pmd))  argument
1032 static inline int pmd_bad(pmd_t pmd)  in pmd_bad()  argument
1034 	return (pmd_flags(pmd) & ~(_PAGE_USER | _PAGE_ACCESSED)) !=  in pmd_bad()
1214 			      pmd_t *pmdp, pmd_t pmd)  in set_pmd_at()  argument
1216 	page_table_check_pmd_set(mm, pmdp, pmd);  in set_pmd_at()
1217 	set_pmd(pmdp, pmd);  in set_pmd_at()
1319 	pmd_t pmd = native_pmdp_get_and_clear(pmdp);  in pmdp_huge_get_and_clear()  local
1321 	page_table_check_pmd_clear(mm, pmd);  in pmdp_huge_get_and_clear()
1323 	return pmd;  in pmdp_huge_get_and_clear()
1357 		unsigned long address, pmd_t *pmdp, pmd_t pmd)  in pmdp_establish()
1359 	page_table_check_pmd_set(vma->vm_mm, pmdp, pmd);  in pmdp_establish()
1361 		return xchg(pmdp, pmd);  in pmdp_establish()
1364 		WRITE_ONCE(*pmdp, pmd);  in pmdp_establish()
1505 		unsigned long addr, pmd_t *pmd)  in update_mmu_cache_pmd()  argument
1544 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)  in pmd_swp_mksoft_dirty()  argument
1546 	return pmd_set_flags(pmd, _PAGE_SWP_SOFT_DIRTY);  in pmd_swp_mksoft_dirty()
1549 static inline int pmd_swp_soft_dirty(pmd_t pmd)  in pmd_swp_soft_dirty()  argument
1551 	return pmd_flags(pmd) & _PAGE_SWP_SOFT_DIRTY;  in pmd_swp_soft_dirty()
1554 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)  in pmd_swp_clear_soft_dirty()  argument
1556 	return pmd_clear_flags(pmd, _PAGE_SWP_SOFT_DIRTY);  in pmd_swp_clear_soft_dirty()
1577 static inline pmd_t pmd_swp_mkuffd_wp(pmd_t pmd)  in pmd_swp_mkuffd_wp()  argument
1579 	return pmd_set_flags(pmd, _PAGE_SWP_UFFD_WP);  in pmd_swp_mkuffd_wp()
1582 static inline int pmd_swp_uffd_wp(pmd_t pmd)  in pmd_swp_uffd_wp()  argument
1584 	return pmd_flags(pmd) & _PAGE_SWP_UFFD_WP;  in pmd_swp_uffd_wp()
1587 static inline pmd_t pmd_swp_clear_uffd_wp(pmd_t pmd)  in pmd_swp_clear_uffd_wp()  argument
1589 	return pmd_clear_flags(pmd, _PAGE_SWP_UFFD_WP);  in pmd_swp_clear_uffd_wp()
1616  * 'pteval' can come from a PTE, PMD or PUD.  We only check
1645 static inline bool pmd_access_permitted(pmd_t pmd, bool write)  in pmd_access_permitted()  argument
1647 	return __pte_access_permitted(pmd_val(pmd), write);  in pmd_access_permitted()
1668 void arch_check_zapped_pmd(struct vm_area_struct *vma, pmd_t pmd);
1687 static inline bool pmd_user_accessible_page(pmd_t pmd)  in pmd_user_accessible_page()  argument
1689 	return pmd_leaf(pmd) && (pmd_val(pmd) & _PAGE_PRESENT) && (pmd_val(pmd) & _PAGE_USER);  in pmd_user_accessible_page()
1720 #define set_pmd_safe(pmdp, pmd) \  argument
1722 	WARN_ON_ONCE(pmd_present(*pmdp) && !pmd_same(*pmdp, pmd)); \
1723 	set_pmd(pmdp, pmd); \