1df4e817bSPasha Tatashin // SPDX-License-Identifier: GPL-2.0 2df4e817bSPasha Tatashin 3df4e817bSPasha Tatashin /* 4df4e817bSPasha Tatashin * Copyright (c) 2021, Google LLC. 5df4e817bSPasha Tatashin * Pasha Tatashin <pasha.tatashin@soleen.com> 6df4e817bSPasha Tatashin */ 7df4e817bSPasha Tatashin #include <linux/mm.h> 8df4e817bSPasha Tatashin #include <linux/page_table_check.h> 9df4e817bSPasha Tatashin 10df4e817bSPasha Tatashin #undef pr_fmt 11df4e817bSPasha Tatashin #define pr_fmt(fmt) "page_table_check: " fmt 12df4e817bSPasha Tatashin 13df4e817bSPasha Tatashin struct page_table_check { 14df4e817bSPasha Tatashin atomic_t anon_map_count; 15df4e817bSPasha Tatashin atomic_t file_map_count; 16df4e817bSPasha Tatashin }; 17df4e817bSPasha Tatashin 18df4e817bSPasha Tatashin static bool __page_table_check_enabled __initdata = 19df4e817bSPasha Tatashin IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED); 20df4e817bSPasha Tatashin 21df4e817bSPasha Tatashin DEFINE_STATIC_KEY_TRUE(page_table_check_disabled); 22df4e817bSPasha Tatashin EXPORT_SYMBOL(page_table_check_disabled); 23df4e817bSPasha Tatashin 24df4e817bSPasha Tatashin static int __init early_page_table_check_param(char *buf) 25df4e817bSPasha Tatashin { 26597da28eSDr. David Alan Gilbert return strtobool(buf, &__page_table_check_enabled); 27df4e817bSPasha Tatashin } 28df4e817bSPasha Tatashin 29df4e817bSPasha Tatashin early_param("page_table_check", early_page_table_check_param); 30df4e817bSPasha Tatashin 31df4e817bSPasha Tatashin static bool __init need_page_table_check(void) 32df4e817bSPasha Tatashin { 33df4e817bSPasha Tatashin return __page_table_check_enabled; 34df4e817bSPasha Tatashin } 35df4e817bSPasha Tatashin 36df4e817bSPasha Tatashin static void __init init_page_table_check(void) 37df4e817bSPasha Tatashin { 38df4e817bSPasha Tatashin if (!__page_table_check_enabled) 39df4e817bSPasha Tatashin return; 40df4e817bSPasha Tatashin static_branch_disable(&page_table_check_disabled); 41df4e817bSPasha Tatashin } 42df4e817bSPasha Tatashin 43df4e817bSPasha Tatashin struct page_ext_operations page_table_check_ops = { 44df4e817bSPasha Tatashin .size = sizeof(struct page_table_check), 45df4e817bSPasha Tatashin .need = need_page_table_check, 46df4e817bSPasha Tatashin .init = init_page_table_check, 47df4e817bSPasha Tatashin }; 48df4e817bSPasha Tatashin 49df4e817bSPasha Tatashin static struct page_table_check *get_page_table_check(struct page_ext *page_ext) 50df4e817bSPasha Tatashin { 51df4e817bSPasha Tatashin BUG_ON(!page_ext); 52df4e817bSPasha Tatashin return (void *)(page_ext) + page_table_check_ops.offset; 53df4e817bSPasha Tatashin } 54df4e817bSPasha Tatashin 55df4e817bSPasha Tatashin static inline bool pte_user_accessible_page(pte_t pte) 56df4e817bSPasha Tatashin { 57df4e817bSPasha Tatashin return (pte_val(pte) & _PAGE_PRESENT) && (pte_val(pte) & _PAGE_USER); 58df4e817bSPasha Tatashin } 59df4e817bSPasha Tatashin 60df4e817bSPasha Tatashin static inline bool pmd_user_accessible_page(pmd_t pmd) 61df4e817bSPasha Tatashin { 62df4e817bSPasha Tatashin return pmd_leaf(pmd) && (pmd_val(pmd) & _PAGE_PRESENT) && 63df4e817bSPasha Tatashin (pmd_val(pmd) & _PAGE_USER); 64df4e817bSPasha Tatashin } 65df4e817bSPasha Tatashin 66df4e817bSPasha Tatashin static inline bool pud_user_accessible_page(pud_t pud) 67df4e817bSPasha Tatashin { 68df4e817bSPasha Tatashin return pud_leaf(pud) && (pud_val(pud) & _PAGE_PRESENT) && 69df4e817bSPasha Tatashin (pud_val(pud) & _PAGE_USER); 70df4e817bSPasha Tatashin } 71df4e817bSPasha Tatashin 72df4e817bSPasha Tatashin /* 73df4e817bSPasha Tatashin * An enty is removed from the page table, decrement the counters for that page 74df4e817bSPasha Tatashin * verify that it is of correct type and counters do not become negative. 75df4e817bSPasha Tatashin */ 76df4e817bSPasha Tatashin static void page_table_check_clear(struct mm_struct *mm, unsigned long addr, 77df4e817bSPasha Tatashin unsigned long pfn, unsigned long pgcnt) 78df4e817bSPasha Tatashin { 79df4e817bSPasha Tatashin struct page_ext *page_ext; 80df4e817bSPasha Tatashin struct page *page; 8164d8b9e1SPasha Tatashin unsigned long i; 82df4e817bSPasha Tatashin bool anon; 83df4e817bSPasha Tatashin 84df4e817bSPasha Tatashin if (!pfn_valid(pfn)) 85df4e817bSPasha Tatashin return; 86df4e817bSPasha Tatashin 87df4e817bSPasha Tatashin page = pfn_to_page(pfn); 88df4e817bSPasha Tatashin page_ext = lookup_page_ext(page); 89df4e817bSPasha Tatashin anon = PageAnon(page); 90df4e817bSPasha Tatashin 91df4e817bSPasha Tatashin for (i = 0; i < pgcnt; i++) { 92df4e817bSPasha Tatashin struct page_table_check *ptc = get_page_table_check(page_ext); 93df4e817bSPasha Tatashin 94df4e817bSPasha Tatashin if (anon) { 95df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->file_map_count)); 96df4e817bSPasha Tatashin BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0); 97df4e817bSPasha Tatashin } else { 98df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->anon_map_count)); 99df4e817bSPasha Tatashin BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0); 100df4e817bSPasha Tatashin } 101df4e817bSPasha Tatashin page_ext = page_ext_next(page_ext); 102df4e817bSPasha Tatashin } 103df4e817bSPasha Tatashin } 104df4e817bSPasha Tatashin 105df4e817bSPasha Tatashin /* 106df4e817bSPasha Tatashin * A new enty is added to the page table, increment the counters for that page 107df4e817bSPasha Tatashin * verify that it is of correct type and is not being mapped with a different 108df4e817bSPasha Tatashin * type to a different process. 109df4e817bSPasha Tatashin */ 110df4e817bSPasha Tatashin static void page_table_check_set(struct mm_struct *mm, unsigned long addr, 111df4e817bSPasha Tatashin unsigned long pfn, unsigned long pgcnt, 112df4e817bSPasha Tatashin bool rw) 113df4e817bSPasha Tatashin { 114df4e817bSPasha Tatashin struct page_ext *page_ext; 115df4e817bSPasha Tatashin struct page *page; 11664d8b9e1SPasha Tatashin unsigned long i; 117df4e817bSPasha Tatashin bool anon; 118df4e817bSPasha Tatashin 119df4e817bSPasha Tatashin if (!pfn_valid(pfn)) 120df4e817bSPasha Tatashin return; 121df4e817bSPasha Tatashin 122df4e817bSPasha Tatashin page = pfn_to_page(pfn); 123df4e817bSPasha Tatashin page_ext = lookup_page_ext(page); 124df4e817bSPasha Tatashin anon = PageAnon(page); 125df4e817bSPasha Tatashin 126df4e817bSPasha Tatashin for (i = 0; i < pgcnt; i++) { 127df4e817bSPasha Tatashin struct page_table_check *ptc = get_page_table_check(page_ext); 128df4e817bSPasha Tatashin 129df4e817bSPasha Tatashin if (anon) { 130df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->file_map_count)); 131df4e817bSPasha Tatashin BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw); 132df4e817bSPasha Tatashin } else { 133df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->anon_map_count)); 134df4e817bSPasha Tatashin BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0); 135df4e817bSPasha Tatashin } 136df4e817bSPasha Tatashin page_ext = page_ext_next(page_ext); 137df4e817bSPasha Tatashin } 138df4e817bSPasha Tatashin } 139df4e817bSPasha Tatashin 140df4e817bSPasha Tatashin /* 141df4e817bSPasha Tatashin * page is on free list, or is being allocated, verify that counters are zeroes 142df4e817bSPasha Tatashin * crash if they are not. 143df4e817bSPasha Tatashin */ 144df4e817bSPasha Tatashin void __page_table_check_zero(struct page *page, unsigned int order) 145df4e817bSPasha Tatashin { 146df4e817bSPasha Tatashin struct page_ext *page_ext = lookup_page_ext(page); 14764d8b9e1SPasha Tatashin unsigned long i; 148df4e817bSPasha Tatashin 149df4e817bSPasha Tatashin BUG_ON(!page_ext); 15064d8b9e1SPasha Tatashin for (i = 0; i < (1ul << order); i++) { 151df4e817bSPasha Tatashin struct page_table_check *ptc = get_page_table_check(page_ext); 152df4e817bSPasha Tatashin 153df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->anon_map_count)); 154df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->file_map_count)); 155df4e817bSPasha Tatashin page_ext = page_ext_next(page_ext); 156df4e817bSPasha Tatashin } 157df4e817bSPasha Tatashin } 158df4e817bSPasha Tatashin 159df4e817bSPasha Tatashin void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, 160df4e817bSPasha Tatashin pte_t pte) 161df4e817bSPasha Tatashin { 162df4e817bSPasha Tatashin if (&init_mm == mm) 163df4e817bSPasha Tatashin return; 164df4e817bSPasha Tatashin 165df4e817bSPasha Tatashin if (pte_user_accessible_page(pte)) { 166df4e817bSPasha Tatashin page_table_check_clear(mm, addr, pte_pfn(pte), 167df4e817bSPasha Tatashin PAGE_SIZE >> PAGE_SHIFT); 168df4e817bSPasha Tatashin } 169df4e817bSPasha Tatashin } 170df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pte_clear); 171df4e817bSPasha Tatashin 172df4e817bSPasha Tatashin void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr, 173df4e817bSPasha Tatashin pmd_t pmd) 174df4e817bSPasha Tatashin { 175df4e817bSPasha Tatashin if (&init_mm == mm) 176df4e817bSPasha Tatashin return; 177df4e817bSPasha Tatashin 178df4e817bSPasha Tatashin if (pmd_user_accessible_page(pmd)) { 179df4e817bSPasha Tatashin page_table_check_clear(mm, addr, pmd_pfn(pmd), 180df4e817bSPasha Tatashin PMD_PAGE_SIZE >> PAGE_SHIFT); 181df4e817bSPasha Tatashin } 182df4e817bSPasha Tatashin } 183df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pmd_clear); 184df4e817bSPasha Tatashin 185df4e817bSPasha Tatashin void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr, 186df4e817bSPasha Tatashin pud_t pud) 187df4e817bSPasha Tatashin { 188df4e817bSPasha Tatashin if (&init_mm == mm) 189df4e817bSPasha Tatashin return; 190df4e817bSPasha Tatashin 191df4e817bSPasha Tatashin if (pud_user_accessible_page(pud)) { 192df4e817bSPasha Tatashin page_table_check_clear(mm, addr, pud_pfn(pud), 193df4e817bSPasha Tatashin PUD_PAGE_SIZE >> PAGE_SHIFT); 194df4e817bSPasha Tatashin } 195df4e817bSPasha Tatashin } 196df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pud_clear); 197df4e817bSPasha Tatashin 198df4e817bSPasha Tatashin void __page_table_check_pte_set(struct mm_struct *mm, unsigned long addr, 199df4e817bSPasha Tatashin pte_t *ptep, pte_t pte) 200df4e817bSPasha Tatashin { 201df4e817bSPasha Tatashin if (&init_mm == mm) 202df4e817bSPasha Tatashin return; 203df4e817bSPasha Tatashin 20464d8b9e1SPasha Tatashin __page_table_check_pte_clear(mm, addr, *ptep); 205df4e817bSPasha Tatashin if (pte_user_accessible_page(pte)) { 206df4e817bSPasha Tatashin page_table_check_set(mm, addr, pte_pfn(pte), 207df4e817bSPasha Tatashin PAGE_SIZE >> PAGE_SHIFT, 208df4e817bSPasha Tatashin pte_write(pte)); 209df4e817bSPasha Tatashin } 210df4e817bSPasha Tatashin } 211df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pte_set); 212df4e817bSPasha Tatashin 213df4e817bSPasha Tatashin void __page_table_check_pmd_set(struct mm_struct *mm, unsigned long addr, 214df4e817bSPasha Tatashin pmd_t *pmdp, pmd_t pmd) 215df4e817bSPasha Tatashin { 216df4e817bSPasha Tatashin if (&init_mm == mm) 217df4e817bSPasha Tatashin return; 218df4e817bSPasha Tatashin 21964d8b9e1SPasha Tatashin __page_table_check_pmd_clear(mm, addr, *pmdp); 220df4e817bSPasha Tatashin if (pmd_user_accessible_page(pmd)) { 221df4e817bSPasha Tatashin page_table_check_set(mm, addr, pmd_pfn(pmd), 222df4e817bSPasha Tatashin PMD_PAGE_SIZE >> PAGE_SHIFT, 223df4e817bSPasha Tatashin pmd_write(pmd)); 224df4e817bSPasha Tatashin } 225df4e817bSPasha Tatashin } 226df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pmd_set); 227df4e817bSPasha Tatashin 228df4e817bSPasha Tatashin void __page_table_check_pud_set(struct mm_struct *mm, unsigned long addr, 229df4e817bSPasha Tatashin pud_t *pudp, pud_t pud) 230df4e817bSPasha Tatashin { 231df4e817bSPasha Tatashin if (&init_mm == mm) 232df4e817bSPasha Tatashin return; 233df4e817bSPasha Tatashin 23464d8b9e1SPasha Tatashin __page_table_check_pud_clear(mm, addr, *pudp); 235df4e817bSPasha Tatashin if (pud_user_accessible_page(pud)) { 236df4e817bSPasha Tatashin page_table_check_set(mm, addr, pud_pfn(pud), 237df4e817bSPasha Tatashin PUD_PAGE_SIZE >> PAGE_SHIFT, 238df4e817bSPasha Tatashin pud_write(pud)); 239df4e817bSPasha Tatashin } 240df4e817bSPasha Tatashin } 241df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pud_set); 24280110bbfSPasha Tatashin 24380110bbfSPasha Tatashin void __page_table_check_pte_clear_range(struct mm_struct *mm, 24480110bbfSPasha Tatashin unsigned long addr, 24580110bbfSPasha Tatashin pmd_t pmd) 24680110bbfSPasha Tatashin { 24780110bbfSPasha Tatashin if (&init_mm == mm) 24880110bbfSPasha Tatashin return; 24980110bbfSPasha Tatashin 25080110bbfSPasha Tatashin if (!pmd_bad(pmd) && !pmd_leaf(pmd)) { 25180110bbfSPasha Tatashin pte_t *ptep = pte_offset_map(&pmd, addr); 25280110bbfSPasha Tatashin unsigned long i; 25380110bbfSPasha Tatashin 25480110bbfSPasha Tatashin for (i = 0; i < PTRS_PER_PTE; i++) { 25580110bbfSPasha Tatashin __page_table_check_pte_clear(mm, addr, *ptep); 25680110bbfSPasha Tatashin addr += PAGE_SIZE; 25780110bbfSPasha Tatashin ptep++; 25880110bbfSPasha Tatashin } 259*24c8e27eSMiaohe Lin pte_unmap(ptep - PTRS_PER_PTE); 26080110bbfSPasha Tatashin } 26180110bbfSPasha Tatashin } 262