1df4e817bSPasha Tatashin // SPDX-License-Identifier: GPL-2.0 2df4e817bSPasha Tatashin 3df4e817bSPasha Tatashin /* 4df4e817bSPasha Tatashin * Copyright (c) 2021, Google LLC. 5df4e817bSPasha Tatashin * Pasha Tatashin <pasha.tatashin@soleen.com> 6df4e817bSPasha Tatashin */ 7f15be1b8SChristophe JAILLET #include <linux/kstrtox.h> 8df4e817bSPasha Tatashin #include <linux/mm.h> 9df4e817bSPasha Tatashin #include <linux/page_table_check.h> 10df4e817bSPasha Tatashin 11df4e817bSPasha Tatashin #undef pr_fmt 12df4e817bSPasha Tatashin #define pr_fmt(fmt) "page_table_check: " fmt 13df4e817bSPasha Tatashin 14df4e817bSPasha Tatashin struct page_table_check { 15df4e817bSPasha Tatashin atomic_t anon_map_count; 16df4e817bSPasha Tatashin atomic_t file_map_count; 17df4e817bSPasha Tatashin }; 18df4e817bSPasha Tatashin 19df4e817bSPasha Tatashin static bool __page_table_check_enabled __initdata = 20df4e817bSPasha Tatashin IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED); 21df4e817bSPasha Tatashin 22df4e817bSPasha Tatashin DEFINE_STATIC_KEY_TRUE(page_table_check_disabled); 23df4e817bSPasha Tatashin EXPORT_SYMBOL(page_table_check_disabled); 24df4e817bSPasha Tatashin 25df4e817bSPasha Tatashin static int __init early_page_table_check_param(char *buf) 26df4e817bSPasha Tatashin { 27f15be1b8SChristophe JAILLET return kstrtobool(buf, &__page_table_check_enabled); 28df4e817bSPasha Tatashin } 29df4e817bSPasha Tatashin 30df4e817bSPasha Tatashin early_param("page_table_check", early_page_table_check_param); 31df4e817bSPasha Tatashin 32df4e817bSPasha Tatashin static bool __init need_page_table_check(void) 33df4e817bSPasha Tatashin { 34df4e817bSPasha Tatashin return __page_table_check_enabled; 35df4e817bSPasha Tatashin } 36df4e817bSPasha Tatashin 37df4e817bSPasha Tatashin static void __init init_page_table_check(void) 38df4e817bSPasha Tatashin { 39df4e817bSPasha Tatashin if (!__page_table_check_enabled) 40df4e817bSPasha Tatashin return; 41df4e817bSPasha Tatashin static_branch_disable(&page_table_check_disabled); 42df4e817bSPasha Tatashin } 43df4e817bSPasha Tatashin 44df4e817bSPasha Tatashin struct page_ext_operations page_table_check_ops = { 45df4e817bSPasha Tatashin .size = sizeof(struct page_table_check), 46df4e817bSPasha Tatashin .need = need_page_table_check, 47df4e817bSPasha Tatashin .init = init_page_table_check, 486189eb82SPasha Tatashin .need_shared_flags = false, 49df4e817bSPasha Tatashin }; 50df4e817bSPasha Tatashin 51df4e817bSPasha Tatashin static struct page_table_check *get_page_table_check(struct page_ext *page_ext) 52df4e817bSPasha Tatashin { 53df4e817bSPasha Tatashin BUG_ON(!page_ext); 54df4e817bSPasha Tatashin return (void *)(page_ext) + page_table_check_ops.offset; 55df4e817bSPasha Tatashin } 56df4e817bSPasha Tatashin 57df4e817bSPasha Tatashin /* 583ae6d3e3SChih-En Lin * An entry is removed from the page table, decrement the counters for that page 59df4e817bSPasha Tatashin * verify that it is of correct type and counters do not become negative. 60df4e817bSPasha Tatashin */ 61df4e817bSPasha Tatashin static void page_table_check_clear(struct mm_struct *mm, unsigned long addr, 62df4e817bSPasha Tatashin unsigned long pfn, unsigned long pgcnt) 63df4e817bSPasha Tatashin { 64df4e817bSPasha Tatashin struct page_ext *page_ext; 65df4e817bSPasha Tatashin struct page *page; 6664d8b9e1SPasha Tatashin unsigned long i; 67df4e817bSPasha Tatashin bool anon; 68df4e817bSPasha Tatashin 69df4e817bSPasha Tatashin if (!pfn_valid(pfn)) 70df4e817bSPasha Tatashin return; 71df4e817bSPasha Tatashin 72df4e817bSPasha Tatashin page = pfn_to_page(pfn); 73b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 74df4e817bSPasha Tatashin anon = PageAnon(page); 75df4e817bSPasha Tatashin 76df4e817bSPasha Tatashin for (i = 0; i < pgcnt; i++) { 77df4e817bSPasha Tatashin struct page_table_check *ptc = get_page_table_check(page_ext); 78df4e817bSPasha Tatashin 79df4e817bSPasha Tatashin if (anon) { 80df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->file_map_count)); 81df4e817bSPasha Tatashin BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0); 82df4e817bSPasha Tatashin } else { 83df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->anon_map_count)); 84df4e817bSPasha Tatashin BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0); 85df4e817bSPasha Tatashin } 86df4e817bSPasha Tatashin page_ext = page_ext_next(page_ext); 87df4e817bSPasha Tatashin } 88b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 89df4e817bSPasha Tatashin } 90df4e817bSPasha Tatashin 91df4e817bSPasha Tatashin /* 923ae6d3e3SChih-En Lin * A new entry is added to the page table, increment the counters for that page 93df4e817bSPasha Tatashin * verify that it is of correct type and is not being mapped with a different 94df4e817bSPasha Tatashin * type to a different process. 95df4e817bSPasha Tatashin */ 96df4e817bSPasha Tatashin static void page_table_check_set(struct mm_struct *mm, unsigned long addr, 97df4e817bSPasha Tatashin unsigned long pfn, unsigned long pgcnt, 98df4e817bSPasha Tatashin bool rw) 99df4e817bSPasha Tatashin { 100df4e817bSPasha Tatashin struct page_ext *page_ext; 101df4e817bSPasha Tatashin struct page *page; 10264d8b9e1SPasha Tatashin unsigned long i; 103df4e817bSPasha Tatashin bool anon; 104df4e817bSPasha Tatashin 105df4e817bSPasha Tatashin if (!pfn_valid(pfn)) 106df4e817bSPasha Tatashin return; 107df4e817bSPasha Tatashin 108df4e817bSPasha Tatashin page = pfn_to_page(pfn); 109b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 110df4e817bSPasha Tatashin anon = PageAnon(page); 111df4e817bSPasha Tatashin 112df4e817bSPasha Tatashin for (i = 0; i < pgcnt; i++) { 113df4e817bSPasha Tatashin struct page_table_check *ptc = get_page_table_check(page_ext); 114df4e817bSPasha Tatashin 115df4e817bSPasha Tatashin if (anon) { 116df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->file_map_count)); 117df4e817bSPasha Tatashin BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw); 118df4e817bSPasha Tatashin } else { 119df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->anon_map_count)); 120df4e817bSPasha Tatashin BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0); 121df4e817bSPasha Tatashin } 122df4e817bSPasha Tatashin page_ext = page_ext_next(page_ext); 123df4e817bSPasha Tatashin } 124b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 125df4e817bSPasha Tatashin } 126df4e817bSPasha Tatashin 127df4e817bSPasha Tatashin /* 128df4e817bSPasha Tatashin * page is on free list, or is being allocated, verify that counters are zeroes 129df4e817bSPasha Tatashin * crash if they are not. 130df4e817bSPasha Tatashin */ 131df4e817bSPasha Tatashin void __page_table_check_zero(struct page *page, unsigned int order) 132df4e817bSPasha Tatashin { 133b1d5488aSCharan Teja Kalla struct page_ext *page_ext; 13464d8b9e1SPasha Tatashin unsigned long i; 135df4e817bSPasha Tatashin 136b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 137df4e817bSPasha Tatashin BUG_ON(!page_ext); 13864d8b9e1SPasha Tatashin for (i = 0; i < (1ul << order); i++) { 139df4e817bSPasha Tatashin struct page_table_check *ptc = get_page_table_check(page_ext); 140df4e817bSPasha Tatashin 141df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->anon_map_count)); 142df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->file_map_count)); 143df4e817bSPasha Tatashin page_ext = page_ext_next(page_ext); 144df4e817bSPasha Tatashin } 145b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 146df4e817bSPasha Tatashin } 147df4e817bSPasha Tatashin 148df4e817bSPasha Tatashin void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, 149df4e817bSPasha Tatashin pte_t pte) 150df4e817bSPasha Tatashin { 151df4e817bSPasha Tatashin if (&init_mm == mm) 152df4e817bSPasha Tatashin return; 153df4e817bSPasha Tatashin 154df4e817bSPasha Tatashin if (pte_user_accessible_page(pte)) { 155df4e817bSPasha Tatashin page_table_check_clear(mm, addr, pte_pfn(pte), 156df4e817bSPasha Tatashin PAGE_SIZE >> PAGE_SHIFT); 157df4e817bSPasha Tatashin } 158df4e817bSPasha Tatashin } 159df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pte_clear); 160df4e817bSPasha Tatashin 161df4e817bSPasha Tatashin void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr, 162df4e817bSPasha Tatashin pmd_t pmd) 163df4e817bSPasha Tatashin { 164df4e817bSPasha Tatashin if (&init_mm == mm) 165df4e817bSPasha Tatashin return; 166df4e817bSPasha Tatashin 167df4e817bSPasha Tatashin if (pmd_user_accessible_page(pmd)) { 168df4e817bSPasha Tatashin page_table_check_clear(mm, addr, pmd_pfn(pmd), 16992fb0524STong Tiangen PMD_SIZE >> PAGE_SHIFT); 170df4e817bSPasha Tatashin } 171df4e817bSPasha Tatashin } 172df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pmd_clear); 173df4e817bSPasha Tatashin 174df4e817bSPasha Tatashin void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr, 175df4e817bSPasha Tatashin pud_t pud) 176df4e817bSPasha Tatashin { 177df4e817bSPasha Tatashin if (&init_mm == mm) 178df4e817bSPasha Tatashin return; 179df4e817bSPasha Tatashin 180df4e817bSPasha Tatashin if (pud_user_accessible_page(pud)) { 181df4e817bSPasha Tatashin page_table_check_clear(mm, addr, pud_pfn(pud), 18292fb0524STong Tiangen PUD_SIZE >> PAGE_SHIFT); 183df4e817bSPasha Tatashin } 184df4e817bSPasha Tatashin } 185df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pud_clear); 186df4e817bSPasha Tatashin 187df4e817bSPasha Tatashin void __page_table_check_pte_set(struct mm_struct *mm, unsigned long addr, 188df4e817bSPasha Tatashin pte_t *ptep, pte_t pte) 189df4e817bSPasha Tatashin { 190df4e817bSPasha Tatashin if (&init_mm == mm) 191df4e817bSPasha Tatashin return; 192df4e817bSPasha Tatashin 19364d8b9e1SPasha Tatashin __page_table_check_pte_clear(mm, addr, *ptep); 194df4e817bSPasha Tatashin if (pte_user_accessible_page(pte)) { 195df4e817bSPasha Tatashin page_table_check_set(mm, addr, pte_pfn(pte), 196df4e817bSPasha Tatashin PAGE_SIZE >> PAGE_SHIFT, 197df4e817bSPasha Tatashin pte_write(pte)); 198df4e817bSPasha Tatashin } 199df4e817bSPasha Tatashin } 200df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pte_set); 201df4e817bSPasha Tatashin 202df4e817bSPasha Tatashin void __page_table_check_pmd_set(struct mm_struct *mm, unsigned long addr, 203df4e817bSPasha Tatashin pmd_t *pmdp, pmd_t pmd) 204df4e817bSPasha Tatashin { 205df4e817bSPasha Tatashin if (&init_mm == mm) 206df4e817bSPasha Tatashin return; 207df4e817bSPasha Tatashin 20864d8b9e1SPasha Tatashin __page_table_check_pmd_clear(mm, addr, *pmdp); 209df4e817bSPasha Tatashin if (pmd_user_accessible_page(pmd)) { 210df4e817bSPasha Tatashin page_table_check_set(mm, addr, pmd_pfn(pmd), 21192fb0524STong Tiangen PMD_SIZE >> PAGE_SHIFT, 212df4e817bSPasha Tatashin pmd_write(pmd)); 213df4e817bSPasha Tatashin } 214df4e817bSPasha Tatashin } 215df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pmd_set); 216df4e817bSPasha Tatashin 217df4e817bSPasha Tatashin void __page_table_check_pud_set(struct mm_struct *mm, unsigned long addr, 218df4e817bSPasha Tatashin pud_t *pudp, pud_t pud) 219df4e817bSPasha Tatashin { 220df4e817bSPasha Tatashin if (&init_mm == mm) 221df4e817bSPasha Tatashin return; 222df4e817bSPasha Tatashin 22364d8b9e1SPasha Tatashin __page_table_check_pud_clear(mm, addr, *pudp); 224df4e817bSPasha Tatashin if (pud_user_accessible_page(pud)) { 225df4e817bSPasha Tatashin page_table_check_set(mm, addr, pud_pfn(pud), 22692fb0524STong Tiangen PUD_SIZE >> PAGE_SHIFT, 227df4e817bSPasha Tatashin pud_write(pud)); 228df4e817bSPasha Tatashin } 229df4e817bSPasha Tatashin } 230df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pud_set); 23180110bbfSPasha Tatashin 23280110bbfSPasha Tatashin void __page_table_check_pte_clear_range(struct mm_struct *mm, 23380110bbfSPasha Tatashin unsigned long addr, 23480110bbfSPasha Tatashin pmd_t pmd) 23580110bbfSPasha Tatashin { 23680110bbfSPasha Tatashin if (&init_mm == mm) 23780110bbfSPasha Tatashin return; 23880110bbfSPasha Tatashin 23980110bbfSPasha Tatashin if (!pmd_bad(pmd) && !pmd_leaf(pmd)) { 24080110bbfSPasha Tatashin pte_t *ptep = pte_offset_map(&pmd, addr); 24180110bbfSPasha Tatashin unsigned long i; 24280110bbfSPasha Tatashin 243*9f2bad09SHugh Dickins if (WARN_ON(!ptep)) 244*9f2bad09SHugh Dickins return; 24580110bbfSPasha Tatashin for (i = 0; i < PTRS_PER_PTE; i++) { 24680110bbfSPasha Tatashin __page_table_check_pte_clear(mm, addr, *ptep); 24780110bbfSPasha Tatashin addr += PAGE_SIZE; 24880110bbfSPasha Tatashin ptep++; 24980110bbfSPasha Tatashin } 25024c8e27eSMiaohe Lin pte_unmap(ptep - PTRS_PER_PTE); 25180110bbfSPasha Tatashin } 25280110bbfSPasha Tatashin } 253