1df4e817bSPasha Tatashin // SPDX-License-Identifier: GPL-2.0 2df4e817bSPasha Tatashin 3df4e817bSPasha Tatashin /* 4df4e817bSPasha Tatashin * Copyright (c) 2021, Google LLC. 5df4e817bSPasha Tatashin * Pasha Tatashin <pasha.tatashin@soleen.com> 6df4e817bSPasha Tatashin */ 7f15be1b8SChristophe JAILLET #include <linux/kstrtox.h> 8df4e817bSPasha Tatashin #include <linux/mm.h> 9df4e817bSPasha Tatashin #include <linux/page_table_check.h> 10df4e817bSPasha Tatashin 11df4e817bSPasha Tatashin #undef pr_fmt 12df4e817bSPasha Tatashin #define pr_fmt(fmt) "page_table_check: " fmt 13df4e817bSPasha Tatashin 14df4e817bSPasha Tatashin struct page_table_check { 15df4e817bSPasha Tatashin atomic_t anon_map_count; 16df4e817bSPasha Tatashin atomic_t file_map_count; 17df4e817bSPasha Tatashin }; 18df4e817bSPasha Tatashin 19df4e817bSPasha Tatashin static bool __page_table_check_enabled __initdata = 20df4e817bSPasha Tatashin IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED); 21df4e817bSPasha Tatashin 22df4e817bSPasha Tatashin DEFINE_STATIC_KEY_TRUE(page_table_check_disabled); 23df4e817bSPasha Tatashin EXPORT_SYMBOL(page_table_check_disabled); 24df4e817bSPasha Tatashin 25df4e817bSPasha Tatashin static int __init early_page_table_check_param(char *buf) 26df4e817bSPasha Tatashin { 27f15be1b8SChristophe JAILLET return kstrtobool(buf, &__page_table_check_enabled); 28df4e817bSPasha Tatashin } 29df4e817bSPasha Tatashin 30df4e817bSPasha Tatashin early_param("page_table_check", early_page_table_check_param); 31df4e817bSPasha Tatashin 32df4e817bSPasha Tatashin static bool __init need_page_table_check(void) 33df4e817bSPasha Tatashin { 34df4e817bSPasha Tatashin return __page_table_check_enabled; 35df4e817bSPasha Tatashin } 36df4e817bSPasha Tatashin 37df4e817bSPasha Tatashin static void __init init_page_table_check(void) 38df4e817bSPasha Tatashin { 39df4e817bSPasha Tatashin if (!__page_table_check_enabled) 40df4e817bSPasha Tatashin return; 41df4e817bSPasha Tatashin static_branch_disable(&page_table_check_disabled); 42df4e817bSPasha Tatashin } 43df4e817bSPasha Tatashin 44df4e817bSPasha Tatashin struct page_ext_operations page_table_check_ops = { 45df4e817bSPasha Tatashin .size = sizeof(struct page_table_check), 46df4e817bSPasha Tatashin .need = need_page_table_check, 47df4e817bSPasha Tatashin .init = init_page_table_check, 486189eb82SPasha Tatashin .need_shared_flags = false, 49df4e817bSPasha Tatashin }; 50df4e817bSPasha Tatashin 51df4e817bSPasha Tatashin static struct page_table_check *get_page_table_check(struct page_ext *page_ext) 52df4e817bSPasha Tatashin { 53df4e817bSPasha Tatashin BUG_ON(!page_ext); 54df4e817bSPasha Tatashin return (void *)(page_ext) + page_table_check_ops.offset; 55df4e817bSPasha Tatashin } 56df4e817bSPasha Tatashin 57df4e817bSPasha Tatashin /* 583ae6d3e3SChih-En Lin * An entry is removed from the page table, decrement the counters for that page 59df4e817bSPasha Tatashin * verify that it is of correct type and counters do not become negative. 60df4e817bSPasha Tatashin */ 6134c876ceSKemeng Shi static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt) 62df4e817bSPasha Tatashin { 63df4e817bSPasha Tatashin struct page_ext *page_ext; 64df4e817bSPasha Tatashin struct page *page; 6564d8b9e1SPasha Tatashin unsigned long i; 66df4e817bSPasha Tatashin bool anon; 67df4e817bSPasha Tatashin 68df4e817bSPasha Tatashin if (!pfn_valid(pfn)) 69df4e817bSPasha Tatashin return; 70df4e817bSPasha Tatashin 71df4e817bSPasha Tatashin page = pfn_to_page(pfn); 72b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 7344d0fb38SRuihan Li 7444d0fb38SRuihan Li BUG_ON(PageSlab(page)); 75df4e817bSPasha Tatashin anon = PageAnon(page); 76df4e817bSPasha Tatashin 77df4e817bSPasha Tatashin for (i = 0; i < pgcnt; i++) { 78df4e817bSPasha Tatashin struct page_table_check *ptc = get_page_table_check(page_ext); 79df4e817bSPasha Tatashin 80df4e817bSPasha Tatashin if (anon) { 81df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->file_map_count)); 82df4e817bSPasha Tatashin BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0); 83df4e817bSPasha Tatashin } else { 84df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->anon_map_count)); 85df4e817bSPasha Tatashin BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0); 86df4e817bSPasha Tatashin } 87df4e817bSPasha Tatashin page_ext = page_ext_next(page_ext); 88df4e817bSPasha Tatashin } 89b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 90df4e817bSPasha Tatashin } 91df4e817bSPasha Tatashin 92df4e817bSPasha Tatashin /* 933ae6d3e3SChih-En Lin * A new entry is added to the page table, increment the counters for that page 94df4e817bSPasha Tatashin * verify that it is of correct type and is not being mapped with a different 95df4e817bSPasha Tatashin * type to a different process. 96df4e817bSPasha Tatashin */ 972f933eafSKemeng Shi static void page_table_check_set(unsigned long pfn, unsigned long pgcnt, 98df4e817bSPasha Tatashin bool rw) 99df4e817bSPasha Tatashin { 100df4e817bSPasha Tatashin struct page_ext *page_ext; 101df4e817bSPasha Tatashin struct page *page; 10264d8b9e1SPasha Tatashin unsigned long i; 103df4e817bSPasha Tatashin bool anon; 104df4e817bSPasha Tatashin 105df4e817bSPasha Tatashin if (!pfn_valid(pfn)) 106df4e817bSPasha Tatashin return; 107df4e817bSPasha Tatashin 108df4e817bSPasha Tatashin page = pfn_to_page(pfn); 109b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 11044d0fb38SRuihan Li 11144d0fb38SRuihan Li BUG_ON(PageSlab(page)); 112df4e817bSPasha Tatashin anon = PageAnon(page); 113df4e817bSPasha Tatashin 114df4e817bSPasha Tatashin for (i = 0; i < pgcnt; i++) { 115df4e817bSPasha Tatashin struct page_table_check *ptc = get_page_table_check(page_ext); 116df4e817bSPasha Tatashin 117df4e817bSPasha Tatashin if (anon) { 118df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->file_map_count)); 119df4e817bSPasha Tatashin BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw); 120df4e817bSPasha Tatashin } else { 121df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->anon_map_count)); 122df4e817bSPasha Tatashin BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0); 123df4e817bSPasha Tatashin } 124df4e817bSPasha Tatashin page_ext = page_ext_next(page_ext); 125df4e817bSPasha Tatashin } 126b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 127df4e817bSPasha Tatashin } 128df4e817bSPasha Tatashin 129df4e817bSPasha Tatashin /* 130df4e817bSPasha Tatashin * page is on free list, or is being allocated, verify that counters are zeroes 131df4e817bSPasha Tatashin * crash if they are not. 132df4e817bSPasha Tatashin */ 133df4e817bSPasha Tatashin void __page_table_check_zero(struct page *page, unsigned int order) 134df4e817bSPasha Tatashin { 135b1d5488aSCharan Teja Kalla struct page_ext *page_ext; 13664d8b9e1SPasha Tatashin unsigned long i; 137df4e817bSPasha Tatashin 13844d0fb38SRuihan Li BUG_ON(PageSlab(page)); 13944d0fb38SRuihan Li 140b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page); 141df4e817bSPasha Tatashin BUG_ON(!page_ext); 14264d8b9e1SPasha Tatashin for (i = 0; i < (1ul << order); i++) { 143df4e817bSPasha Tatashin struct page_table_check *ptc = get_page_table_check(page_ext); 144df4e817bSPasha Tatashin 145df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->anon_map_count)); 146df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->file_map_count)); 147df4e817bSPasha Tatashin page_ext = page_ext_next(page_ext); 148df4e817bSPasha Tatashin } 149b1d5488aSCharan Teja Kalla page_ext_put(page_ext); 150df4e817bSPasha Tatashin } 151df4e817bSPasha Tatashin 152aa232204SKemeng Shi void __page_table_check_pte_clear(struct mm_struct *mm, pte_t pte) 153df4e817bSPasha Tatashin { 154df4e817bSPasha Tatashin if (&init_mm == mm) 155df4e817bSPasha Tatashin return; 156df4e817bSPasha Tatashin 157df4e817bSPasha Tatashin if (pte_user_accessible_page(pte)) { 15834c876ceSKemeng Shi page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT); 159df4e817bSPasha Tatashin } 160df4e817bSPasha Tatashin } 161df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pte_clear); 162df4e817bSPasha Tatashin 163*1831414cSKemeng Shi void __page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd) 164df4e817bSPasha Tatashin { 165df4e817bSPasha Tatashin if (&init_mm == mm) 166df4e817bSPasha Tatashin return; 167df4e817bSPasha Tatashin 168df4e817bSPasha Tatashin if (pmd_user_accessible_page(pmd)) { 16934c876ceSKemeng Shi page_table_check_clear(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT); 170df4e817bSPasha Tatashin } 171df4e817bSPasha Tatashin } 172df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pmd_clear); 173df4e817bSPasha Tatashin 174df4e817bSPasha Tatashin void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr, 175df4e817bSPasha Tatashin pud_t pud) 176df4e817bSPasha Tatashin { 177df4e817bSPasha Tatashin if (&init_mm == mm) 178df4e817bSPasha Tatashin return; 179df4e817bSPasha Tatashin 180df4e817bSPasha Tatashin if (pud_user_accessible_page(pud)) { 18134c876ceSKemeng Shi page_table_check_clear(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT); 182df4e817bSPasha Tatashin } 183df4e817bSPasha Tatashin } 184df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pud_clear); 185df4e817bSPasha Tatashin 186df4e817bSPasha Tatashin void __page_table_check_pte_set(struct mm_struct *mm, unsigned long addr, 187df4e817bSPasha Tatashin pte_t *ptep, pte_t pte) 188df4e817bSPasha Tatashin { 189df4e817bSPasha Tatashin if (&init_mm == mm) 190df4e817bSPasha Tatashin return; 191df4e817bSPasha Tatashin 192aa232204SKemeng Shi __page_table_check_pte_clear(mm, ptep_get(ptep)); 193df4e817bSPasha Tatashin if (pte_user_accessible_page(pte)) { 1942f933eafSKemeng Shi page_table_check_set(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT, 195df4e817bSPasha Tatashin pte_write(pte)); 196df4e817bSPasha Tatashin } 197df4e817bSPasha Tatashin } 198df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pte_set); 199df4e817bSPasha Tatashin 200df4e817bSPasha Tatashin void __page_table_check_pmd_set(struct mm_struct *mm, unsigned long addr, 201df4e817bSPasha Tatashin pmd_t *pmdp, pmd_t pmd) 202df4e817bSPasha Tatashin { 203df4e817bSPasha Tatashin if (&init_mm == mm) 204df4e817bSPasha Tatashin return; 205df4e817bSPasha Tatashin 206*1831414cSKemeng Shi __page_table_check_pmd_clear(mm, *pmdp); 207df4e817bSPasha Tatashin if (pmd_user_accessible_page(pmd)) { 2082f933eafSKemeng Shi page_table_check_set(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT, 209df4e817bSPasha Tatashin pmd_write(pmd)); 210df4e817bSPasha Tatashin } 211df4e817bSPasha Tatashin } 212df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pmd_set); 213df4e817bSPasha Tatashin 214df4e817bSPasha Tatashin void __page_table_check_pud_set(struct mm_struct *mm, unsigned long addr, 215df4e817bSPasha Tatashin pud_t *pudp, pud_t pud) 216df4e817bSPasha Tatashin { 217df4e817bSPasha Tatashin if (&init_mm == mm) 218df4e817bSPasha Tatashin return; 219df4e817bSPasha Tatashin 22064d8b9e1SPasha Tatashin __page_table_check_pud_clear(mm, addr, *pudp); 221df4e817bSPasha Tatashin if (pud_user_accessible_page(pud)) { 2222f933eafSKemeng Shi page_table_check_set(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT, 223df4e817bSPasha Tatashin pud_write(pud)); 224df4e817bSPasha Tatashin } 225df4e817bSPasha Tatashin } 226df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pud_set); 22780110bbfSPasha Tatashin 22880110bbfSPasha Tatashin void __page_table_check_pte_clear_range(struct mm_struct *mm, 22980110bbfSPasha Tatashin unsigned long addr, 23080110bbfSPasha Tatashin pmd_t pmd) 23180110bbfSPasha Tatashin { 23280110bbfSPasha Tatashin if (&init_mm == mm) 23380110bbfSPasha Tatashin return; 23480110bbfSPasha Tatashin 23580110bbfSPasha Tatashin if (!pmd_bad(pmd) && !pmd_leaf(pmd)) { 23680110bbfSPasha Tatashin pte_t *ptep = pte_offset_map(&pmd, addr); 23780110bbfSPasha Tatashin unsigned long i; 23880110bbfSPasha Tatashin 2399f2bad09SHugh Dickins if (WARN_ON(!ptep)) 2409f2bad09SHugh Dickins return; 24180110bbfSPasha Tatashin for (i = 0; i < PTRS_PER_PTE; i++) { 242aa232204SKemeng Shi __page_table_check_pte_clear(mm, ptep_get(ptep)); 24380110bbfSPasha Tatashin addr += PAGE_SIZE; 24480110bbfSPasha Tatashin ptep++; 24580110bbfSPasha Tatashin } 24624c8e27eSMiaohe Lin pte_unmap(ptep - PTRS_PER_PTE); 24780110bbfSPasha Tatashin } 24880110bbfSPasha Tatashin } 249