1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Copyright (c) 2021, Google LLC. 5 * Pasha Tatashin <pasha.tatashin@soleen.com> 6 */ 7 #include <linux/kstrtox.h> 8 #include <linux/mm.h> 9 #include <linux/page_table_check.h> 10 #include <linux/swap.h> 11 #include <linux/leafops.h> 12 13 #undef pr_fmt 14 #define pr_fmt(fmt) "page_table_check: " fmt 15 16 struct page_table_check { 17 atomic_t anon_map_count; 18 atomic_t file_map_count; 19 }; 20 21 static bool __page_table_check_enabled __initdata = 22 IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED); 23 24 DEFINE_STATIC_KEY_TRUE(page_table_check_disabled); 25 EXPORT_SYMBOL(page_table_check_disabled); 26 27 static int __init early_page_table_check_param(char *buf) 28 { 29 return kstrtobool(buf, &__page_table_check_enabled); 30 } 31 32 early_param("page_table_check", early_page_table_check_param); 33 34 static bool __init need_page_table_check(void) 35 { 36 return __page_table_check_enabled; 37 } 38 39 static void __init init_page_table_check(void) 40 { 41 if (!__page_table_check_enabled) 42 return; 43 static_branch_disable(&page_table_check_disabled); 44 } 45 46 struct page_ext_operations page_table_check_ops = { 47 .size = sizeof(struct page_table_check), 48 .need = need_page_table_check, 49 .init = init_page_table_check, 50 .need_shared_flags = false, 51 }; 52 53 static struct page_table_check *get_page_table_check(struct page_ext *page_ext) 54 { 55 BUG_ON(!page_ext); 56 return page_ext_data(page_ext, &page_table_check_ops); 57 } 58 59 /* 60 * An entry is removed from the page table, decrement the counters for that page 61 * verify that it is of correct type and counters do not become negative. 62 */ 63 static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt) 64 { 65 struct page_ext_iter iter; 66 struct page_ext *page_ext; 67 struct page *page; 68 bool anon; 69 70 if (!pfn_valid(pfn)) 71 return; 72 73 page = pfn_to_page(pfn); 74 BUG_ON(PageSlab(page)); 75 anon = PageAnon(page); 76 77 rcu_read_lock(); 78 for_each_page_ext(page, pgcnt, page_ext, iter) { 79 struct page_table_check *ptc = get_page_table_check(page_ext); 80 81 if (anon) { 82 BUG_ON(atomic_read(&ptc->file_map_count)); 83 BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0); 84 } else { 85 BUG_ON(atomic_read(&ptc->anon_map_count)); 86 BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0); 87 } 88 } 89 rcu_read_unlock(); 90 } 91 92 /* 93 * A new entry is added to the page table, increment the counters for that page 94 * verify that it is of correct type and is not being mapped with a different 95 * type to a different process. 96 */ 97 static void page_table_check_set(unsigned long pfn, unsigned long pgcnt, 98 bool rw) 99 { 100 struct page_ext_iter iter; 101 struct page_ext *page_ext; 102 struct page *page; 103 bool anon; 104 105 if (!pfn_valid(pfn)) 106 return; 107 108 page = pfn_to_page(pfn); 109 BUG_ON(PageSlab(page)); 110 anon = PageAnon(page); 111 112 rcu_read_lock(); 113 for_each_page_ext(page, pgcnt, page_ext, iter) { 114 struct page_table_check *ptc = get_page_table_check(page_ext); 115 116 if (anon) { 117 BUG_ON(atomic_read(&ptc->file_map_count)); 118 BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw); 119 } else { 120 BUG_ON(atomic_read(&ptc->anon_map_count)); 121 BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0); 122 } 123 } 124 rcu_read_unlock(); 125 } 126 127 /* 128 * page is on free list, or is being allocated, verify that counters are zeroes 129 * crash if they are not. 130 */ 131 void __page_table_check_zero(struct page *page, unsigned int order) 132 { 133 struct page_ext_iter iter; 134 struct page_ext *page_ext; 135 136 BUG_ON(PageSlab(page)); 137 138 rcu_read_lock(); 139 for_each_page_ext(page, 1 << order, page_ext, iter) { 140 struct page_table_check *ptc = get_page_table_check(page_ext); 141 142 BUG_ON(atomic_read(&ptc->anon_map_count)); 143 BUG_ON(atomic_read(&ptc->file_map_count)); 144 } 145 rcu_read_unlock(); 146 } 147 148 void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, 149 pte_t pte) 150 { 151 if (&init_mm == mm) 152 return; 153 154 if (pte_user_accessible_page(mm, addr, pte) && !pte_special(pte)) 155 page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT); 156 } 157 EXPORT_SYMBOL(__page_table_check_pte_clear); 158 159 static inline bool page_table_check_huge_zero_pmd(pmd_t pmd) 160 { 161 unsigned long pfn = pmd_pfn(pmd); 162 163 if (!pfn_valid(pfn)) 164 return false; 165 166 return is_huge_zero_folio(page_folio(pfn_to_page(pfn))); 167 } 168 169 void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr, 170 pmd_t pmd) 171 { 172 if (&init_mm == mm) 173 return; 174 175 if (pmd_user_accessible_page(mm, addr, pmd) && 176 !page_table_check_huge_zero_pmd(pmd)) 177 page_table_check_clear(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT); 178 } 179 EXPORT_SYMBOL(__page_table_check_pmd_clear); 180 181 void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr, 182 pud_t pud) 183 { 184 if (&init_mm == mm) 185 return; 186 187 if (pud_user_accessible_page(mm, addr, pud)) 188 page_table_check_clear(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT); 189 } 190 EXPORT_SYMBOL(__page_table_check_pud_clear); 191 192 /* Whether the swap entry cached writable information */ 193 static inline bool softleaf_cached_writable(softleaf_t entry) 194 { 195 return softleaf_is_device_private_write(entry) || 196 softleaf_is_migration_write(entry); 197 } 198 199 static void page_table_check_pte_flags(pte_t pte) 200 { 201 if (pte_present(pte)) { 202 WARN_ON_ONCE(pte_uffd_wp(pte) && pte_write(pte)); 203 } else if (pte_swp_uffd_wp(pte)) { 204 const softleaf_t entry = softleaf_from_pte(pte); 205 206 WARN_ON_ONCE(softleaf_cached_writable(entry)); 207 } 208 } 209 210 void __page_table_check_ptes_set(struct mm_struct *mm, unsigned long addr, 211 pte_t *ptep, pte_t pte, unsigned int nr) 212 { 213 unsigned int i; 214 215 if (&init_mm == mm) 216 return; 217 218 page_table_check_pte_flags(pte); 219 220 for (i = 0; i < nr; i++) 221 __page_table_check_pte_clear(mm, addr + PAGE_SIZE * i, ptep_get(ptep + i)); 222 if (pte_user_accessible_page(mm, addr, pte) && !pte_special(pte)) 223 page_table_check_set(pte_pfn(pte), nr, pte_write(pte)); 224 } 225 EXPORT_SYMBOL(__page_table_check_ptes_set); 226 227 static inline void page_table_check_pmd_flags(pmd_t pmd) 228 { 229 if (pmd_present(pmd)) { 230 if (pmd_uffd_wp(pmd)) 231 WARN_ON_ONCE(pmd_write(pmd)); 232 } else if (pmd_swp_uffd_wp(pmd)) { 233 const softleaf_t entry = softleaf_from_pmd(pmd); 234 235 WARN_ON_ONCE(softleaf_cached_writable(entry)); 236 } 237 } 238 239 void __page_table_check_pmds_set(struct mm_struct *mm, unsigned long addr, 240 pmd_t *pmdp, pmd_t pmd, unsigned int nr) 241 { 242 unsigned long stride = PMD_SIZE >> PAGE_SHIFT; 243 unsigned int i; 244 245 if (&init_mm == mm) 246 return; 247 248 page_table_check_pmd_flags(pmd); 249 250 for (i = 0; i < nr; i++) 251 __page_table_check_pmd_clear(mm, addr + PMD_SIZE * i, *(pmdp + i)); 252 if (pmd_user_accessible_page(mm, addr, pmd) && 253 !page_table_check_huge_zero_pmd(pmd)) 254 page_table_check_set(pmd_pfn(pmd), stride * nr, pmd_write(pmd)); 255 } 256 EXPORT_SYMBOL(__page_table_check_pmds_set); 257 258 void __page_table_check_puds_set(struct mm_struct *mm, unsigned long addr, 259 pud_t *pudp, pud_t pud, unsigned int nr) 260 { 261 unsigned long stride = PUD_SIZE >> PAGE_SHIFT; 262 unsigned int i; 263 264 if (&init_mm == mm) 265 return; 266 267 for (i = 0; i < nr; i++) 268 __page_table_check_pud_clear(mm, addr + PUD_SIZE * i, *(pudp + i)); 269 if (pud_user_accessible_page(mm, addr, pud)) 270 page_table_check_set(pud_pfn(pud), stride * nr, pud_write(pud)); 271 } 272 EXPORT_SYMBOL(__page_table_check_puds_set); 273 274 void __page_table_check_pte_clear_range(struct mm_struct *mm, 275 unsigned long addr, 276 pmd_t pmd) 277 { 278 if (&init_mm == mm) 279 return; 280 281 if (!pmd_bad(pmd) && !pmd_leaf(pmd)) { 282 pte_t *ptep = pte_offset_map(&pmd, addr); 283 unsigned long i; 284 285 if (WARN_ON(!ptep)) 286 return; 287 for (i = 0; i < PTRS_PER_PTE; i++) { 288 __page_table_check_pte_clear(mm, addr, ptep_get(ptep)); 289 addr += PAGE_SIZE; 290 ptep++; 291 } 292 pte_unmap(ptep - PTRS_PER_PTE); 293 } 294 } 295