1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2019 SiFive 4 */ 5 6 #include <linux/pagewalk.h> 7 #include <linux/pgtable.h> 8 #include <asm/tlbflush.h> 9 #include <asm/bitops.h> 10 #include <asm/set_memory.h> 11 12 struct pageattr_masks { 13 pgprot_t set_mask; 14 pgprot_t clear_mask; 15 }; 16 17 static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk) 18 { 19 struct pageattr_masks *masks = walk->private; 20 unsigned long new_val = val; 21 22 new_val &= ~(pgprot_val(masks->clear_mask)); 23 new_val |= (pgprot_val(masks->set_mask)); 24 25 return new_val; 26 } 27 28 static int pageattr_pgd_entry(pgd_t *pgd, unsigned long addr, 29 unsigned long next, struct mm_walk *walk) 30 { 31 pgd_t val = READ_ONCE(*pgd); 32 33 if (pgd_leaf(val)) { 34 val = __pgd(set_pageattr_masks(pgd_val(val), walk)); 35 set_pgd(pgd, val); 36 } 37 38 return 0; 39 } 40 41 static int pageattr_p4d_entry(p4d_t *p4d, unsigned long addr, 42 unsigned long next, struct mm_walk *walk) 43 { 44 p4d_t val = READ_ONCE(*p4d); 45 46 if (p4d_leaf(val)) { 47 val = __p4d(set_pageattr_masks(p4d_val(val), walk)); 48 set_p4d(p4d, val); 49 } 50 51 return 0; 52 } 53 54 static int pageattr_pud_entry(pud_t *pud, unsigned long addr, 55 unsigned long next, struct mm_walk *walk) 56 { 57 pud_t val = READ_ONCE(*pud); 58 59 if (pud_leaf(val)) { 60 val = __pud(set_pageattr_masks(pud_val(val), walk)); 61 set_pud(pud, val); 62 } 63 64 return 0; 65 } 66 67 static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr, 68 unsigned long next, struct mm_walk *walk) 69 { 70 pmd_t val = READ_ONCE(*pmd); 71 72 if (pmd_leaf(val)) { 73 val = __pmd(set_pageattr_masks(pmd_val(val), walk)); 74 set_pmd(pmd, val); 75 } 76 77 return 0; 78 } 79 80 static int pageattr_pte_entry(pte_t *pte, unsigned long addr, 81 unsigned long next, struct mm_walk *walk) 82 { 83 pte_t val = READ_ONCE(*pte); 84 85 val = __pte(set_pageattr_masks(pte_val(val), walk)); 86 set_pte(pte, val); 87 88 return 0; 89 } 90 91 static int pageattr_pte_hole(unsigned long addr, unsigned long next, 92 int depth, struct mm_walk *walk) 93 { 94 /* Nothing to do here */ 95 return 0; 96 } 97 98 static const struct mm_walk_ops pageattr_ops = { 99 .pgd_entry = pageattr_pgd_entry, 100 .p4d_entry = pageattr_p4d_entry, 101 .pud_entry = pageattr_pud_entry, 102 .pmd_entry = pageattr_pmd_entry, 103 .pte_entry = pageattr_pte_entry, 104 .pte_hole = pageattr_pte_hole, 105 .walk_lock = PGWALK_RDLOCK, 106 }; 107 108 static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, 109 pgprot_t clear_mask) 110 { 111 int ret; 112 unsigned long start = addr; 113 unsigned long end = start + PAGE_SIZE * numpages; 114 struct pageattr_masks masks = { 115 .set_mask = set_mask, 116 .clear_mask = clear_mask 117 }; 118 119 if (!numpages) 120 return 0; 121 122 mmap_write_lock(&init_mm); 123 ret = walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL, 124 &masks); 125 mmap_write_unlock(&init_mm); 126 127 flush_tlb_kernel_range(start, end); 128 129 return ret; 130 } 131 132 int set_memory_rw_nx(unsigned long addr, int numpages) 133 { 134 return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE), 135 __pgprot(_PAGE_EXEC)); 136 } 137 138 int set_memory_ro(unsigned long addr, int numpages) 139 { 140 return __set_memory(addr, numpages, __pgprot(_PAGE_READ), 141 __pgprot(_PAGE_WRITE)); 142 } 143 144 int set_memory_rw(unsigned long addr, int numpages) 145 { 146 return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE), 147 __pgprot(0)); 148 } 149 150 int set_memory_x(unsigned long addr, int numpages) 151 { 152 return __set_memory(addr, numpages, __pgprot(_PAGE_EXEC), __pgprot(0)); 153 } 154 155 int set_memory_nx(unsigned long addr, int numpages) 156 { 157 return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC)); 158 } 159 160 int set_direct_map_invalid_noflush(struct page *page) 161 { 162 int ret; 163 unsigned long start = (unsigned long)page_address(page); 164 unsigned long end = start + PAGE_SIZE; 165 struct pageattr_masks masks = { 166 .set_mask = __pgprot(0), 167 .clear_mask = __pgprot(_PAGE_PRESENT) 168 }; 169 170 mmap_read_lock(&init_mm); 171 ret = walk_page_range(&init_mm, start, end, &pageattr_ops, &masks); 172 mmap_read_unlock(&init_mm); 173 174 return ret; 175 } 176 177 int set_direct_map_default_noflush(struct page *page) 178 { 179 int ret; 180 unsigned long start = (unsigned long)page_address(page); 181 unsigned long end = start + PAGE_SIZE; 182 struct pageattr_masks masks = { 183 .set_mask = PAGE_KERNEL, 184 .clear_mask = __pgprot(0) 185 }; 186 187 mmap_read_lock(&init_mm); 188 ret = walk_page_range(&init_mm, start, end, &pageattr_ops, &masks); 189 mmap_read_unlock(&init_mm); 190 191 return ret; 192 } 193 194 #ifdef CONFIG_DEBUG_PAGEALLOC 195 void __kernel_map_pages(struct page *page, int numpages, int enable) 196 { 197 if (!debug_pagealloc_enabled()) 198 return; 199 200 if (enable) 201 __set_memory((unsigned long)page_address(page), numpages, 202 __pgprot(_PAGE_PRESENT), __pgprot(0)); 203 else 204 __set_memory((unsigned long)page_address(page), numpages, 205 __pgprot(0), __pgprot(_PAGE_PRESENT)); 206 } 207 #endif 208 209 bool kernel_page_present(struct page *page) 210 { 211 unsigned long addr = (unsigned long)page_address(page); 212 pgd_t *pgd; 213 pud_t *pud; 214 p4d_t *p4d; 215 pmd_t *pmd; 216 pte_t *pte; 217 218 pgd = pgd_offset_k(addr); 219 if (!pgd_present(*pgd)) 220 return false; 221 if (pgd_leaf(*pgd)) 222 return true; 223 224 p4d = p4d_offset(pgd, addr); 225 if (!p4d_present(*p4d)) 226 return false; 227 if (p4d_leaf(*p4d)) 228 return true; 229 230 pud = pud_offset(p4d, addr); 231 if (!pud_present(*pud)) 232 return false; 233 if (pud_leaf(*pud)) 234 return true; 235 236 pmd = pmd_offset(pud, addr); 237 if (!pmd_present(*pmd)) 238 return false; 239 if (pmd_leaf(*pmd)) 240 return true; 241 242 pte = pte_offset_kernel(pmd, addr); 243 return pte_present(*pte); 244 } 245