1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright IBM Corp. 2011 4 * Author(s): Jan Glauber <jang@linux.vnet.ibm.com> 5 */ 6 #include <linux/cpufeature.h> 7 #include <linux/hugetlb.h> 8 #include <linux/proc_fs.h> 9 #include <linux/vmalloc.h> 10 #include <linux/mm.h> 11 #include <asm/cacheflush.h> 12 #include <asm/facility.h> 13 #include <asm/pgalloc.h> 14 #include <asm/kfence.h> 15 #include <asm/page.h> 16 #include <asm/asm.h> 17 #include <asm/set_memory.h> 18 19 void __storage_key_init_range(unsigned long start, unsigned long end) 20 { 21 unsigned long boundary, size; 22 23 while (start < end) { 24 if (cpu_has_edat1()) { 25 /* set storage keys for a 1MB frame */ 26 size = 1UL << 20; 27 boundary = (start + size) & ~(size - 1); 28 if (boundary <= end) { 29 do { 30 start = sske_frame(start, PAGE_DEFAULT_KEY); 31 } while (start < boundary); 32 continue; 33 } 34 } 35 page_set_storage_key(start, PAGE_DEFAULT_KEY, 1); 36 start += PAGE_SIZE; 37 } 38 } 39 40 #ifdef CONFIG_PROC_FS 41 atomic_long_t __bootdata_preserved(direct_pages_count[PG_DIRECT_MAP_MAX]); 42 43 void arch_report_meminfo(struct seq_file *m) 44 { 45 seq_printf(m, "DirectMap4k: %8lu kB\n", 46 atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_4K]) << 2); 47 seq_printf(m, "DirectMap1M: %8lu kB\n", 48 atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_1M]) << 10); 49 seq_printf(m, "DirectMap2G: %8lu kB\n", 50 atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_2G]) << 21); 51 } 52 #endif /* CONFIG_PROC_FS */ 53 54 static void pgt_set(unsigned long *old, unsigned long new, unsigned long addr, 55 unsigned long dtt) 56 { 57 unsigned long *table, mask; 58 59 mask = 0; 60 if (cpu_has_edat2()) { 61 switch (dtt) { 62 case CRDTE_DTT_REGION3: 63 mask = ~(PTRS_PER_PUD * sizeof(pud_t) - 1); 64 break; 65 case CRDTE_DTT_SEGMENT: 66 mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); 67 break; 68 case CRDTE_DTT_PAGE: 69 mask = ~(PTRS_PER_PTE * sizeof(pte_t) - 1); 70 break; 71 } 72 table = (unsigned long *)((unsigned long)old & mask); 73 crdte(*old, new, table, dtt, addr, get_lowcore()->kernel_asce.val); 74 } else { 75 cspg(old, *old, new); 76 } 77 } 78 79 static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end, 80 unsigned long flags) 81 { 82 pte_t *ptep, new; 83 84 if (flags == SET_MEMORY_4K) 85 return 0; 86 ptep = pte_offset_kernel(pmdp, addr); 87 do { 88 new = ptep_get(ptep); 89 if (pte_none(new)) 90 return -EINVAL; 91 if (flags & SET_MEMORY_RO) 92 new = pte_wrprotect(new); 93 else if (flags & SET_MEMORY_RW) 94 new = pte_mkwrite_novma(pte_mkdirty(new)); 95 if (flags & SET_MEMORY_NX) 96 new = set_pte_bit(new, __pgprot(_PAGE_NOEXEC)); 97 else if (flags & SET_MEMORY_X) 98 new = clear_pte_bit(new, __pgprot(_PAGE_NOEXEC)); 99 if (flags & SET_MEMORY_INV) { 100 new = set_pte_bit(new, __pgprot(_PAGE_INVALID)); 101 } else if (flags & SET_MEMORY_DEF) { 102 new = __pte(pte_val(new) & PAGE_MASK); 103 new = set_pte_bit(new, PAGE_KERNEL); 104 } 105 pgt_set((unsigned long *)ptep, pte_val(new), addr, CRDTE_DTT_PAGE); 106 ptep++; 107 addr += PAGE_SIZE; 108 } while (addr < end); 109 return 0; 110 } 111 112 static int split_pmd_page(pmd_t *pmdp, unsigned long addr) 113 { 114 unsigned long pte_addr, prot; 115 pte_t *pt_dir, *ptep; 116 pmd_t new, pmd; 117 int i, ro, nx; 118 119 pt_dir = vmem_pte_alloc(); 120 if (!pt_dir) 121 return -ENOMEM; 122 pmd = pmdp_get(pmdp); 123 pte_addr = pmd_pfn(pmd) << PAGE_SHIFT; 124 ro = !!(pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT); 125 nx = !!(pmd_val(pmd) & _SEGMENT_ENTRY_NOEXEC); 126 prot = pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL); 127 if (!nx) 128 prot &= ~_PAGE_NOEXEC; 129 ptep = pt_dir; 130 for (i = 0; i < PTRS_PER_PTE; i++) { 131 set_pte(ptep, __pte(pte_addr | prot)); 132 pte_addr += PAGE_SIZE; 133 ptep++; 134 } 135 new = __pmd(__pa(pt_dir) | _SEGMENT_ENTRY); 136 pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); 137 update_page_count(PG_DIRECT_MAP_4K, PTRS_PER_PTE); 138 update_page_count(PG_DIRECT_MAP_1M, -1); 139 return 0; 140 } 141 142 static void modify_pmd_page(pmd_t *pmdp, unsigned long addr, 143 unsigned long flags) 144 { 145 pmd_t new = pmdp_get(pmdp); 146 147 if (flags & SET_MEMORY_RO) 148 new = pmd_wrprotect(new); 149 else if (flags & SET_MEMORY_RW) 150 new = pmd_mkwrite_novma(pmd_mkdirty(new)); 151 if (flags & SET_MEMORY_NX) 152 new = set_pmd_bit(new, __pgprot(_SEGMENT_ENTRY_NOEXEC)); 153 else if (flags & SET_MEMORY_X) 154 new = clear_pmd_bit(new, __pgprot(_SEGMENT_ENTRY_NOEXEC)); 155 if (flags & SET_MEMORY_INV) { 156 new = set_pmd_bit(new, __pgprot(_SEGMENT_ENTRY_INVALID)); 157 } else if (flags & SET_MEMORY_DEF) { 158 new = __pmd(pmd_val(new) & PMD_MASK); 159 new = set_pmd_bit(new, SEGMENT_KERNEL); 160 } 161 pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); 162 } 163 164 static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end, 165 unsigned long flags) 166 { 167 unsigned long next; 168 pmd_t *pmdp, pmd; 169 int need_split; 170 int rc = 0; 171 172 pmdp = pmd_offset(pudp, addr); 173 do { 174 pmd = pmdp_get(pmdp); 175 if (pmd_none(pmd)) 176 return -EINVAL; 177 next = pmd_addr_end(addr, end); 178 if (pmd_leaf(pmd)) { 179 need_split = !!(flags & SET_MEMORY_4K); 180 need_split |= !!(addr & ~PMD_MASK); 181 need_split |= !!(addr + PMD_SIZE > next); 182 if (need_split) { 183 rc = split_pmd_page(pmdp, addr); 184 if (rc) 185 return rc; 186 continue; 187 } 188 modify_pmd_page(pmdp, addr, flags); 189 } else { 190 rc = walk_pte_level(pmdp, addr, next, flags); 191 if (rc) 192 return rc; 193 } 194 pmdp++; 195 addr = next; 196 } while (addr < end); 197 return rc; 198 } 199 200 int split_pud_page(pud_t *pudp, unsigned long addr) 201 { 202 unsigned long pmd_addr, prot; 203 pmd_t *pm_dir, *pmdp; 204 pud_t new, pud; 205 int i, ro, nx; 206 207 pm_dir = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY); 208 if (!pm_dir) 209 return -ENOMEM; 210 pud = pudp_get(pudp); 211 pmd_addr = pud_pfn(pud) << PAGE_SHIFT; 212 ro = !!(pud_val(pud) & _REGION_ENTRY_PROTECT); 213 nx = !!(pud_val(pud) & _REGION_ENTRY_NOEXEC); 214 prot = pgprot_val(ro ? SEGMENT_KERNEL_RO : SEGMENT_KERNEL); 215 if (!nx) 216 prot &= ~_SEGMENT_ENTRY_NOEXEC; 217 pmdp = pm_dir; 218 for (i = 0; i < PTRS_PER_PMD; i++) { 219 set_pmd(pmdp, __pmd(pmd_addr | prot)); 220 pmd_addr += PMD_SIZE; 221 pmdp++; 222 } 223 new = __pud(__pa(pm_dir) | _REGION3_ENTRY); 224 pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); 225 update_page_count(PG_DIRECT_MAP_1M, PTRS_PER_PMD); 226 update_page_count(PG_DIRECT_MAP_2G, -1); 227 return 0; 228 } 229 230 static void modify_pud_page(pud_t *pudp, unsigned long addr, 231 unsigned long flags) 232 { 233 pud_t new = pudp_get(pudp); 234 235 if (flags & SET_MEMORY_RO) 236 new = pud_wrprotect(new); 237 else if (flags & SET_MEMORY_RW) 238 new = pud_mkwrite(pud_mkdirty(new)); 239 if (flags & SET_MEMORY_NX) 240 new = set_pud_bit(new, __pgprot(_REGION_ENTRY_NOEXEC)); 241 else if (flags & SET_MEMORY_X) 242 new = clear_pud_bit(new, __pgprot(_REGION_ENTRY_NOEXEC)); 243 if (flags & SET_MEMORY_INV) { 244 new = set_pud_bit(new, __pgprot(_REGION_ENTRY_INVALID)); 245 } else if (flags & SET_MEMORY_DEF) { 246 new = __pud(pud_val(new) & PUD_MASK); 247 new = set_pud_bit(new, REGION3_KERNEL); 248 } 249 pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); 250 } 251 252 static int walk_pud_level(p4d_t *p4d, unsigned long addr, unsigned long end, 253 unsigned long flags) 254 { 255 unsigned long next; 256 pud_t *pudp, pud; 257 int need_split; 258 int rc = 0; 259 260 pudp = pud_offset(p4d, addr); 261 do { 262 pud = pudp_get(pudp); 263 if (pud_none(pud)) 264 return -EINVAL; 265 next = pud_addr_end(addr, end); 266 if (pud_leaf(pud)) { 267 need_split = !!(flags & SET_MEMORY_4K); 268 need_split |= !!(addr & ~PUD_MASK); 269 need_split |= !!(addr + PUD_SIZE > next); 270 if (need_split) { 271 rc = split_pud_page(pudp, addr); 272 if (rc) 273 break; 274 continue; 275 } 276 modify_pud_page(pudp, addr, flags); 277 } else { 278 rc = walk_pmd_level(pudp, addr, next, flags); 279 } 280 pudp++; 281 addr = next; 282 } while (addr < end && !rc); 283 return rc; 284 } 285 286 static int walk_p4d_level(pgd_t *pgd, unsigned long addr, unsigned long end, 287 unsigned long flags) 288 { 289 unsigned long next; 290 p4d_t *p4dp; 291 int rc = 0; 292 293 p4dp = p4d_offset(pgd, addr); 294 do { 295 if (p4d_none(p4dp_get(p4dp))) 296 return -EINVAL; 297 next = p4d_addr_end(addr, end); 298 rc = walk_pud_level(p4dp, addr, next, flags); 299 p4dp++; 300 addr = next; 301 } while (addr < end && !rc); 302 return rc; 303 } 304 305 DEFINE_MUTEX(cpa_mutex); 306 307 static int change_page_attr(unsigned long addr, unsigned long end, 308 unsigned long flags) 309 { 310 unsigned long next; 311 int rc = -EINVAL; 312 pgd_t *pgdp; 313 314 pgdp = pgd_offset_k(addr); 315 do { 316 if (pgd_none(pgdp_get(pgdp))) 317 break; 318 next = pgd_addr_end(addr, end); 319 rc = walk_p4d_level(pgdp, addr, next, flags); 320 if (rc) 321 break; 322 } while (pgdp++, addr = next, addr < end && !rc); 323 return rc; 324 } 325 326 static int change_page_attr_alias(unsigned long addr, unsigned long end, 327 unsigned long flags) 328 { 329 unsigned long alias, offset, va_start, va_end; 330 struct vm_struct *area; 331 int rc = 0; 332 333 /* 334 * Changes to read-only permissions on kernel VA mappings are also 335 * applied to the kernel direct mapping. Execute permissions are 336 * intentionally not transferred to keep all allocated pages within 337 * the direct mapping non-executable. 338 */ 339 flags &= SET_MEMORY_RO | SET_MEMORY_RW; 340 if (!flags) 341 return 0; 342 area = NULL; 343 while (addr < end) { 344 if (!area) 345 area = find_vm_area((void *)addr); 346 if (!area || !(area->flags & VM_ALLOC)) 347 return 0; 348 va_start = (unsigned long)area->addr; 349 va_end = va_start + area->nr_pages * PAGE_SIZE; 350 offset = (addr - va_start) >> PAGE_SHIFT; 351 alias = (unsigned long)page_address(area->pages[offset]); 352 rc = change_page_attr(alias, alias + PAGE_SIZE, flags); 353 if (rc) 354 break; 355 addr += PAGE_SIZE; 356 if (addr >= va_end) 357 area = NULL; 358 } 359 return rc; 360 } 361 362 int __set_memory(unsigned long addr, unsigned long numpages, unsigned long flags) 363 { 364 unsigned long end; 365 int rc; 366 367 if (!cpu_has_nx()) 368 flags &= ~(SET_MEMORY_NX | SET_MEMORY_X); 369 if (!flags) 370 return 0; 371 if (!numpages) 372 return 0; 373 addr &= PAGE_MASK; 374 end = addr + numpages * PAGE_SIZE; 375 mutex_lock(&cpa_mutex); 376 rc = change_page_attr(addr, end, flags); 377 if (rc) 378 goto out; 379 rc = change_page_attr_alias(addr, end, flags); 380 out: 381 mutex_unlock(&cpa_mutex); 382 return rc; 383 } 384 385 int set_direct_map_invalid_noflush(struct page *page) 386 { 387 return __set_memory((unsigned long)page_to_virt(page), 1, SET_MEMORY_INV); 388 } 389 390 int set_direct_map_default_noflush(struct page *page) 391 { 392 return __set_memory((unsigned long)page_to_virt(page), 1, SET_MEMORY_DEF); 393 } 394 395 int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) 396 { 397 unsigned long flags; 398 399 if (valid) 400 flags = SET_MEMORY_DEF; 401 else 402 flags = SET_MEMORY_INV; 403 404 return __set_memory((unsigned long)page_to_virt(page), nr, flags); 405 } 406 407 bool kernel_page_present(struct page *page) 408 { 409 unsigned long addr; 410 unsigned int cc; 411 412 addr = (unsigned long)page_address(page); 413 asm volatile( 414 " lra %[addr],0(%[addr])\n" 415 CC_IPM(cc) 416 : CC_OUT(cc, cc), [addr] "+a" (addr) 417 : 418 : CC_CLOBBER); 419 return CC_TRANSFORM(cc) == 0; 420 } 421 422 #if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE) 423 424 static void ipte_range(pte_t *pte, unsigned long address, int nr) 425 { 426 int i; 427 428 if (test_facility(13)) { 429 __ptep_ipte_range(address, nr - 1, pte, IPTE_GLOBAL); 430 return; 431 } 432 for (i = 0; i < nr; i++) { 433 __ptep_ipte(address, pte, 0, 0, IPTE_GLOBAL); 434 address += PAGE_SIZE; 435 pte++; 436 } 437 } 438 439 void __kernel_map_pages(struct page *page, int numpages, int enable) 440 { 441 unsigned long address; 442 pte_t *ptep, pte; 443 int nr, i, j; 444 445 for (i = 0; i < numpages;) { 446 address = (unsigned long)page_to_virt(page + i); 447 ptep = virt_to_kpte(address); 448 nr = (unsigned long)ptep >> ilog2(sizeof(long)); 449 nr = PTRS_PER_PTE - (nr & (PTRS_PER_PTE - 1)); 450 nr = min(numpages - i, nr); 451 if (enable) { 452 for (j = 0; j < nr; j++) { 453 pte = ptep_get(ptep); 454 pte = clear_pte_bit(pte, __pgprot(_PAGE_INVALID)); 455 set_pte(ptep, pte); 456 address += PAGE_SIZE; 457 ptep++; 458 } 459 } else { 460 ipte_range(ptep, address, nr); 461 } 462 i += nr; 463 } 464 } 465 466 #endif /* CONFIG_DEBUG_PAGEALLOC */ 467