1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Page table allocation functions 4 * 5 * Copyright IBM Corp. 2016 6 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 7 */ 8 9 #include <linux/sysctl.h> 10 #include <linux/slab.h> 11 #include <linux/mm.h> 12 #include <asm/mmu_context.h> 13 #include <asm/page-states.h> 14 #include <asm/pgalloc.h> 15 #include <asm/tlbflush.h> 16 17 unsigned long *crst_table_alloc_noprof(struct mm_struct *mm) 18 { 19 gfp_t gfp = GFP_KERNEL_ACCOUNT; 20 struct ptdesc *ptdesc; 21 unsigned long *table; 22 23 if (mm == &init_mm) 24 gfp &= ~__GFP_ACCOUNT; 25 ptdesc = pagetable_alloc_noprof(gfp, CRST_ALLOC_ORDER); 26 if (!ptdesc) 27 return NULL; 28 table = ptdesc_address(ptdesc); 29 __arch_set_page_dat(table, 1UL << CRST_ALLOC_ORDER); 30 return table; 31 } 32 33 void crst_table_free(struct mm_struct *mm, unsigned long *table) 34 { 35 if (!table) 36 return; 37 pagetable_free(virt_to_ptdesc(table)); 38 } 39 40 static void __crst_table_upgrade(void *arg) 41 { 42 struct mm_struct *mm = arg; 43 struct ctlreg asce; 44 45 /* change all active ASCEs to avoid the creation of new TLBs */ 46 if (current->active_mm == mm) { 47 asce.val = mm->context.asce; 48 get_lowcore()->user_asce = asce; 49 local_ctl_load(7, &asce); 50 if (!test_thread_flag(TIF_ASCE_PRIMARY)) 51 local_ctl_load(1, &asce); 52 } 53 __tlb_flush_local(); 54 } 55 56 int crst_table_upgrade(struct mm_struct *mm, unsigned long end) 57 { 58 unsigned long *table, *pgd; 59 int rc, notify; 60 61 mmap_assert_write_locked(mm); 62 /* upgrade should only happen from 3 to 4, 3 to 5, or 4 to 5 levels */ 63 VM_BUG_ON(mm->context.asce_limit < _REGION2_SIZE); 64 rc = 0; 65 notify = 0; 66 while (mm->context.asce_limit < end) { 67 table = crst_table_alloc(mm); 68 if (!table) { 69 rc = -ENOMEM; 70 break; 71 } 72 spin_lock_bh(&mm->page_table_lock); 73 pgd = (unsigned long *)mm->pgd; 74 if (mm->context.asce_limit == _REGION2_SIZE) { 75 crst_table_init(table, _REGION2_ENTRY_EMPTY); 76 p4d_populate(mm, (p4d_t *)table, (pud_t *)pgd); 77 pagetable_p4d_ctor(virt_to_ptdesc(table)); 78 mm->pgd = (pgd_t *)table; 79 mm->context.asce_limit = _REGION1_SIZE; 80 mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | 81 _ASCE_USER_BITS | _ASCE_TYPE_REGION2; 82 mm_inc_nr_puds(mm); 83 } else { 84 crst_table_init(table, _REGION1_ENTRY_EMPTY); 85 pgd_populate(mm, (pgd_t *)table, (p4d_t *)pgd); 86 pagetable_pgd_ctor(virt_to_ptdesc(table)); 87 mm->pgd = (pgd_t *)table; 88 mm->context.asce_limit = TASK_SIZE_MAX; 89 mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | 90 _ASCE_USER_BITS | _ASCE_TYPE_REGION1; 91 } 92 notify = 1; 93 spin_unlock_bh(&mm->page_table_lock); 94 } 95 if (notify) 96 on_each_cpu(__crst_table_upgrade, mm, 0); 97 return rc; 98 } 99 100 unsigned long *page_table_alloc_noprof(struct mm_struct *mm) 101 { 102 gfp_t gfp = GFP_KERNEL_ACCOUNT; 103 struct ptdesc *ptdesc; 104 unsigned long *table; 105 106 if (mm == &init_mm) 107 gfp &= ~__GFP_ACCOUNT; 108 ptdesc = pagetable_alloc_noprof(gfp, 0); 109 if (!ptdesc) 110 return NULL; 111 if (!pagetable_pte_ctor(mm, ptdesc)) { 112 pagetable_free(ptdesc); 113 return NULL; 114 } 115 table = ptdesc_address(ptdesc); 116 __arch_set_page_dat(table, 1); 117 memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE); 118 memset64((u64 *)table + PTRS_PER_PTE, 0, PTRS_PER_PTE); 119 return table; 120 } 121 122 void page_table_free(struct mm_struct *mm, unsigned long *table) 123 { 124 struct ptdesc *ptdesc = virt_to_ptdesc(table); 125 126 if (pagetable_is_reserved(ptdesc)) 127 return free_reserved_ptdesc(ptdesc); 128 pagetable_dtor_free(ptdesc); 129 } 130 131 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 132 static void pte_free_now(struct rcu_head *head) 133 { 134 struct ptdesc *ptdesc = container_of(head, struct ptdesc, pt_rcu_head); 135 136 pagetable_dtor_free(ptdesc); 137 } 138 139 void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable) 140 { 141 struct ptdesc *ptdesc = virt_to_ptdesc(pgtable); 142 143 call_rcu(&ptdesc->pt_rcu_head, pte_free_now); 144 } 145 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 146 147 /* 148 * Base infrastructure required to generate basic asces, region, segment, 149 * and page tables that do not make use of enhanced features like EDAT1. 150 */ 151 152 static struct kmem_cache *base_pgt_cache; 153 154 static unsigned long *base_pgt_alloc(void) 155 { 156 unsigned long *table; 157 158 table = kmem_cache_alloc(base_pgt_cache, GFP_KERNEL); 159 if (table) 160 memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE); 161 return table; 162 } 163 164 static void base_pgt_free(unsigned long *table) 165 { 166 kmem_cache_free(base_pgt_cache, table); 167 } 168 169 static unsigned long *base_crst_alloc(unsigned long val) 170 { 171 unsigned long *table; 172 struct ptdesc *ptdesc; 173 174 ptdesc = pagetable_alloc(GFP_KERNEL, CRST_ALLOC_ORDER); 175 if (!ptdesc) 176 return NULL; 177 table = ptdesc_address(ptdesc); 178 crst_table_init(table, val); 179 return table; 180 } 181 182 static void base_crst_free(unsigned long *table) 183 { 184 if (!table) 185 return; 186 pagetable_free(virt_to_ptdesc(table)); 187 } 188 189 #define BASE_ADDR_END_FUNC(NAME, SIZE) \ 190 static inline unsigned long base_##NAME##_addr_end(unsigned long addr, \ 191 unsigned long end) \ 192 { \ 193 unsigned long next = (addr + (SIZE)) & ~((SIZE) - 1); \ 194 \ 195 return (next - 1) < (end - 1) ? next : end; \ 196 } 197 198 BASE_ADDR_END_FUNC(page, PAGE_SIZE) 199 BASE_ADDR_END_FUNC(segment, _SEGMENT_SIZE) 200 BASE_ADDR_END_FUNC(region3, _REGION3_SIZE) 201 BASE_ADDR_END_FUNC(region2, _REGION2_SIZE) 202 BASE_ADDR_END_FUNC(region1, _REGION1_SIZE) 203 204 static inline unsigned long base_lra(unsigned long address) 205 { 206 unsigned long real; 207 208 asm volatile( 209 " lra %0,0(%1)" 210 : "=d" (real) : "a" (address) : "cc"); 211 return real; 212 } 213 214 static int base_page_walk(unsigned long *origin, unsigned long addr, 215 unsigned long end, int alloc) 216 { 217 unsigned long *pte, next; 218 219 if (!alloc) 220 return 0; 221 pte = origin; 222 pte += (addr & _PAGE_INDEX) >> PAGE_SHIFT; 223 do { 224 next = base_page_addr_end(addr, end); 225 *pte = base_lra(addr); 226 } while (pte++, addr = next, addr < end); 227 return 0; 228 } 229 230 static int base_segment_walk(unsigned long *origin, unsigned long addr, 231 unsigned long end, int alloc) 232 { 233 unsigned long *ste, next, *table; 234 int rc; 235 236 ste = origin; 237 ste += (addr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT; 238 do { 239 next = base_segment_addr_end(addr, end); 240 if (*ste & _SEGMENT_ENTRY_INVALID) { 241 if (!alloc) 242 continue; 243 table = base_pgt_alloc(); 244 if (!table) 245 return -ENOMEM; 246 *ste = __pa(table) | _SEGMENT_ENTRY; 247 } 248 table = __va(*ste & _SEGMENT_ENTRY_ORIGIN); 249 rc = base_page_walk(table, addr, next, alloc); 250 if (rc) 251 return rc; 252 if (!alloc) 253 base_pgt_free(table); 254 } while (ste++, addr = next, addr < end); 255 return 0; 256 } 257 258 static int base_region3_walk(unsigned long *origin, unsigned long addr, 259 unsigned long end, int alloc) 260 { 261 unsigned long *rtte, next, *table; 262 int rc; 263 264 rtte = origin; 265 rtte += (addr & _REGION3_INDEX) >> _REGION3_SHIFT; 266 do { 267 next = base_region3_addr_end(addr, end); 268 if (*rtte & _REGION_ENTRY_INVALID) { 269 if (!alloc) 270 continue; 271 table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY); 272 if (!table) 273 return -ENOMEM; 274 *rtte = __pa(table) | _REGION3_ENTRY; 275 } 276 table = __va(*rtte & _REGION_ENTRY_ORIGIN); 277 rc = base_segment_walk(table, addr, next, alloc); 278 if (rc) 279 return rc; 280 if (!alloc) 281 base_crst_free(table); 282 } while (rtte++, addr = next, addr < end); 283 return 0; 284 } 285 286 static int base_region2_walk(unsigned long *origin, unsigned long addr, 287 unsigned long end, int alloc) 288 { 289 unsigned long *rste, next, *table; 290 int rc; 291 292 rste = origin; 293 rste += (addr & _REGION2_INDEX) >> _REGION2_SHIFT; 294 do { 295 next = base_region2_addr_end(addr, end); 296 if (*rste & _REGION_ENTRY_INVALID) { 297 if (!alloc) 298 continue; 299 table = base_crst_alloc(_REGION3_ENTRY_EMPTY); 300 if (!table) 301 return -ENOMEM; 302 *rste = __pa(table) | _REGION2_ENTRY; 303 } 304 table = __va(*rste & _REGION_ENTRY_ORIGIN); 305 rc = base_region3_walk(table, addr, next, alloc); 306 if (rc) 307 return rc; 308 if (!alloc) 309 base_crst_free(table); 310 } while (rste++, addr = next, addr < end); 311 return 0; 312 } 313 314 static int base_region1_walk(unsigned long *origin, unsigned long addr, 315 unsigned long end, int alloc) 316 { 317 unsigned long *rfte, next, *table; 318 int rc; 319 320 rfte = origin; 321 rfte += (addr & _REGION1_INDEX) >> _REGION1_SHIFT; 322 do { 323 next = base_region1_addr_end(addr, end); 324 if (*rfte & _REGION_ENTRY_INVALID) { 325 if (!alloc) 326 continue; 327 table = base_crst_alloc(_REGION2_ENTRY_EMPTY); 328 if (!table) 329 return -ENOMEM; 330 *rfte = __pa(table) | _REGION1_ENTRY; 331 } 332 table = __va(*rfte & _REGION_ENTRY_ORIGIN); 333 rc = base_region2_walk(table, addr, next, alloc); 334 if (rc) 335 return rc; 336 if (!alloc) 337 base_crst_free(table); 338 } while (rfte++, addr = next, addr < end); 339 return 0; 340 } 341 342 /** 343 * base_asce_free - free asce and tables returned from base_asce_alloc() 344 * @asce: asce to be freed 345 * 346 * Frees all region, segment, and page tables that were allocated with a 347 * corresponding base_asce_alloc() call. 348 */ 349 void base_asce_free(unsigned long asce) 350 { 351 unsigned long *table = __va(asce & _ASCE_ORIGIN); 352 353 if (!asce) 354 return; 355 switch (asce & _ASCE_TYPE_MASK) { 356 case _ASCE_TYPE_SEGMENT: 357 base_segment_walk(table, 0, _REGION3_SIZE, 0); 358 break; 359 case _ASCE_TYPE_REGION3: 360 base_region3_walk(table, 0, _REGION2_SIZE, 0); 361 break; 362 case _ASCE_TYPE_REGION2: 363 base_region2_walk(table, 0, _REGION1_SIZE, 0); 364 break; 365 case _ASCE_TYPE_REGION1: 366 base_region1_walk(table, 0, TASK_SIZE_MAX, 0); 367 break; 368 } 369 base_crst_free(table); 370 } 371 372 static int base_pgt_cache_init(void) 373 { 374 static DEFINE_MUTEX(base_pgt_cache_mutex); 375 unsigned long sz = _PAGE_TABLE_SIZE; 376 377 if (base_pgt_cache) 378 return 0; 379 mutex_lock(&base_pgt_cache_mutex); 380 if (!base_pgt_cache) 381 base_pgt_cache = kmem_cache_create("base_pgt", sz, sz, 0, NULL); 382 mutex_unlock(&base_pgt_cache_mutex); 383 return base_pgt_cache ? 0 : -ENOMEM; 384 } 385 386 /** 387 * base_asce_alloc - create kernel mapping without enhanced DAT features 388 * @addr: virtual start address of kernel mapping 389 * @num_pages: number of consecutive pages 390 * 391 * Generate an asce, including all required region, segment and page tables, 392 * that can be used to access the virtual kernel mapping. The difference is 393 * that the returned asce does not make use of any enhanced DAT features like 394 * e.g. large pages. This is required for some I/O functions that pass an 395 * asce, like e.g. some service call requests. 396 * 397 * Note: the returned asce may NEVER be attached to any cpu. It may only be 398 * used for I/O requests. tlb entries that might result because the 399 * asce was attached to a cpu won't be cleared. 400 */ 401 unsigned long base_asce_alloc(unsigned long addr, unsigned long num_pages) 402 { 403 unsigned long asce, *table, end; 404 int rc; 405 406 if (base_pgt_cache_init()) 407 return 0; 408 end = addr + num_pages * PAGE_SIZE; 409 if (end <= _REGION3_SIZE) { 410 table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY); 411 if (!table) 412 return 0; 413 rc = base_segment_walk(table, addr, end, 1); 414 asce = __pa(table) | _ASCE_TYPE_SEGMENT | _ASCE_TABLE_LENGTH; 415 } else if (end <= _REGION2_SIZE) { 416 table = base_crst_alloc(_REGION3_ENTRY_EMPTY); 417 if (!table) 418 return 0; 419 rc = base_region3_walk(table, addr, end, 1); 420 asce = __pa(table) | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; 421 } else if (end <= _REGION1_SIZE) { 422 table = base_crst_alloc(_REGION2_ENTRY_EMPTY); 423 if (!table) 424 return 0; 425 rc = base_region2_walk(table, addr, end, 1); 426 asce = __pa(table) | _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH; 427 } else { 428 table = base_crst_alloc(_REGION1_ENTRY_EMPTY); 429 if (!table) 430 return 0; 431 rc = base_region1_walk(table, addr, end, 1); 432 asce = __pa(table) | _ASCE_TYPE_REGION1 | _ASCE_TABLE_LENGTH; 433 } 434 if (rc) { 435 base_asce_free(asce); 436 asce = 0; 437 } 438 return asce; 439 } 440