1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright 2005, Paul Mackerras, IBM Corporation. 4 * Copyright 2009, Benjamin Herrenschmidt, IBM Corporation. 5 * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation. 6 */ 7 8 #include <linux/sched.h> 9 #include <linux/mm_types.h> 10 #include <linux/mm.h> 11 #include <linux/page_table_check.h> 12 #include <linux/stop_machine.h> 13 14 #include <asm/sections.h> 15 #include <asm/mmu.h> 16 #include <asm/tlb.h> 17 #include <asm/firmware.h> 18 19 #include <mm/mmu_decl.h> 20 21 #include <trace/events/thp.h> 22 23 #if H_PGTABLE_RANGE > (USER_VSID_RANGE * (TASK_SIZE_USER64 / TASK_CONTEXT_SIZE)) 24 #warning Limited user VSID range means pagetable space is wasted 25 #endif 26 27 #ifdef CONFIG_SPARSEMEM_VMEMMAP 28 /* 29 * vmemmap is the starting address of the virtual address space where 30 * struct pages are allocated for all possible PFNs present on the system 31 * including holes and bad memory (hence sparse). These virtual struct 32 * pages are stored in sequence in this virtual address space irrespective 33 * of the fact whether the corresponding PFN is valid or not. This achieves 34 * constant relationship between address of struct page and its PFN. 35 * 36 * During boot or memory hotplug operation when a new memory section is 37 * added, physical memory allocation (including hash table bolting) will 38 * be performed for the set of struct pages which are part of the memory 39 * section. This saves memory by not allocating struct pages for PFNs 40 * which are not valid. 41 * 42 * ---------------------------------------------- 43 * | PHYSICAL ALLOCATION OF VIRTUAL STRUCT PAGES| 44 * ---------------------------------------------- 45 * 46 * f000000000000000 c000000000000000 47 * vmemmap +--------------+ +--------------+ 48 * + | page struct | +--------------> | page struct | 49 * | +--------------+ +--------------+ 50 * | | page struct | +--------------> | page struct | 51 * | +--------------+ | +--------------+ 52 * | | page struct | + +------> | page struct | 53 * | +--------------+ | +--------------+ 54 * | | page struct | | +--> | page struct | 55 * | +--------------+ | | +--------------+ 56 * | | page struct | | | 57 * | +--------------+ | | 58 * | | page struct | | | 59 * | +--------------+ | | 60 * | | page struct | | | 61 * | +--------------+ | | 62 * | | page struct | | | 63 * | +--------------+ | | 64 * | | page struct | +-------+ | 65 * | +--------------+ | 66 * | | page struct | +-----------+ 67 * | +--------------+ 68 * | | page struct | No mapping 69 * | +--------------+ 70 * | | page struct | No mapping 71 * v +--------------+ 72 * 73 * ----------------------------------------- 74 * | RELATION BETWEEN STRUCT PAGES AND PFNS| 75 * ----------------------------------------- 76 * 77 * vmemmap +--------------+ +---------------+ 78 * + | page struct | +-------------> | PFN | 79 * | +--------------+ +---------------+ 80 * | | page struct | +-------------> | PFN | 81 * | +--------------+ +---------------+ 82 * | | page struct | +-------------> | PFN | 83 * | +--------------+ +---------------+ 84 * | | page struct | +-------------> | PFN | 85 * | +--------------+ +---------------+ 86 * | | | 87 * | +--------------+ 88 * | | | 89 * | +--------------+ 90 * | | | 91 * | +--------------+ +---------------+ 92 * | | page struct | +-------------> | PFN | 93 * | +--------------+ +---------------+ 94 * | | | 95 * | +--------------+ 96 * | | | 97 * | +--------------+ +---------------+ 98 * | | page struct | +-------------> | PFN | 99 * | +--------------+ +---------------+ 100 * | | page struct | +-------------> | PFN | 101 * v +--------------+ +---------------+ 102 */ 103 /* 104 * On hash-based CPUs, the vmemmap is bolted in the hash table. 105 * 106 */ 107 int __meminit hash__vmemmap_create_mapping(unsigned long start, 108 unsigned long page_size, 109 unsigned long phys) 110 { 111 int rc; 112 113 if ((start + page_size) >= H_VMEMMAP_END) { 114 pr_warn("Outside the supported range\n"); 115 return -1; 116 } 117 118 rc = htab_bolt_mapping(start, start + page_size, phys, 119 pgprot_val(PAGE_KERNEL), 120 mmu_vmemmap_psize, mmu_kernel_ssize); 121 if (rc < 0) { 122 int rc2 = htab_remove_mapping(start, start + page_size, 123 mmu_vmemmap_psize, 124 mmu_kernel_ssize); 125 BUG_ON(rc2 && (rc2 != -ENOENT)); 126 } 127 return rc; 128 } 129 130 #ifdef CONFIG_MEMORY_HOTPLUG 131 void hash__vmemmap_remove_mapping(unsigned long start, 132 unsigned long page_size) 133 { 134 int rc = htab_remove_mapping(start, start + page_size, 135 mmu_vmemmap_psize, 136 mmu_kernel_ssize); 137 BUG_ON((rc < 0) && (rc != -ENOENT)); 138 WARN_ON(rc == -ENOENT); 139 } 140 #endif 141 #endif /* CONFIG_SPARSEMEM_VMEMMAP */ 142 143 /* 144 * map_kernel_page currently only called by __ioremap 145 * map_kernel_page adds an entry to the ioremap page table 146 * and adds an entry to the HPT, possibly bolting it 147 */ 148 int hash__map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot) 149 { 150 pgd_t *pgdp; 151 p4d_t *p4dp; 152 pud_t *pudp; 153 pmd_t *pmdp; 154 pte_t *ptep; 155 156 BUILD_BUG_ON(TASK_SIZE_USER64 > H_PGTABLE_RANGE); 157 if (slab_is_available()) { 158 pgdp = pgd_offset_k(ea); 159 p4dp = p4d_offset(pgdp, ea); 160 pudp = pud_alloc(&init_mm, p4dp, ea); 161 if (!pudp) 162 return -ENOMEM; 163 pmdp = pmd_alloc(&init_mm, pudp, ea); 164 if (!pmdp) 165 return -ENOMEM; 166 ptep = pte_alloc_kernel(pmdp, ea); 167 if (!ptep) 168 return -ENOMEM; 169 set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, prot)); 170 } else { 171 /* 172 * If the mm subsystem is not fully up, we cannot create a 173 * linux page table entry for this mapping. Simply bolt an 174 * entry in the hardware page table. 175 * 176 */ 177 if (htab_bolt_mapping(ea, ea + PAGE_SIZE, pa, pgprot_val(prot), 178 mmu_io_psize, mmu_kernel_ssize)) { 179 printk(KERN_ERR "Failed to do bolted mapping IO " 180 "memory at %016lx !\n", pa); 181 return -ENOMEM; 182 } 183 } 184 185 smp_wmb(); 186 return 0; 187 } 188 189 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 190 191 unsigned long hash__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, 192 pmd_t *pmdp, unsigned long clr, 193 unsigned long set) 194 { 195 __be64 old_be, tmp; 196 unsigned long old; 197 198 #ifdef CONFIG_DEBUG_VM 199 WARN_ON(!hash__pmd_trans_huge(*pmdp)); 200 assert_spin_locked(pmd_lockptr(mm, pmdp)); 201 #endif 202 203 __asm__ __volatile__( 204 "1: ldarx %0,0,%3\n\ 205 and. %1,%0,%6\n\ 206 bne- 1b \n\ 207 andc %1,%0,%4 \n\ 208 or %1,%1,%7\n\ 209 stdcx. %1,0,%3 \n\ 210 bne- 1b" 211 : "=&r" (old_be), "=&r" (tmp), "=m" (*pmdp) 212 : "r" (pmdp), "r" (cpu_to_be64(clr)), "m" (*pmdp), 213 "r" (cpu_to_be64(H_PAGE_BUSY)), "r" (cpu_to_be64(set)) 214 : "cc" ); 215 216 old = be64_to_cpu(old_be); 217 218 trace_hugepage_update_pmd(addr, old, clr, set); 219 if (old & H_PAGE_HASHPTE) 220 hpte_do_hugepage_flush(mm, addr, pmdp, old); 221 return old; 222 } 223 224 static void do_nothing(void *arg) 225 { 226 227 } 228 229 /* 230 * Serialize against __find_linux_pte() which does lock-less 231 * lookup in page tables with local interrupts disabled. For huge pages 232 * it casts pmd_t to pte_t. Since format of pte_t is different from 233 * pmd_t we want to prevent transit from pmd pointing to page table 234 * to pmd pointing to huge page (and back) while interrupts are disabled. 235 * We clear pmd to possibly replace it with page table pointer in 236 * different code paths. So make sure we wait for the parallel 237 * __find_linux_pte() to finish. 238 */ 239 static void serialize_against_pte_lookup(struct mm_struct *mm) 240 { 241 smp_mb(); 242 smp_call_function_many(mm_cpumask(mm), do_nothing, mm, 1); 243 } 244 245 pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, 246 pmd_t *pmdp) 247 { 248 pmd_t pmd; 249 250 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 251 VM_BUG_ON(pmd_trans_huge(*pmdp)); 252 253 pmd = *pmdp; 254 pmd_clear(pmdp); 255 256 page_table_check_pmd_clear(vma->vm_mm, address, pmd); 257 258 /* 259 * Wait for all pending hash_page to finish. This is needed 260 * in case of subpage collapse. When we collapse normal pages 261 * to hugepage, we first clear the pmd, then invalidate all 262 * the PTE entries. The assumption here is that any low level 263 * page fault will see a none pmd and take the slow path that 264 * will wait on mmap_lock. But we could very well be in a 265 * hash_page with local ptep pointer value. Such a hash page 266 * can result in adding new HPTE entries for normal subpages. 267 * That means we could be modifying the page content as we 268 * copy them to a huge page. So wait for parallel hash_page 269 * to finish before invalidating HPTE entries. We can do this 270 * by sending an IPI to all the cpus and executing a dummy 271 * function there. 272 */ 273 serialize_against_pte_lookup(vma->vm_mm); 274 /* 275 * Now invalidate the hpte entries in the range 276 * covered by pmd. This make sure we take a 277 * fault and will find the pmd as none, which will 278 * result in a major fault which takes mmap_lock and 279 * hence wait for collapse to complete. Without this 280 * the __collapse_huge_page_copy can result in copying 281 * the old content. 282 */ 283 flush_hash_table_pmd_range(vma->vm_mm, &pmd, address); 284 return pmd; 285 } 286 287 /* 288 * We want to put the pgtable in pmd and use pgtable for tracking 289 * the base page size hptes 290 */ 291 void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, 292 pgtable_t pgtable) 293 { 294 pgtable_t *pgtable_slot; 295 296 assert_spin_locked(pmd_lockptr(mm, pmdp)); 297 /* 298 * we store the pgtable in the second half of PMD 299 */ 300 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD; 301 *pgtable_slot = pgtable; 302 /* 303 * expose the deposited pgtable to other cpus. 304 * before we set the hugepage PTE at pmd level 305 * hash fault code looks at the deposted pgtable 306 * to store hash index values. 307 */ 308 smp_wmb(); 309 } 310 311 pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) 312 { 313 pgtable_t pgtable; 314 pgtable_t *pgtable_slot; 315 316 assert_spin_locked(pmd_lockptr(mm, pmdp)); 317 318 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD; 319 pgtable = *pgtable_slot; 320 /* 321 * Once we withdraw, mark the entry NULL. 322 */ 323 *pgtable_slot = NULL; 324 /* 325 * We store HPTE information in the deposited PTE fragment. 326 * zero out the content on withdraw. 327 */ 328 memset(pgtable, 0, PTE_FRAG_SIZE); 329 return pgtable; 330 } 331 332 /* 333 * A linux hugepage PMD was changed and the corresponding hash table entries 334 * neesd to be flushed. 335 */ 336 void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr, 337 pmd_t *pmdp, unsigned long old_pmd) 338 { 339 int ssize; 340 unsigned int psize; 341 unsigned long vsid; 342 unsigned long flags = 0; 343 344 /* get the base page size,vsid and segment size */ 345 #ifdef CONFIG_DEBUG_VM 346 psize = get_slice_psize(mm, addr); 347 BUG_ON(psize == MMU_PAGE_16M); 348 #endif 349 if (old_pmd & H_PAGE_COMBO) 350 psize = MMU_PAGE_4K; 351 else 352 psize = MMU_PAGE_64K; 353 354 if (!is_kernel_addr(addr)) { 355 ssize = user_segment_size(addr); 356 vsid = get_user_vsid(&mm->context, addr, ssize); 357 WARN_ON(vsid == 0); 358 } else { 359 vsid = get_kernel_vsid(addr, mmu_kernel_ssize); 360 ssize = mmu_kernel_ssize; 361 } 362 363 if (mm_is_thread_local(mm)) 364 flags |= HPTE_LOCAL_UPDATE; 365 366 return flush_hash_hugepage(vsid, addr, pmdp, psize, ssize, flags); 367 } 368 369 pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm, 370 unsigned long addr, pmd_t *pmdp) 371 { 372 pmd_t old_pmd; 373 pgtable_t pgtable; 374 unsigned long old; 375 pgtable_t *pgtable_slot; 376 377 old = pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0); 378 old_pmd = __pmd(old); 379 /* 380 * We have pmd == none and we are holding page_table_lock. 381 * So we can safely go and clear the pgtable hash 382 * index info. 383 */ 384 pgtable_slot = (pgtable_t *)pmdp + PTRS_PER_PMD; 385 pgtable = *pgtable_slot; 386 /* 387 * Let's zero out old valid and hash index details 388 * hash fault look at them. 389 */ 390 memset(pgtable, 0, PTE_FRAG_SIZE); 391 return old_pmd; 392 } 393 394 int hash__has_transparent_hugepage(void) 395 { 396 397 if (!mmu_has_feature(MMU_FTR_16M_PAGE)) 398 return 0; 399 /* 400 * We support THP only if PMD_SIZE is 16MB. 401 */ 402 if (mmu_psize_defs[MMU_PAGE_16M].shift != PMD_SHIFT) 403 return 0; 404 /* 405 * We need to make sure that we support 16MB hugepage in a segment 406 * with base page size 64K or 4K. We only enable THP with a PAGE_SIZE 407 * of 64K. 408 */ 409 /* 410 * If we have 64K HPTE, we will be using that by default 411 */ 412 if (mmu_psize_defs[MMU_PAGE_64K].shift && 413 (mmu_psize_defs[MMU_PAGE_64K].penc[MMU_PAGE_16M] == -1)) 414 return 0; 415 /* 416 * Ok we only have 4K HPTE 417 */ 418 if (mmu_psize_defs[MMU_PAGE_4K].penc[MMU_PAGE_16M] == -1) 419 return 0; 420 421 return 1; 422 } 423 EXPORT_SYMBOL_GPL(hash__has_transparent_hugepage); 424 425 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 426 427 #ifdef CONFIG_STRICT_KERNEL_RWX 428 429 struct change_memory_parms { 430 unsigned long start, end, newpp; 431 unsigned int step, nr_cpus; 432 atomic_t master_cpu; 433 atomic_t cpu_counter; 434 }; 435 436 // We'd rather this was on the stack but it has to be in the RMO 437 static struct change_memory_parms chmem_parms; 438 439 // And therefore we need a lock to protect it from concurrent use 440 static DEFINE_MUTEX(chmem_lock); 441 442 static void change_memory_range(unsigned long start, unsigned long end, 443 unsigned int step, unsigned long newpp) 444 { 445 unsigned long idx; 446 447 pr_debug("Changing page protection on range 0x%lx-0x%lx, to 0x%lx, step 0x%x\n", 448 start, end, newpp, step); 449 450 for (idx = start; idx < end; idx += step) 451 /* Not sure if we can do much with the return value */ 452 mmu_hash_ops.hpte_updateboltedpp(newpp, idx, mmu_linear_psize, 453 mmu_kernel_ssize); 454 } 455 456 static int notrace chmem_secondary_loop(struct change_memory_parms *parms) 457 { 458 unsigned long msr, tmp, flags; 459 int *p; 460 461 p = &parms->cpu_counter.counter; 462 463 local_irq_save(flags); 464 hard_irq_disable(); 465 466 asm volatile ( 467 // Switch to real mode and leave interrupts off 468 "mfmsr %[msr] ;" 469 "li %[tmp], %[MSR_IR_DR] ;" 470 "andc %[tmp], %[msr], %[tmp] ;" 471 "mtmsrd %[tmp] ;" 472 473 // Tell the master we are in real mode 474 "1: " 475 "lwarx %[tmp], 0, %[p] ;" 476 "addic %[tmp], %[tmp], -1 ;" 477 "stwcx. %[tmp], 0, %[p] ;" 478 "bne- 1b ;" 479 480 // Spin until the counter goes to zero 481 "2: ;" 482 "lwz %[tmp], 0(%[p]) ;" 483 "cmpwi %[tmp], 0 ;" 484 "bne- 2b ;" 485 486 // Switch back to virtual mode 487 "mtmsrd %[msr] ;" 488 489 : // outputs 490 [msr] "=&r" (msr), [tmp] "=&b" (tmp), "+m" (*p) 491 : // inputs 492 [p] "b" (p), [MSR_IR_DR] "i" (MSR_IR | MSR_DR) 493 : // clobbers 494 "cc", "xer" 495 ); 496 497 local_irq_restore(flags); 498 499 return 0; 500 } 501 502 static int change_memory_range_fn(void *data) 503 { 504 struct change_memory_parms *parms = data; 505 506 // First CPU goes through, all others wait. 507 if (atomic_xchg(&parms->master_cpu, 1) == 1) 508 return chmem_secondary_loop(parms); 509 510 // Wait for all but one CPU (this one) to call-in 511 while (atomic_read(&parms->cpu_counter) > 1) 512 barrier(); 513 514 change_memory_range(parms->start, parms->end, parms->step, parms->newpp); 515 516 mb(); 517 518 // Signal the other CPUs that we're done 519 atomic_dec(&parms->cpu_counter); 520 521 return 0; 522 } 523 524 static bool hash__change_memory_range(unsigned long start, unsigned long end, 525 unsigned long newpp) 526 { 527 unsigned int step, shift; 528 529 shift = mmu_psize_defs[mmu_linear_psize].shift; 530 step = 1 << shift; 531 532 start = ALIGN_DOWN(start, step); 533 end = ALIGN(end, step); // aligns up 534 535 if (start >= end) 536 return false; 537 538 if (firmware_has_feature(FW_FEATURE_LPAR)) { 539 mutex_lock(&chmem_lock); 540 541 chmem_parms.start = start; 542 chmem_parms.end = end; 543 chmem_parms.step = step; 544 chmem_parms.newpp = newpp; 545 atomic_set(&chmem_parms.master_cpu, 0); 546 547 cpus_read_lock(); 548 549 atomic_set(&chmem_parms.cpu_counter, num_online_cpus()); 550 551 // Ensure state is consistent before we call the other CPUs 552 mb(); 553 554 stop_machine_cpuslocked(change_memory_range_fn, &chmem_parms, 555 cpu_online_mask); 556 557 cpus_read_unlock(); 558 mutex_unlock(&chmem_lock); 559 } else 560 change_memory_range(start, end, step, newpp); 561 562 return true; 563 } 564 565 void hash__mark_rodata_ro(void) 566 { 567 unsigned long start, end, pp; 568 569 start = (unsigned long)_stext; 570 end = (unsigned long)__end_rodata; 571 572 pp = htab_convert_pte_flags(pgprot_val(PAGE_KERNEL_ROX), HPTE_USE_KERNEL_KEY); 573 574 WARN_ON(!hash__change_memory_range(start, end, pp)); 575 } 576 577 void hash__mark_initmem_nx(void) 578 { 579 unsigned long start, end, pp; 580 581 start = (unsigned long)__init_begin; 582 end = (unsigned long)__init_end; 583 584 pp = htab_convert_pte_flags(pgprot_val(PAGE_KERNEL), HPTE_USE_KERNEL_KEY); 585 586 WARN_ON(!hash__change_memory_range(start, end, pp)); 587 } 588 #endif 589