1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * PowerPC64 port by Mike Corrigan and Dave Engebretsen 4 * {mikejc|engebret}@us.ibm.com 5 * 6 * Copyright (c) 2000 Mike Corrigan <mikejc@us.ibm.com> 7 * 8 * SMP scalability work: 9 * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM 10 * 11 * Module name: htab.c 12 * 13 * Description: 14 * PowerPC Hashed Page Table functions 15 */ 16 17 #undef DEBUG 18 #undef DEBUG_LOW 19 20 #define pr_fmt(fmt) "hash-mmu: " fmt 21 #include <linux/spinlock.h> 22 #include <linux/errno.h> 23 #include <linux/sched/mm.h> 24 #include <linux/proc_fs.h> 25 #include <linux/stat.h> 26 #include <linux/sysctl.h> 27 #include <linux/export.h> 28 #include <linux/ctype.h> 29 #include <linux/cache.h> 30 #include <linux/init.h> 31 #include <linux/signal.h> 32 #include <linux/memblock.h> 33 #include <linux/context_tracking.h> 34 #include <linux/libfdt.h> 35 #include <linux/pkeys.h> 36 #include <linux/hugetlb.h> 37 #include <linux/cpu.h> 38 39 #include <asm/debugfs.h> 40 #include <asm/processor.h> 41 #include <asm/pgtable.h> 42 #include <asm/mmu.h> 43 #include <asm/mmu_context.h> 44 #include <asm/page.h> 45 #include <asm/types.h> 46 #include <linux/uaccess.h> 47 #include <asm/machdep.h> 48 #include <asm/prom.h> 49 #include <asm/io.h> 50 #include <asm/eeh.h> 51 #include <asm/tlb.h> 52 #include <asm/cacheflush.h> 53 #include <asm/cputable.h> 54 #include <asm/sections.h> 55 #include <asm/copro.h> 56 #include <asm/udbg.h> 57 #include <asm/code-patching.h> 58 #include <asm/fadump.h> 59 #include <asm/firmware.h> 60 #include <asm/tm.h> 61 #include <asm/trace.h> 62 #include <asm/ps3.h> 63 #include <asm/pte-walk.h> 64 #include <asm/asm-prototypes.h> 65 #include <asm/ultravisor.h> 66 67 #include <mm/mmu_decl.h> 68 69 #ifdef DEBUG 70 #define DBG(fmt...) udbg_printf(fmt) 71 #else 72 #define DBG(fmt...) 73 #endif 74 75 #ifdef DEBUG_LOW 76 #define DBG_LOW(fmt...) udbg_printf(fmt) 77 #else 78 #define DBG_LOW(fmt...) 79 #endif 80 81 #define KB (1024) 82 #define MB (1024*KB) 83 #define GB (1024L*MB) 84 85 /* 86 * Note: pte --> Linux PTE 87 * HPTE --> PowerPC Hashed Page Table Entry 88 * 89 * Execution context: 90 * htab_initialize is called with the MMU off (of course), but 91 * the kernel has been copied down to zero so it can directly 92 * reference global data. At this point it is very difficult 93 * to print debug info. 94 * 95 */ 96 97 static unsigned long _SDR1; 98 struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT]; 99 EXPORT_SYMBOL_GPL(mmu_psize_defs); 100 101 u8 hpte_page_sizes[1 << LP_BITS]; 102 EXPORT_SYMBOL_GPL(hpte_page_sizes); 103 104 struct hash_pte *htab_address; 105 unsigned long htab_size_bytes; 106 unsigned long htab_hash_mask; 107 EXPORT_SYMBOL_GPL(htab_hash_mask); 108 int mmu_linear_psize = MMU_PAGE_4K; 109 EXPORT_SYMBOL_GPL(mmu_linear_psize); 110 int mmu_virtual_psize = MMU_PAGE_4K; 111 int mmu_vmalloc_psize = MMU_PAGE_4K; 112 #ifdef CONFIG_SPARSEMEM_VMEMMAP 113 int mmu_vmemmap_psize = MMU_PAGE_4K; 114 #endif 115 int mmu_io_psize = MMU_PAGE_4K; 116 int mmu_kernel_ssize = MMU_SEGSIZE_256M; 117 EXPORT_SYMBOL_GPL(mmu_kernel_ssize); 118 int mmu_highuser_ssize = MMU_SEGSIZE_256M; 119 u16 mmu_slb_size = 64; 120 EXPORT_SYMBOL_GPL(mmu_slb_size); 121 #ifdef CONFIG_PPC_64K_PAGES 122 int mmu_ci_restrictions; 123 #endif 124 #ifdef CONFIG_DEBUG_PAGEALLOC 125 static u8 *linear_map_hash_slots; 126 static unsigned long linear_map_hash_count; 127 static DEFINE_SPINLOCK(linear_map_hash_lock); 128 #endif /* CONFIG_DEBUG_PAGEALLOC */ 129 struct mmu_hash_ops mmu_hash_ops; 130 EXPORT_SYMBOL(mmu_hash_ops); 131 132 /* 133 * These are definitions of page sizes arrays to be used when none 134 * is provided by the firmware. 135 */ 136 137 /* 138 * Fallback (4k pages only) 139 */ 140 static struct mmu_psize_def mmu_psize_defaults[] = { 141 [MMU_PAGE_4K] = { 142 .shift = 12, 143 .sllp = 0, 144 .penc = {[MMU_PAGE_4K] = 0, [1 ... MMU_PAGE_COUNT - 1] = -1}, 145 .avpnm = 0, 146 .tlbiel = 0, 147 }, 148 }; 149 150 /* 151 * POWER4, GPUL, POWER5 152 * 153 * Support for 16Mb large pages 154 */ 155 static struct mmu_psize_def mmu_psize_defaults_gp[] = { 156 [MMU_PAGE_4K] = { 157 .shift = 12, 158 .sllp = 0, 159 .penc = {[MMU_PAGE_4K] = 0, [1 ... MMU_PAGE_COUNT - 1] = -1}, 160 .avpnm = 0, 161 .tlbiel = 1, 162 }, 163 [MMU_PAGE_16M] = { 164 .shift = 24, 165 .sllp = SLB_VSID_L, 166 .penc = {[0 ... MMU_PAGE_16M - 1] = -1, [MMU_PAGE_16M] = 0, 167 [MMU_PAGE_16M + 1 ... MMU_PAGE_COUNT - 1] = -1 }, 168 .avpnm = 0x1UL, 169 .tlbiel = 0, 170 }, 171 }; 172 173 /* 174 * 'R' and 'C' update notes: 175 * - Under pHyp or KVM, the updatepp path will not set C, thus it *will* 176 * create writeable HPTEs without C set, because the hcall H_PROTECT 177 * that we use in that case will not update C 178 * - The above is however not a problem, because we also don't do that 179 * fancy "no flush" variant of eviction and we use H_REMOVE which will 180 * do the right thing and thus we don't have the race I described earlier 181 * 182 * - Under bare metal, we do have the race, so we need R and C set 183 * - We make sure R is always set and never lost 184 * - C is _PAGE_DIRTY, and *should* always be set for a writeable mapping 185 */ 186 unsigned long htab_convert_pte_flags(unsigned long pteflags) 187 { 188 unsigned long rflags = 0; 189 190 /* _PAGE_EXEC -> NOEXEC */ 191 if ((pteflags & _PAGE_EXEC) == 0) 192 rflags |= HPTE_R_N; 193 /* 194 * PPP bits: 195 * Linux uses slb key 0 for kernel and 1 for user. 196 * kernel RW areas are mapped with PPP=0b000 197 * User area is mapped with PPP=0b010 for read/write 198 * or PPP=0b011 for read-only (including writeable but clean pages). 199 */ 200 if (pteflags & _PAGE_PRIVILEGED) { 201 /* 202 * Kernel read only mapped with ppp bits 0b110 203 */ 204 if (!(pteflags & _PAGE_WRITE)) { 205 if (mmu_has_feature(MMU_FTR_KERNEL_RO)) 206 rflags |= (HPTE_R_PP0 | 0x2); 207 else 208 rflags |= 0x3; 209 } 210 } else { 211 if (pteflags & _PAGE_RWX) 212 rflags |= 0x2; 213 if (!((pteflags & _PAGE_WRITE) && (pteflags & _PAGE_DIRTY))) 214 rflags |= 0x1; 215 } 216 /* 217 * We can't allow hardware to update hpte bits. Hence always 218 * set 'R' bit and set 'C' if it is a write fault 219 */ 220 rflags |= HPTE_R_R; 221 222 if (pteflags & _PAGE_DIRTY) 223 rflags |= HPTE_R_C; 224 /* 225 * Add in WIG bits 226 */ 227 228 if ((pteflags & _PAGE_CACHE_CTL) == _PAGE_TOLERANT) 229 rflags |= HPTE_R_I; 230 else if ((pteflags & _PAGE_CACHE_CTL) == _PAGE_NON_IDEMPOTENT) 231 rflags |= (HPTE_R_I | HPTE_R_G); 232 else if ((pteflags & _PAGE_CACHE_CTL) == _PAGE_SAO) 233 rflags |= (HPTE_R_W | HPTE_R_I | HPTE_R_M); 234 else 235 /* 236 * Add memory coherence if cache inhibited is not set 237 */ 238 rflags |= HPTE_R_M; 239 240 rflags |= pte_to_hpte_pkey_bits(pteflags); 241 return rflags; 242 } 243 244 int htab_bolt_mapping(unsigned long vstart, unsigned long vend, 245 unsigned long pstart, unsigned long prot, 246 int psize, int ssize) 247 { 248 unsigned long vaddr, paddr; 249 unsigned int step, shift; 250 int ret = 0; 251 252 shift = mmu_psize_defs[psize].shift; 253 step = 1 << shift; 254 255 prot = htab_convert_pte_flags(prot); 256 257 DBG("htab_bolt_mapping(%lx..%lx -> %lx (%lx,%d,%d)\n", 258 vstart, vend, pstart, prot, psize, ssize); 259 260 for (vaddr = vstart, paddr = pstart; vaddr < vend; 261 vaddr += step, paddr += step) { 262 unsigned long hash, hpteg; 263 unsigned long vsid = get_kernel_vsid(vaddr, ssize); 264 unsigned long vpn = hpt_vpn(vaddr, vsid, ssize); 265 unsigned long tprot = prot; 266 bool secondary_hash = false; 267 268 /* 269 * If we hit a bad address return error. 270 */ 271 if (!vsid) 272 return -1; 273 /* Make kernel text executable */ 274 if (overlaps_kernel_text(vaddr, vaddr + step)) 275 tprot &= ~HPTE_R_N; 276 277 /* 278 * If relocatable, check if it overlaps interrupt vectors that 279 * are copied down to real 0. For relocatable kernel 280 * (e.g. kdump case) we copy interrupt vectors down to real 281 * address 0. Mark that region as executable. This is 282 * because on p8 system with relocation on exception feature 283 * enabled, exceptions are raised with MMU (IR=DR=1) ON. Hence 284 * in order to execute the interrupt handlers in virtual 285 * mode the vector region need to be marked as executable. 286 */ 287 if ((PHYSICAL_START > MEMORY_START) && 288 overlaps_interrupt_vector_text(vaddr, vaddr + step)) 289 tprot &= ~HPTE_R_N; 290 291 hash = hpt_hash(vpn, shift, ssize); 292 hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP); 293 294 BUG_ON(!mmu_hash_ops.hpte_insert); 295 repeat: 296 ret = mmu_hash_ops.hpte_insert(hpteg, vpn, paddr, tprot, 297 HPTE_V_BOLTED, psize, psize, 298 ssize); 299 if (ret == -1) { 300 /* 301 * Try to to keep bolted entries in primary. 302 * Remove non bolted entries and try insert again 303 */ 304 ret = mmu_hash_ops.hpte_remove(hpteg); 305 if (ret != -1) 306 ret = mmu_hash_ops.hpte_insert(hpteg, vpn, paddr, tprot, 307 HPTE_V_BOLTED, psize, psize, 308 ssize); 309 if (ret == -1 && !secondary_hash) { 310 secondary_hash = true; 311 hpteg = ((~hash & htab_hash_mask) * HPTES_PER_GROUP); 312 goto repeat; 313 } 314 } 315 316 if (ret < 0) 317 break; 318 319 cond_resched(); 320 #ifdef CONFIG_DEBUG_PAGEALLOC 321 if (debug_pagealloc_enabled() && 322 (paddr >> PAGE_SHIFT) < linear_map_hash_count) 323 linear_map_hash_slots[paddr >> PAGE_SHIFT] = ret | 0x80; 324 #endif /* CONFIG_DEBUG_PAGEALLOC */ 325 } 326 return ret < 0 ? ret : 0; 327 } 328 329 int htab_remove_mapping(unsigned long vstart, unsigned long vend, 330 int psize, int ssize) 331 { 332 unsigned long vaddr; 333 unsigned int step, shift; 334 int rc; 335 int ret = 0; 336 337 shift = mmu_psize_defs[psize].shift; 338 step = 1 << shift; 339 340 if (!mmu_hash_ops.hpte_removebolted) 341 return -ENODEV; 342 343 for (vaddr = vstart; vaddr < vend; vaddr += step) { 344 rc = mmu_hash_ops.hpte_removebolted(vaddr, psize, ssize); 345 if (rc == -ENOENT) { 346 ret = -ENOENT; 347 continue; 348 } 349 if (rc < 0) 350 return rc; 351 } 352 353 return ret; 354 } 355 356 static bool disable_1tb_segments = false; 357 358 static int __init parse_disable_1tb_segments(char *p) 359 { 360 disable_1tb_segments = true; 361 return 0; 362 } 363 early_param("disable_1tb_segments", parse_disable_1tb_segments); 364 365 static int __init htab_dt_scan_seg_sizes(unsigned long node, 366 const char *uname, int depth, 367 void *data) 368 { 369 const char *type = of_get_flat_dt_prop(node, "device_type", NULL); 370 const __be32 *prop; 371 int size = 0; 372 373 /* We are scanning "cpu" nodes only */ 374 if (type == NULL || strcmp(type, "cpu") != 0) 375 return 0; 376 377 prop = of_get_flat_dt_prop(node, "ibm,processor-segment-sizes", &size); 378 if (prop == NULL) 379 return 0; 380 for (; size >= 4; size -= 4, ++prop) { 381 if (be32_to_cpu(prop[0]) == 40) { 382 DBG("1T segment support detected\n"); 383 384 if (disable_1tb_segments) { 385 DBG("1T segments disabled by command line\n"); 386 break; 387 } 388 389 cur_cpu_spec->mmu_features |= MMU_FTR_1T_SEGMENT; 390 return 1; 391 } 392 } 393 cur_cpu_spec->mmu_features &= ~MMU_FTR_NO_SLBIE_B; 394 return 0; 395 } 396 397 static int __init get_idx_from_shift(unsigned int shift) 398 { 399 int idx = -1; 400 401 switch (shift) { 402 case 0xc: 403 idx = MMU_PAGE_4K; 404 break; 405 case 0x10: 406 idx = MMU_PAGE_64K; 407 break; 408 case 0x14: 409 idx = MMU_PAGE_1M; 410 break; 411 case 0x18: 412 idx = MMU_PAGE_16M; 413 break; 414 case 0x22: 415 idx = MMU_PAGE_16G; 416 break; 417 } 418 return idx; 419 } 420 421 static int __init htab_dt_scan_page_sizes(unsigned long node, 422 const char *uname, int depth, 423 void *data) 424 { 425 const char *type = of_get_flat_dt_prop(node, "device_type", NULL); 426 const __be32 *prop; 427 int size = 0; 428 429 /* We are scanning "cpu" nodes only */ 430 if (type == NULL || strcmp(type, "cpu") != 0) 431 return 0; 432 433 prop = of_get_flat_dt_prop(node, "ibm,segment-page-sizes", &size); 434 if (!prop) 435 return 0; 436 437 pr_info("Page sizes from device-tree:\n"); 438 size /= 4; 439 cur_cpu_spec->mmu_features &= ~(MMU_FTR_16M_PAGE); 440 while(size > 0) { 441 unsigned int base_shift = be32_to_cpu(prop[0]); 442 unsigned int slbenc = be32_to_cpu(prop[1]); 443 unsigned int lpnum = be32_to_cpu(prop[2]); 444 struct mmu_psize_def *def; 445 int idx, base_idx; 446 447 size -= 3; prop += 3; 448 base_idx = get_idx_from_shift(base_shift); 449 if (base_idx < 0) { 450 /* skip the pte encoding also */ 451 prop += lpnum * 2; size -= lpnum * 2; 452 continue; 453 } 454 def = &mmu_psize_defs[base_idx]; 455 if (base_idx == MMU_PAGE_16M) 456 cur_cpu_spec->mmu_features |= MMU_FTR_16M_PAGE; 457 458 def->shift = base_shift; 459 if (base_shift <= 23) 460 def->avpnm = 0; 461 else 462 def->avpnm = (1 << (base_shift - 23)) - 1; 463 def->sllp = slbenc; 464 /* 465 * We don't know for sure what's up with tlbiel, so 466 * for now we only set it for 4K and 64K pages 467 */ 468 if (base_idx == MMU_PAGE_4K || base_idx == MMU_PAGE_64K) 469 def->tlbiel = 1; 470 else 471 def->tlbiel = 0; 472 473 while (size > 0 && lpnum) { 474 unsigned int shift = be32_to_cpu(prop[0]); 475 int penc = be32_to_cpu(prop[1]); 476 477 prop += 2; size -= 2; 478 lpnum--; 479 480 idx = get_idx_from_shift(shift); 481 if (idx < 0) 482 continue; 483 484 if (penc == -1) 485 pr_err("Invalid penc for base_shift=%d " 486 "shift=%d\n", base_shift, shift); 487 488 def->penc[idx] = penc; 489 pr_info("base_shift=%d: shift=%d, sllp=0x%04lx," 490 " avpnm=0x%08lx, tlbiel=%d, penc=%d\n", 491 base_shift, shift, def->sllp, 492 def->avpnm, def->tlbiel, def->penc[idx]); 493 } 494 } 495 496 return 1; 497 } 498 499 #ifdef CONFIG_HUGETLB_PAGE 500 /* 501 * Scan for 16G memory blocks that have been set aside for huge pages 502 * and reserve those blocks for 16G huge pages. 503 */ 504 static int __init htab_dt_scan_hugepage_blocks(unsigned long node, 505 const char *uname, int depth, 506 void *data) { 507 const char *type = of_get_flat_dt_prop(node, "device_type", NULL); 508 const __be64 *addr_prop; 509 const __be32 *page_count_prop; 510 unsigned int expected_pages; 511 long unsigned int phys_addr; 512 long unsigned int block_size; 513 514 /* We are scanning "memory" nodes only */ 515 if (type == NULL || strcmp(type, "memory") != 0) 516 return 0; 517 518 /* 519 * This property is the log base 2 of the number of virtual pages that 520 * will represent this memory block. 521 */ 522 page_count_prop = of_get_flat_dt_prop(node, "ibm,expected#pages", NULL); 523 if (page_count_prop == NULL) 524 return 0; 525 expected_pages = (1 << be32_to_cpu(page_count_prop[0])); 526 addr_prop = of_get_flat_dt_prop(node, "reg", NULL); 527 if (addr_prop == NULL) 528 return 0; 529 phys_addr = be64_to_cpu(addr_prop[0]); 530 block_size = be64_to_cpu(addr_prop[1]); 531 if (block_size != (16 * GB)) 532 return 0; 533 printk(KERN_INFO "Huge page(16GB) memory: " 534 "addr = 0x%lX size = 0x%lX pages = %d\n", 535 phys_addr, block_size, expected_pages); 536 if (phys_addr + block_size * expected_pages <= memblock_end_of_DRAM()) { 537 memblock_reserve(phys_addr, block_size * expected_pages); 538 pseries_add_gpage(phys_addr, block_size, expected_pages); 539 } 540 return 0; 541 } 542 #endif /* CONFIG_HUGETLB_PAGE */ 543 544 static void mmu_psize_set_default_penc(void) 545 { 546 int bpsize, apsize; 547 for (bpsize = 0; bpsize < MMU_PAGE_COUNT; bpsize++) 548 for (apsize = 0; apsize < MMU_PAGE_COUNT; apsize++) 549 mmu_psize_defs[bpsize].penc[apsize] = -1; 550 } 551 552 #ifdef CONFIG_PPC_64K_PAGES 553 554 static bool might_have_hea(void) 555 { 556 /* 557 * The HEA ethernet adapter requires awareness of the 558 * GX bus. Without that awareness we can easily assume 559 * we will never see an HEA ethernet device. 560 */ 561 #ifdef CONFIG_IBMEBUS 562 return !cpu_has_feature(CPU_FTR_ARCH_207S) && 563 firmware_has_feature(FW_FEATURE_SPLPAR); 564 #else 565 return false; 566 #endif 567 } 568 569 #endif /* #ifdef CONFIG_PPC_64K_PAGES */ 570 571 static void __init htab_scan_page_sizes(void) 572 { 573 int rc; 574 575 /* se the invalid penc to -1 */ 576 mmu_psize_set_default_penc(); 577 578 /* Default to 4K pages only */ 579 memcpy(mmu_psize_defs, mmu_psize_defaults, 580 sizeof(mmu_psize_defaults)); 581 582 /* 583 * Try to find the available page sizes in the device-tree 584 */ 585 rc = of_scan_flat_dt(htab_dt_scan_page_sizes, NULL); 586 if (rc == 0 && early_mmu_has_feature(MMU_FTR_16M_PAGE)) { 587 /* 588 * Nothing in the device-tree, but the CPU supports 16M pages, 589 * so let's fallback on a known size list for 16M capable CPUs. 590 */ 591 memcpy(mmu_psize_defs, mmu_psize_defaults_gp, 592 sizeof(mmu_psize_defaults_gp)); 593 } 594 595 #ifdef CONFIG_HUGETLB_PAGE 596 if (!hugetlb_disabled) { 597 /* Reserve 16G huge page memory sections for huge pages */ 598 of_scan_flat_dt(htab_dt_scan_hugepage_blocks, NULL); 599 } 600 #endif /* CONFIG_HUGETLB_PAGE */ 601 } 602 603 /* 604 * Fill in the hpte_page_sizes[] array. 605 * We go through the mmu_psize_defs[] array looking for all the 606 * supported base/actual page size combinations. Each combination 607 * has a unique pagesize encoding (penc) value in the low bits of 608 * the LP field of the HPTE. For actual page sizes less than 1MB, 609 * some of the upper LP bits are used for RPN bits, meaning that 610 * we need to fill in several entries in hpte_page_sizes[]. 611 * 612 * In diagrammatic form, with r = RPN bits and z = page size bits: 613 * PTE LP actual page size 614 * rrrr rrrz >=8KB 615 * rrrr rrzz >=16KB 616 * rrrr rzzz >=32KB 617 * rrrr zzzz >=64KB 618 * ... 619 * 620 * The zzzz bits are implementation-specific but are chosen so that 621 * no encoding for a larger page size uses the same value in its 622 * low-order N bits as the encoding for the 2^(12+N) byte page size 623 * (if it exists). 624 */ 625 static void init_hpte_page_sizes(void) 626 { 627 long int ap, bp; 628 long int shift, penc; 629 630 for (bp = 0; bp < MMU_PAGE_COUNT; ++bp) { 631 if (!mmu_psize_defs[bp].shift) 632 continue; /* not a supported page size */ 633 for (ap = bp; ap < MMU_PAGE_COUNT; ++ap) { 634 penc = mmu_psize_defs[bp].penc[ap]; 635 if (penc == -1 || !mmu_psize_defs[ap].shift) 636 continue; 637 shift = mmu_psize_defs[ap].shift - LP_SHIFT; 638 if (shift <= 0) 639 continue; /* should never happen */ 640 /* 641 * For page sizes less than 1MB, this loop 642 * replicates the entry for all possible values 643 * of the rrrr bits. 644 */ 645 while (penc < (1 << LP_BITS)) { 646 hpte_page_sizes[penc] = (ap << 4) | bp; 647 penc += 1 << shift; 648 } 649 } 650 } 651 } 652 653 static void __init htab_init_page_sizes(void) 654 { 655 init_hpte_page_sizes(); 656 657 if (!debug_pagealloc_enabled()) { 658 /* 659 * Pick a size for the linear mapping. Currently, we only 660 * support 16M, 1M and 4K which is the default 661 */ 662 if (mmu_psize_defs[MMU_PAGE_16M].shift) 663 mmu_linear_psize = MMU_PAGE_16M; 664 else if (mmu_psize_defs[MMU_PAGE_1M].shift) 665 mmu_linear_psize = MMU_PAGE_1M; 666 } 667 668 #ifdef CONFIG_PPC_64K_PAGES 669 /* 670 * Pick a size for the ordinary pages. Default is 4K, we support 671 * 64K for user mappings and vmalloc if supported by the processor. 672 * We only use 64k for ioremap if the processor 673 * (and firmware) support cache-inhibited large pages. 674 * If not, we use 4k and set mmu_ci_restrictions so that 675 * hash_page knows to switch processes that use cache-inhibited 676 * mappings to 4k pages. 677 */ 678 if (mmu_psize_defs[MMU_PAGE_64K].shift) { 679 mmu_virtual_psize = MMU_PAGE_64K; 680 mmu_vmalloc_psize = MMU_PAGE_64K; 681 if (mmu_linear_psize == MMU_PAGE_4K) 682 mmu_linear_psize = MMU_PAGE_64K; 683 if (mmu_has_feature(MMU_FTR_CI_LARGE_PAGE)) { 684 /* 685 * When running on pSeries using 64k pages for ioremap 686 * would stop us accessing the HEA ethernet. So if we 687 * have the chance of ever seeing one, stay at 4k. 688 */ 689 if (!might_have_hea()) 690 mmu_io_psize = MMU_PAGE_64K; 691 } else 692 mmu_ci_restrictions = 1; 693 } 694 #endif /* CONFIG_PPC_64K_PAGES */ 695 696 #ifdef CONFIG_SPARSEMEM_VMEMMAP 697 /* 698 * We try to use 16M pages for vmemmap if that is supported 699 * and we have at least 1G of RAM at boot 700 */ 701 if (mmu_psize_defs[MMU_PAGE_16M].shift && 702 memblock_phys_mem_size() >= 0x40000000) 703 mmu_vmemmap_psize = MMU_PAGE_16M; 704 else 705 mmu_vmemmap_psize = mmu_virtual_psize; 706 #endif /* CONFIG_SPARSEMEM_VMEMMAP */ 707 708 printk(KERN_DEBUG "Page orders: linear mapping = %d, " 709 "virtual = %d, io = %d" 710 #ifdef CONFIG_SPARSEMEM_VMEMMAP 711 ", vmemmap = %d" 712 #endif 713 "\n", 714 mmu_psize_defs[mmu_linear_psize].shift, 715 mmu_psize_defs[mmu_virtual_psize].shift, 716 mmu_psize_defs[mmu_io_psize].shift 717 #ifdef CONFIG_SPARSEMEM_VMEMMAP 718 ,mmu_psize_defs[mmu_vmemmap_psize].shift 719 #endif 720 ); 721 } 722 723 static int __init htab_dt_scan_pftsize(unsigned long node, 724 const char *uname, int depth, 725 void *data) 726 { 727 const char *type = of_get_flat_dt_prop(node, "device_type", NULL); 728 const __be32 *prop; 729 730 /* We are scanning "cpu" nodes only */ 731 if (type == NULL || strcmp(type, "cpu") != 0) 732 return 0; 733 734 prop = of_get_flat_dt_prop(node, "ibm,pft-size", NULL); 735 if (prop != NULL) { 736 /* pft_size[0] is the NUMA CEC cookie */ 737 ppc64_pft_size = be32_to_cpu(prop[1]); 738 return 1; 739 } 740 return 0; 741 } 742 743 unsigned htab_shift_for_mem_size(unsigned long mem_size) 744 { 745 unsigned memshift = __ilog2(mem_size); 746 unsigned pshift = mmu_psize_defs[mmu_virtual_psize].shift; 747 unsigned pteg_shift; 748 749 /* round mem_size up to next power of 2 */ 750 if ((1UL << memshift) < mem_size) 751 memshift += 1; 752 753 /* aim for 2 pages / pteg */ 754 pteg_shift = memshift - (pshift + 1); 755 756 /* 757 * 2^11 PTEGS of 128 bytes each, ie. 2^18 bytes is the minimum htab 758 * size permitted by the architecture. 759 */ 760 return max(pteg_shift + 7, 18U); 761 } 762 763 static unsigned long __init htab_get_table_size(void) 764 { 765 /* 766 * If hash size isn't already provided by the platform, we try to 767 * retrieve it from the device-tree. If it's not there neither, we 768 * calculate it now based on the total RAM size 769 */ 770 if (ppc64_pft_size == 0) 771 of_scan_flat_dt(htab_dt_scan_pftsize, NULL); 772 if (ppc64_pft_size) 773 return 1UL << ppc64_pft_size; 774 775 return 1UL << htab_shift_for_mem_size(memblock_phys_mem_size()); 776 } 777 778 #ifdef CONFIG_MEMORY_HOTPLUG 779 int resize_hpt_for_hotplug(unsigned long new_mem_size) 780 { 781 unsigned target_hpt_shift; 782 783 if (!mmu_hash_ops.resize_hpt) 784 return 0; 785 786 target_hpt_shift = htab_shift_for_mem_size(new_mem_size); 787 788 /* 789 * To avoid lots of HPT resizes if memory size is fluctuating 790 * across a boundary, we deliberately have some hysterisis 791 * here: we immediately increase the HPT size if the target 792 * shift exceeds the current shift, but we won't attempt to 793 * reduce unless the target shift is at least 2 below the 794 * current shift 795 */ 796 if (target_hpt_shift > ppc64_pft_size || 797 target_hpt_shift < ppc64_pft_size - 1) 798 return mmu_hash_ops.resize_hpt(target_hpt_shift); 799 800 return 0; 801 } 802 803 int hash__create_section_mapping(unsigned long start, unsigned long end, int nid) 804 { 805 int rc; 806 807 if (end >= H_VMALLOC_START) { 808 pr_warn("Outside the supported range\n"); 809 return -1; 810 } 811 812 rc = htab_bolt_mapping(start, end, __pa(start), 813 pgprot_val(PAGE_KERNEL), mmu_linear_psize, 814 mmu_kernel_ssize); 815 816 if (rc < 0) { 817 int rc2 = htab_remove_mapping(start, end, mmu_linear_psize, 818 mmu_kernel_ssize); 819 BUG_ON(rc2 && (rc2 != -ENOENT)); 820 } 821 return rc; 822 } 823 824 int hash__remove_section_mapping(unsigned long start, unsigned long end) 825 { 826 int rc = htab_remove_mapping(start, end, mmu_linear_psize, 827 mmu_kernel_ssize); 828 WARN_ON(rc < 0); 829 return rc; 830 } 831 #endif /* CONFIG_MEMORY_HOTPLUG */ 832 833 static void __init hash_init_partition_table(phys_addr_t hash_table, 834 unsigned long htab_size) 835 { 836 mmu_partition_table_init(); 837 838 /* 839 * PS field (VRMA page size) is not used for LPID 0, hence set to 0. 840 * For now, UPRT is 0 and we have no segment table. 841 */ 842 htab_size = __ilog2(htab_size) - 18; 843 mmu_partition_table_set_entry(0, hash_table | htab_size, 0, false); 844 pr_info("Partition table %p\n", partition_tb); 845 } 846 847 static void __init htab_initialize(void) 848 { 849 unsigned long table; 850 unsigned long pteg_count; 851 unsigned long prot; 852 unsigned long base = 0, size = 0; 853 struct memblock_region *reg; 854 855 DBG(" -> htab_initialize()\n"); 856 857 if (mmu_has_feature(MMU_FTR_1T_SEGMENT)) { 858 mmu_kernel_ssize = MMU_SEGSIZE_1T; 859 mmu_highuser_ssize = MMU_SEGSIZE_1T; 860 printk(KERN_INFO "Using 1TB segments\n"); 861 } 862 863 /* 864 * Calculate the required size of the htab. We want the number of 865 * PTEGs to equal one half the number of real pages. 866 */ 867 htab_size_bytes = htab_get_table_size(); 868 pteg_count = htab_size_bytes >> 7; 869 870 htab_hash_mask = pteg_count - 1; 871 872 if (firmware_has_feature(FW_FEATURE_LPAR) || 873 firmware_has_feature(FW_FEATURE_PS3_LV1)) { 874 /* Using a hypervisor which owns the htab */ 875 htab_address = NULL; 876 _SDR1 = 0; 877 #ifdef CONFIG_FA_DUMP 878 /* 879 * If firmware assisted dump is active firmware preserves 880 * the contents of htab along with entire partition memory. 881 * Clear the htab if firmware assisted dump is active so 882 * that we dont end up using old mappings. 883 */ 884 if (is_fadump_active() && mmu_hash_ops.hpte_clear_all) 885 mmu_hash_ops.hpte_clear_all(); 886 #endif 887 } else { 888 unsigned long limit = MEMBLOCK_ALLOC_ANYWHERE; 889 890 #ifdef CONFIG_PPC_CELL 891 /* 892 * Cell may require the hash table down low when using the 893 * Axon IOMMU in order to fit the dynamic region over it, see 894 * comments in cell/iommu.c 895 */ 896 if (fdt_subnode_offset(initial_boot_params, 0, "axon") > 0) { 897 limit = 0x80000000; 898 pr_info("Hash table forced below 2G for Axon IOMMU\n"); 899 } 900 #endif /* CONFIG_PPC_CELL */ 901 902 table = memblock_phys_alloc_range(htab_size_bytes, 903 htab_size_bytes, 904 0, limit); 905 if (!table) 906 panic("ERROR: Failed to allocate %pa bytes below %pa\n", 907 &htab_size_bytes, &limit); 908 909 DBG("Hash table allocated at %lx, size: %lx\n", table, 910 htab_size_bytes); 911 912 htab_address = __va(table); 913 914 /* htab absolute addr + encoded htabsize */ 915 _SDR1 = table + __ilog2(htab_size_bytes) - 18; 916 917 /* Initialize the HPT with no entries */ 918 memset((void *)table, 0, htab_size_bytes); 919 920 if (!cpu_has_feature(CPU_FTR_ARCH_300)) 921 /* Set SDR1 */ 922 mtspr(SPRN_SDR1, _SDR1); 923 else 924 hash_init_partition_table(table, htab_size_bytes); 925 } 926 927 prot = pgprot_val(PAGE_KERNEL); 928 929 #ifdef CONFIG_DEBUG_PAGEALLOC 930 if (debug_pagealloc_enabled()) { 931 linear_map_hash_count = memblock_end_of_DRAM() >> PAGE_SHIFT; 932 linear_map_hash_slots = memblock_alloc_try_nid( 933 linear_map_hash_count, 1, MEMBLOCK_LOW_LIMIT, 934 ppc64_rma_size, NUMA_NO_NODE); 935 if (!linear_map_hash_slots) 936 panic("%s: Failed to allocate %lu bytes max_addr=%pa\n", 937 __func__, linear_map_hash_count, &ppc64_rma_size); 938 } 939 #endif /* CONFIG_DEBUG_PAGEALLOC */ 940 941 /* create bolted the linear mapping in the hash table */ 942 for_each_memblock(memory, reg) { 943 base = (unsigned long)__va(reg->base); 944 size = reg->size; 945 946 DBG("creating mapping for region: %lx..%lx (prot: %lx)\n", 947 base, size, prot); 948 949 if ((base + size) >= H_VMALLOC_START) { 950 pr_warn("Outside the supported range\n"); 951 continue; 952 } 953 954 BUG_ON(htab_bolt_mapping(base, base + size, __pa(base), 955 prot, mmu_linear_psize, mmu_kernel_ssize)); 956 } 957 memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE); 958 959 /* 960 * If we have a memory_limit and we've allocated TCEs then we need to 961 * explicitly map the TCE area at the top of RAM. We also cope with the 962 * case that the TCEs start below memory_limit. 963 * tce_alloc_start/end are 16MB aligned so the mapping should work 964 * for either 4K or 16MB pages. 965 */ 966 if (tce_alloc_start) { 967 tce_alloc_start = (unsigned long)__va(tce_alloc_start); 968 tce_alloc_end = (unsigned long)__va(tce_alloc_end); 969 970 if (base + size >= tce_alloc_start) 971 tce_alloc_start = base + size + 1; 972 973 BUG_ON(htab_bolt_mapping(tce_alloc_start, tce_alloc_end, 974 __pa(tce_alloc_start), prot, 975 mmu_linear_psize, mmu_kernel_ssize)); 976 } 977 978 979 DBG(" <- htab_initialize()\n"); 980 } 981 #undef KB 982 #undef MB 983 984 void __init hash__early_init_devtree(void) 985 { 986 /* Initialize segment sizes */ 987 of_scan_flat_dt(htab_dt_scan_seg_sizes, NULL); 988 989 /* Initialize page sizes */ 990 htab_scan_page_sizes(); 991 } 992 993 static struct hash_mm_context init_hash_mm_context; 994 void __init hash__early_init_mmu(void) 995 { 996 #ifndef CONFIG_PPC_64K_PAGES 997 /* 998 * We have code in __hash_page_4K() and elsewhere, which assumes it can 999 * do the following: 1000 * new_pte |= (slot << H_PAGE_F_GIX_SHIFT) & (H_PAGE_F_SECOND | H_PAGE_F_GIX); 1001 * 1002 * Where the slot number is between 0-15, and values of 8-15 indicate 1003 * the secondary bucket. For that code to work H_PAGE_F_SECOND and 1004 * H_PAGE_F_GIX must occupy four contiguous bits in the PTE, and 1005 * H_PAGE_F_SECOND must be placed above H_PAGE_F_GIX. Assert that here 1006 * with a BUILD_BUG_ON(). 1007 */ 1008 BUILD_BUG_ON(H_PAGE_F_SECOND != (1ul << (H_PAGE_F_GIX_SHIFT + 3))); 1009 #endif /* CONFIG_PPC_64K_PAGES */ 1010 1011 htab_init_page_sizes(); 1012 1013 /* 1014 * initialize page table size 1015 */ 1016 __pte_frag_nr = H_PTE_FRAG_NR; 1017 __pte_frag_size_shift = H_PTE_FRAG_SIZE_SHIFT; 1018 __pmd_frag_nr = H_PMD_FRAG_NR; 1019 __pmd_frag_size_shift = H_PMD_FRAG_SIZE_SHIFT; 1020 1021 __pte_index_size = H_PTE_INDEX_SIZE; 1022 __pmd_index_size = H_PMD_INDEX_SIZE; 1023 __pud_index_size = H_PUD_INDEX_SIZE; 1024 __pgd_index_size = H_PGD_INDEX_SIZE; 1025 __pud_cache_index = H_PUD_CACHE_INDEX; 1026 __pte_table_size = H_PTE_TABLE_SIZE; 1027 __pmd_table_size = H_PMD_TABLE_SIZE; 1028 __pud_table_size = H_PUD_TABLE_SIZE; 1029 __pgd_table_size = H_PGD_TABLE_SIZE; 1030 /* 1031 * 4k use hugepd format, so for hash set then to 1032 * zero 1033 */ 1034 __pmd_val_bits = HASH_PMD_VAL_BITS; 1035 __pud_val_bits = HASH_PUD_VAL_BITS; 1036 __pgd_val_bits = HASH_PGD_VAL_BITS; 1037 1038 __kernel_virt_start = H_KERN_VIRT_START; 1039 __vmalloc_start = H_VMALLOC_START; 1040 __vmalloc_end = H_VMALLOC_END; 1041 __kernel_io_start = H_KERN_IO_START; 1042 __kernel_io_end = H_KERN_IO_END; 1043 vmemmap = (struct page *)H_VMEMMAP_START; 1044 ioremap_bot = IOREMAP_BASE; 1045 1046 #ifdef CONFIG_PCI 1047 pci_io_base = ISA_IO_BASE; 1048 #endif 1049 1050 /* Select appropriate backend */ 1051 if (firmware_has_feature(FW_FEATURE_PS3_LV1)) 1052 ps3_early_mm_init(); 1053 else if (firmware_has_feature(FW_FEATURE_LPAR)) 1054 hpte_init_pseries(); 1055 else if (IS_ENABLED(CONFIG_PPC_NATIVE)) 1056 hpte_init_native(); 1057 1058 if (!mmu_hash_ops.hpte_insert) 1059 panic("hash__early_init_mmu: No MMU hash ops defined!\n"); 1060 1061 /* 1062 * Initialize the MMU Hash table and create the linear mapping 1063 * of memory. Has to be done before SLB initialization as this is 1064 * currently where the page size encoding is obtained. 1065 */ 1066 htab_initialize(); 1067 1068 init_mm.context.hash_context = &init_hash_mm_context; 1069 mm_ctx_set_slb_addr_limit(&init_mm.context, SLB_ADDR_LIMIT_DEFAULT); 1070 1071 pr_info("Initializing hash mmu with SLB\n"); 1072 /* Initialize SLB management */ 1073 slb_initialize(); 1074 1075 if (cpu_has_feature(CPU_FTR_ARCH_206) 1076 && cpu_has_feature(CPU_FTR_HVMODE)) 1077 tlbiel_all(); 1078 } 1079 1080 #ifdef CONFIG_SMP 1081 void hash__early_init_mmu_secondary(void) 1082 { 1083 /* Initialize hash table for that CPU */ 1084 if (!firmware_has_feature(FW_FEATURE_LPAR)) { 1085 1086 if (!cpu_has_feature(CPU_FTR_ARCH_300)) 1087 mtspr(SPRN_SDR1, _SDR1); 1088 else 1089 set_ptcr_when_no_uv(__pa(partition_tb) | 1090 (PATB_SIZE_SHIFT - 12)); 1091 } 1092 /* Initialize SLB */ 1093 slb_initialize(); 1094 1095 if (cpu_has_feature(CPU_FTR_ARCH_206) 1096 && cpu_has_feature(CPU_FTR_HVMODE)) 1097 tlbiel_all(); 1098 } 1099 #endif /* CONFIG_SMP */ 1100 1101 /* 1102 * Called by asm hashtable.S for doing lazy icache flush 1103 */ 1104 unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap) 1105 { 1106 struct page *page; 1107 1108 if (!pfn_valid(pte_pfn(pte))) 1109 return pp; 1110 1111 page = pte_page(pte); 1112 1113 /* page is dirty */ 1114 if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) { 1115 if (trap == 0x400) { 1116 flush_dcache_icache_page(page); 1117 set_bit(PG_arch_1, &page->flags); 1118 } else 1119 pp |= HPTE_R_N; 1120 } 1121 return pp; 1122 } 1123 1124 #ifdef CONFIG_PPC_MM_SLICES 1125 static unsigned int get_paca_psize(unsigned long addr) 1126 { 1127 unsigned char *psizes; 1128 unsigned long index, mask_index; 1129 1130 if (addr < SLICE_LOW_TOP) { 1131 psizes = get_paca()->mm_ctx_low_slices_psize; 1132 index = GET_LOW_SLICE_INDEX(addr); 1133 } else { 1134 psizes = get_paca()->mm_ctx_high_slices_psize; 1135 index = GET_HIGH_SLICE_INDEX(addr); 1136 } 1137 mask_index = index & 0x1; 1138 return (psizes[index >> 1] >> (mask_index * 4)) & 0xF; 1139 } 1140 1141 #else 1142 unsigned int get_paca_psize(unsigned long addr) 1143 { 1144 return get_paca()->mm_ctx_user_psize; 1145 } 1146 #endif 1147 1148 /* 1149 * Demote a segment to using 4k pages. 1150 * For now this makes the whole process use 4k pages. 1151 */ 1152 #ifdef CONFIG_PPC_64K_PAGES 1153 void demote_segment_4k(struct mm_struct *mm, unsigned long addr) 1154 { 1155 if (get_slice_psize(mm, addr) == MMU_PAGE_4K) 1156 return; 1157 slice_set_range_psize(mm, addr, 1, MMU_PAGE_4K); 1158 copro_flush_all_slbs(mm); 1159 if ((get_paca_psize(addr) != MMU_PAGE_4K) && (current->mm == mm)) { 1160 1161 copy_mm_to_paca(mm); 1162 slb_flush_and_restore_bolted(); 1163 } 1164 } 1165 #endif /* CONFIG_PPC_64K_PAGES */ 1166 1167 #ifdef CONFIG_PPC_SUBPAGE_PROT 1168 /* 1169 * This looks up a 2-bit protection code for a 4k subpage of a 64k page. 1170 * Userspace sets the subpage permissions using the subpage_prot system call. 1171 * 1172 * Result is 0: full permissions, _PAGE_RW: read-only, 1173 * _PAGE_RWX: no access. 1174 */ 1175 static int subpage_protection(struct mm_struct *mm, unsigned long ea) 1176 { 1177 struct subpage_prot_table *spt = mm_ctx_subpage_prot(&mm->context); 1178 u32 spp = 0; 1179 u32 **sbpm, *sbpp; 1180 1181 if (!spt) 1182 return 0; 1183 1184 if (ea >= spt->maxaddr) 1185 return 0; 1186 if (ea < 0x100000000UL) { 1187 /* addresses below 4GB use spt->low_prot */ 1188 sbpm = spt->low_prot; 1189 } else { 1190 sbpm = spt->protptrs[ea >> SBP_L3_SHIFT]; 1191 if (!sbpm) 1192 return 0; 1193 } 1194 sbpp = sbpm[(ea >> SBP_L2_SHIFT) & (SBP_L2_COUNT - 1)]; 1195 if (!sbpp) 1196 return 0; 1197 spp = sbpp[(ea >> PAGE_SHIFT) & (SBP_L1_COUNT - 1)]; 1198 1199 /* extract 2-bit bitfield for this 4k subpage */ 1200 spp >>= 30 - 2 * ((ea >> 12) & 0xf); 1201 1202 /* 1203 * 0 -> full premission 1204 * 1 -> Read only 1205 * 2 -> no access. 1206 * We return the flag that need to be cleared. 1207 */ 1208 spp = ((spp & 2) ? _PAGE_RWX : 0) | ((spp & 1) ? _PAGE_WRITE : 0); 1209 return spp; 1210 } 1211 1212 #else /* CONFIG_PPC_SUBPAGE_PROT */ 1213 static inline int subpage_protection(struct mm_struct *mm, unsigned long ea) 1214 { 1215 return 0; 1216 } 1217 #endif 1218 1219 void hash_failure_debug(unsigned long ea, unsigned long access, 1220 unsigned long vsid, unsigned long trap, 1221 int ssize, int psize, int lpsize, unsigned long pte) 1222 { 1223 if (!printk_ratelimit()) 1224 return; 1225 pr_info("mm: Hashing failure ! EA=0x%lx access=0x%lx current=%s\n", 1226 ea, access, current->comm); 1227 pr_info(" trap=0x%lx vsid=0x%lx ssize=%d base psize=%d psize %d pte=0x%lx\n", 1228 trap, vsid, ssize, psize, lpsize, pte); 1229 } 1230 1231 static void check_paca_psize(unsigned long ea, struct mm_struct *mm, 1232 int psize, bool user_region) 1233 { 1234 if (user_region) { 1235 if (psize != get_paca_psize(ea)) { 1236 copy_mm_to_paca(mm); 1237 slb_flush_and_restore_bolted(); 1238 } 1239 } else if (get_paca()->vmalloc_sllp != 1240 mmu_psize_defs[mmu_vmalloc_psize].sllp) { 1241 get_paca()->vmalloc_sllp = 1242 mmu_psize_defs[mmu_vmalloc_psize].sllp; 1243 slb_vmalloc_update(); 1244 } 1245 } 1246 1247 /* 1248 * Result code is: 1249 * 0 - handled 1250 * 1 - normal page fault 1251 * -1 - critical hash insertion error 1252 * -2 - access not permitted by subpage protection mechanism 1253 */ 1254 int hash_page_mm(struct mm_struct *mm, unsigned long ea, 1255 unsigned long access, unsigned long trap, 1256 unsigned long flags) 1257 { 1258 bool is_thp; 1259 enum ctx_state prev_state = exception_enter(); 1260 pgd_t *pgdir; 1261 unsigned long vsid; 1262 pte_t *ptep; 1263 unsigned hugeshift; 1264 int rc, user_region = 0; 1265 int psize, ssize; 1266 1267 DBG_LOW("hash_page(ea=%016lx, access=%lx, trap=%lx\n", 1268 ea, access, trap); 1269 trace_hash_fault(ea, access, trap); 1270 1271 /* Get region & vsid */ 1272 switch (get_region_id(ea)) { 1273 case USER_REGION_ID: 1274 user_region = 1; 1275 if (! mm) { 1276 DBG_LOW(" user region with no mm !\n"); 1277 rc = 1; 1278 goto bail; 1279 } 1280 psize = get_slice_psize(mm, ea); 1281 ssize = user_segment_size(ea); 1282 vsid = get_user_vsid(&mm->context, ea, ssize); 1283 break; 1284 case VMALLOC_REGION_ID: 1285 vsid = get_kernel_vsid(ea, mmu_kernel_ssize); 1286 psize = mmu_vmalloc_psize; 1287 ssize = mmu_kernel_ssize; 1288 break; 1289 1290 case IO_REGION_ID: 1291 vsid = get_kernel_vsid(ea, mmu_kernel_ssize); 1292 psize = mmu_io_psize; 1293 ssize = mmu_kernel_ssize; 1294 break; 1295 default: 1296 /* 1297 * Not a valid range 1298 * Send the problem up to do_page_fault() 1299 */ 1300 rc = 1; 1301 goto bail; 1302 } 1303 DBG_LOW(" mm=%p, mm->pgdir=%p, vsid=%016lx\n", mm, mm->pgd, vsid); 1304 1305 /* Bad address. */ 1306 if (!vsid) { 1307 DBG_LOW("Bad address!\n"); 1308 rc = 1; 1309 goto bail; 1310 } 1311 /* Get pgdir */ 1312 pgdir = mm->pgd; 1313 if (pgdir == NULL) { 1314 rc = 1; 1315 goto bail; 1316 } 1317 1318 /* Check CPU locality */ 1319 if (user_region && mm_is_thread_local(mm)) 1320 flags |= HPTE_LOCAL_UPDATE; 1321 1322 #ifndef CONFIG_PPC_64K_PAGES 1323 /* 1324 * If we use 4K pages and our psize is not 4K, then we might 1325 * be hitting a special driver mapping, and need to align the 1326 * address before we fetch the PTE. 1327 * 1328 * It could also be a hugepage mapping, in which case this is 1329 * not necessary, but it's not harmful, either. 1330 */ 1331 if (psize != MMU_PAGE_4K) 1332 ea &= ~((1ul << mmu_psize_defs[psize].shift) - 1); 1333 #endif /* CONFIG_PPC_64K_PAGES */ 1334 1335 /* Get PTE and page size from page tables */ 1336 ptep = find_linux_pte(pgdir, ea, &is_thp, &hugeshift); 1337 if (ptep == NULL || !pte_present(*ptep)) { 1338 DBG_LOW(" no PTE !\n"); 1339 rc = 1; 1340 goto bail; 1341 } 1342 1343 /* Add _PAGE_PRESENT to the required access perm */ 1344 access |= _PAGE_PRESENT; 1345 1346 /* 1347 * Pre-check access permissions (will be re-checked atomically 1348 * in __hash_page_XX but this pre-check is a fast path 1349 */ 1350 if (!check_pte_access(access, pte_val(*ptep))) { 1351 DBG_LOW(" no access !\n"); 1352 rc = 1; 1353 goto bail; 1354 } 1355 1356 if (hugeshift) { 1357 if (is_thp) 1358 rc = __hash_page_thp(ea, access, vsid, (pmd_t *)ptep, 1359 trap, flags, ssize, psize); 1360 #ifdef CONFIG_HUGETLB_PAGE 1361 else 1362 rc = __hash_page_huge(ea, access, vsid, ptep, trap, 1363 flags, ssize, hugeshift, psize); 1364 #else 1365 else { 1366 /* 1367 * if we have hugeshift, and is not transhuge with 1368 * hugetlb disabled, something is really wrong. 1369 */ 1370 rc = 1; 1371 WARN_ON(1); 1372 } 1373 #endif 1374 if (current->mm == mm) 1375 check_paca_psize(ea, mm, psize, user_region); 1376 1377 goto bail; 1378 } 1379 1380 #ifndef CONFIG_PPC_64K_PAGES 1381 DBG_LOW(" i-pte: %016lx\n", pte_val(*ptep)); 1382 #else 1383 DBG_LOW(" i-pte: %016lx %016lx\n", pte_val(*ptep), 1384 pte_val(*(ptep + PTRS_PER_PTE))); 1385 #endif 1386 /* Do actual hashing */ 1387 #ifdef CONFIG_PPC_64K_PAGES 1388 /* If H_PAGE_4K_PFN is set, make sure this is a 4k segment */ 1389 if ((pte_val(*ptep) & H_PAGE_4K_PFN) && psize == MMU_PAGE_64K) { 1390 demote_segment_4k(mm, ea); 1391 psize = MMU_PAGE_4K; 1392 } 1393 1394 /* 1395 * If this PTE is non-cacheable and we have restrictions on 1396 * using non cacheable large pages, then we switch to 4k 1397 */ 1398 if (mmu_ci_restrictions && psize == MMU_PAGE_64K && pte_ci(*ptep)) { 1399 if (user_region) { 1400 demote_segment_4k(mm, ea); 1401 psize = MMU_PAGE_4K; 1402 } else if (ea < VMALLOC_END) { 1403 /* 1404 * some driver did a non-cacheable mapping 1405 * in vmalloc space, so switch vmalloc 1406 * to 4k pages 1407 */ 1408 printk(KERN_ALERT "Reducing vmalloc segment " 1409 "to 4kB pages because of " 1410 "non-cacheable mapping\n"); 1411 psize = mmu_vmalloc_psize = MMU_PAGE_4K; 1412 copro_flush_all_slbs(mm); 1413 } 1414 } 1415 1416 #endif /* CONFIG_PPC_64K_PAGES */ 1417 1418 if (current->mm == mm) 1419 check_paca_psize(ea, mm, psize, user_region); 1420 1421 #ifdef CONFIG_PPC_64K_PAGES 1422 if (psize == MMU_PAGE_64K) 1423 rc = __hash_page_64K(ea, access, vsid, ptep, trap, 1424 flags, ssize); 1425 else 1426 #endif /* CONFIG_PPC_64K_PAGES */ 1427 { 1428 int spp = subpage_protection(mm, ea); 1429 if (access & spp) 1430 rc = -2; 1431 else 1432 rc = __hash_page_4K(ea, access, vsid, ptep, trap, 1433 flags, ssize, spp); 1434 } 1435 1436 /* 1437 * Dump some info in case of hash insertion failure, they should 1438 * never happen so it is really useful to know if/when they do 1439 */ 1440 if (rc == -1) 1441 hash_failure_debug(ea, access, vsid, trap, ssize, psize, 1442 psize, pte_val(*ptep)); 1443 #ifndef CONFIG_PPC_64K_PAGES 1444 DBG_LOW(" o-pte: %016lx\n", pte_val(*ptep)); 1445 #else 1446 DBG_LOW(" o-pte: %016lx %016lx\n", pte_val(*ptep), 1447 pte_val(*(ptep + PTRS_PER_PTE))); 1448 #endif 1449 DBG_LOW(" -> rc=%d\n", rc); 1450 1451 bail: 1452 exception_exit(prev_state); 1453 return rc; 1454 } 1455 EXPORT_SYMBOL_GPL(hash_page_mm); 1456 1457 int hash_page(unsigned long ea, unsigned long access, unsigned long trap, 1458 unsigned long dsisr) 1459 { 1460 unsigned long flags = 0; 1461 struct mm_struct *mm = current->mm; 1462 1463 if ((get_region_id(ea) == VMALLOC_REGION_ID) || 1464 (get_region_id(ea) == IO_REGION_ID)) 1465 mm = &init_mm; 1466 1467 if (dsisr & DSISR_NOHPTE) 1468 flags |= HPTE_NOHPTE_UPDATE; 1469 1470 return hash_page_mm(mm, ea, access, trap, flags); 1471 } 1472 EXPORT_SYMBOL_GPL(hash_page); 1473 1474 int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr, 1475 unsigned long msr) 1476 { 1477 unsigned long access = _PAGE_PRESENT | _PAGE_READ; 1478 unsigned long flags = 0; 1479 struct mm_struct *mm = current->mm; 1480 unsigned int region_id = get_region_id(ea); 1481 1482 if ((region_id == VMALLOC_REGION_ID) || (region_id == IO_REGION_ID)) 1483 mm = &init_mm; 1484 1485 if (dsisr & DSISR_NOHPTE) 1486 flags |= HPTE_NOHPTE_UPDATE; 1487 1488 if (dsisr & DSISR_ISSTORE) 1489 access |= _PAGE_WRITE; 1490 /* 1491 * We set _PAGE_PRIVILEGED only when 1492 * kernel mode access kernel space. 1493 * 1494 * _PAGE_PRIVILEGED is NOT set 1495 * 1) when kernel mode access user space 1496 * 2) user space access kernel space. 1497 */ 1498 access |= _PAGE_PRIVILEGED; 1499 if ((msr & MSR_PR) || (region_id == USER_REGION_ID)) 1500 access &= ~_PAGE_PRIVILEGED; 1501 1502 if (trap == 0x400) 1503 access |= _PAGE_EXEC; 1504 1505 return hash_page_mm(mm, ea, access, trap, flags); 1506 } 1507 1508 #ifdef CONFIG_PPC_MM_SLICES 1509 static bool should_hash_preload(struct mm_struct *mm, unsigned long ea) 1510 { 1511 int psize = get_slice_psize(mm, ea); 1512 1513 /* We only prefault standard pages for now */ 1514 if (unlikely(psize != mm_ctx_user_psize(&mm->context))) 1515 return false; 1516 1517 /* 1518 * Don't prefault if subpage protection is enabled for the EA. 1519 */ 1520 if (unlikely((psize == MMU_PAGE_4K) && subpage_protection(mm, ea))) 1521 return false; 1522 1523 return true; 1524 } 1525 #else 1526 static bool should_hash_preload(struct mm_struct *mm, unsigned long ea) 1527 { 1528 return true; 1529 } 1530 #endif 1531 1532 static void hash_preload(struct mm_struct *mm, unsigned long ea, 1533 bool is_exec, unsigned long trap) 1534 { 1535 int hugepage_shift; 1536 unsigned long vsid; 1537 pgd_t *pgdir; 1538 pte_t *ptep; 1539 unsigned long flags; 1540 int rc, ssize, update_flags = 0; 1541 unsigned long access = _PAGE_PRESENT | _PAGE_READ | (is_exec ? _PAGE_EXEC : 0); 1542 1543 BUG_ON(get_region_id(ea) != USER_REGION_ID); 1544 1545 if (!should_hash_preload(mm, ea)) 1546 return; 1547 1548 DBG_LOW("hash_preload(mm=%p, mm->pgdir=%p, ea=%016lx, access=%lx," 1549 " trap=%lx\n", mm, mm->pgd, ea, access, trap); 1550 1551 /* Get Linux PTE if available */ 1552 pgdir = mm->pgd; 1553 if (pgdir == NULL) 1554 return; 1555 1556 /* Get VSID */ 1557 ssize = user_segment_size(ea); 1558 vsid = get_user_vsid(&mm->context, ea, ssize); 1559 if (!vsid) 1560 return; 1561 /* 1562 * Hash doesn't like irqs. Walking linux page table with irq disabled 1563 * saves us from holding multiple locks. 1564 */ 1565 local_irq_save(flags); 1566 1567 /* 1568 * THP pages use update_mmu_cache_pmd. We don't do 1569 * hash preload there. Hence can ignore THP here 1570 */ 1571 ptep = find_current_mm_pte(pgdir, ea, NULL, &hugepage_shift); 1572 if (!ptep) 1573 goto out_exit; 1574 1575 WARN_ON(hugepage_shift); 1576 #ifdef CONFIG_PPC_64K_PAGES 1577 /* If either H_PAGE_4K_PFN or cache inhibited is set (and we are on 1578 * a 64K kernel), then we don't preload, hash_page() will take 1579 * care of it once we actually try to access the page. 1580 * That way we don't have to duplicate all of the logic for segment 1581 * page size demotion here 1582 */ 1583 if ((pte_val(*ptep) & H_PAGE_4K_PFN) || pte_ci(*ptep)) 1584 goto out_exit; 1585 #endif /* CONFIG_PPC_64K_PAGES */ 1586 1587 /* Is that local to this CPU ? */ 1588 if (mm_is_thread_local(mm)) 1589 update_flags |= HPTE_LOCAL_UPDATE; 1590 1591 /* Hash it in */ 1592 #ifdef CONFIG_PPC_64K_PAGES 1593 if (mm_ctx_user_psize(&mm->context) == MMU_PAGE_64K) 1594 rc = __hash_page_64K(ea, access, vsid, ptep, trap, 1595 update_flags, ssize); 1596 else 1597 #endif /* CONFIG_PPC_64K_PAGES */ 1598 rc = __hash_page_4K(ea, access, vsid, ptep, trap, update_flags, 1599 ssize, subpage_protection(mm, ea)); 1600 1601 /* Dump some info in case of hash insertion failure, they should 1602 * never happen so it is really useful to know if/when they do 1603 */ 1604 if (rc == -1) 1605 hash_failure_debug(ea, access, vsid, trap, ssize, 1606 mm_ctx_user_psize(&mm->context), 1607 mm_ctx_user_psize(&mm->context), 1608 pte_val(*ptep)); 1609 out_exit: 1610 local_irq_restore(flags); 1611 } 1612 1613 /* 1614 * This is called at the end of handling a user page fault, when the 1615 * fault has been handled by updating a PTE in the linux page tables. 1616 * We use it to preload an HPTE into the hash table corresponding to 1617 * the updated linux PTE. 1618 * 1619 * This must always be called with the pte lock held. 1620 */ 1621 void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, 1622 pte_t *ptep) 1623 { 1624 /* 1625 * We don't need to worry about _PAGE_PRESENT here because we are 1626 * called with either mm->page_table_lock held or ptl lock held 1627 */ 1628 unsigned long trap; 1629 bool is_exec; 1630 1631 if (radix_enabled()) { 1632 prefetch((void *)address); 1633 return; 1634 } 1635 1636 /* We only want HPTEs for linux PTEs that have _PAGE_ACCESSED set */ 1637 if (!pte_young(*ptep) || address >= TASK_SIZE) 1638 return; 1639 1640 /* 1641 * We try to figure out if we are coming from an instruction 1642 * access fault and pass that down to __hash_page so we avoid 1643 * double-faulting on execution of fresh text. We have to test 1644 * for regs NULL since init will get here first thing at boot. 1645 * 1646 * We also avoid filling the hash if not coming from a fault. 1647 */ 1648 1649 trap = current->thread.regs ? TRAP(current->thread.regs) : 0UL; 1650 switch (trap) { 1651 case 0x300: 1652 is_exec = false; 1653 break; 1654 case 0x400: 1655 is_exec = true; 1656 break; 1657 default: 1658 return; 1659 } 1660 1661 hash_preload(vma->vm_mm, address, is_exec, trap); 1662 } 1663 1664 #ifdef CONFIG_PPC_MEM_KEYS 1665 /* 1666 * Return the protection key associated with the given address and the 1667 * mm_struct. 1668 */ 1669 u16 get_mm_addr_key(struct mm_struct *mm, unsigned long address) 1670 { 1671 pte_t *ptep; 1672 u16 pkey = 0; 1673 unsigned long flags; 1674 1675 if (!mm || !mm->pgd) 1676 return 0; 1677 1678 local_irq_save(flags); 1679 ptep = find_linux_pte(mm->pgd, address, NULL, NULL); 1680 if (ptep) 1681 pkey = pte_to_pkey_bits(pte_val(READ_ONCE(*ptep))); 1682 local_irq_restore(flags); 1683 1684 return pkey; 1685 } 1686 #endif /* CONFIG_PPC_MEM_KEYS */ 1687 1688 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1689 static inline void tm_flush_hash_page(int local) 1690 { 1691 /* 1692 * Transactions are not aborted by tlbiel, only tlbie. Without, syncing a 1693 * page back to a block device w/PIO could pick up transactional data 1694 * (bad!) so we force an abort here. Before the sync the page will be 1695 * made read-only, which will flush_hash_page. BIG ISSUE here: if the 1696 * kernel uses a page from userspace without unmapping it first, it may 1697 * see the speculated version. 1698 */ 1699 if (local && cpu_has_feature(CPU_FTR_TM) && current->thread.regs && 1700 MSR_TM_ACTIVE(current->thread.regs->msr)) { 1701 tm_enable(); 1702 tm_abort(TM_CAUSE_TLBI); 1703 } 1704 } 1705 #else 1706 static inline void tm_flush_hash_page(int local) 1707 { 1708 } 1709 #endif 1710 1711 /* 1712 * Return the global hash slot, corresponding to the given PTE, which contains 1713 * the HPTE. 1714 */ 1715 unsigned long pte_get_hash_gslot(unsigned long vpn, unsigned long shift, 1716 int ssize, real_pte_t rpte, unsigned int subpg_index) 1717 { 1718 unsigned long hash, gslot, hidx; 1719 1720 hash = hpt_hash(vpn, shift, ssize); 1721 hidx = __rpte_to_hidx(rpte, subpg_index); 1722 if (hidx & _PTEIDX_SECONDARY) 1723 hash = ~hash; 1724 gslot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 1725 gslot += hidx & _PTEIDX_GROUP_IX; 1726 return gslot; 1727 } 1728 1729 /* 1730 * WARNING: This is called from hash_low_64.S, if you change this prototype, 1731 * do not forget to update the assembly call site ! 1732 */ 1733 void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize, 1734 unsigned long flags) 1735 { 1736 unsigned long index, shift, gslot; 1737 int local = flags & HPTE_LOCAL_UPDATE; 1738 1739 DBG_LOW("flush_hash_page(vpn=%016lx)\n", vpn); 1740 pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) { 1741 gslot = pte_get_hash_gslot(vpn, shift, ssize, pte, index); 1742 DBG_LOW(" sub %ld: gslot=%lx\n", index, gslot); 1743 /* 1744 * We use same base page size and actual psize, because we don't 1745 * use these functions for hugepage 1746 */ 1747 mmu_hash_ops.hpte_invalidate(gslot, vpn, psize, psize, 1748 ssize, local); 1749 } pte_iterate_hashed_end(); 1750 1751 tm_flush_hash_page(local); 1752 } 1753 1754 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1755 void flush_hash_hugepage(unsigned long vsid, unsigned long addr, 1756 pmd_t *pmdp, unsigned int psize, int ssize, 1757 unsigned long flags) 1758 { 1759 int i, max_hpte_count, valid; 1760 unsigned long s_addr; 1761 unsigned char *hpte_slot_array; 1762 unsigned long hidx, shift, vpn, hash, slot; 1763 int local = flags & HPTE_LOCAL_UPDATE; 1764 1765 s_addr = addr & HPAGE_PMD_MASK; 1766 hpte_slot_array = get_hpte_slot_array(pmdp); 1767 /* 1768 * IF we try to do a HUGE PTE update after a withdraw is done. 1769 * we will find the below NULL. This happens when we do 1770 * split_huge_pmd 1771 */ 1772 if (!hpte_slot_array) 1773 return; 1774 1775 if (mmu_hash_ops.hugepage_invalidate) { 1776 mmu_hash_ops.hugepage_invalidate(vsid, s_addr, hpte_slot_array, 1777 psize, ssize, local); 1778 goto tm_abort; 1779 } 1780 /* 1781 * No bluk hpte removal support, invalidate each entry 1782 */ 1783 shift = mmu_psize_defs[psize].shift; 1784 max_hpte_count = HPAGE_PMD_SIZE >> shift; 1785 for (i = 0; i < max_hpte_count; i++) { 1786 /* 1787 * 8 bits per each hpte entries 1788 * 000| [ secondary group (one bit) | hidx (3 bits) | valid bit] 1789 */ 1790 valid = hpte_valid(hpte_slot_array, i); 1791 if (!valid) 1792 continue; 1793 hidx = hpte_hash_index(hpte_slot_array, i); 1794 1795 /* get the vpn */ 1796 addr = s_addr + (i * (1ul << shift)); 1797 vpn = hpt_vpn(addr, vsid, ssize); 1798 hash = hpt_hash(vpn, shift, ssize); 1799 if (hidx & _PTEIDX_SECONDARY) 1800 hash = ~hash; 1801 1802 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 1803 slot += hidx & _PTEIDX_GROUP_IX; 1804 mmu_hash_ops.hpte_invalidate(slot, vpn, psize, 1805 MMU_PAGE_16M, ssize, local); 1806 } 1807 tm_abort: 1808 tm_flush_hash_page(local); 1809 } 1810 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1811 1812 void flush_hash_range(unsigned long number, int local) 1813 { 1814 if (mmu_hash_ops.flush_hash_range) 1815 mmu_hash_ops.flush_hash_range(number, local); 1816 else { 1817 int i; 1818 struct ppc64_tlb_batch *batch = 1819 this_cpu_ptr(&ppc64_tlb_batch); 1820 1821 for (i = 0; i < number; i++) 1822 flush_hash_page(batch->vpn[i], batch->pte[i], 1823 batch->psize, batch->ssize, local); 1824 } 1825 } 1826 1827 /* 1828 * low_hash_fault is called when we the low level hash code failed 1829 * to instert a PTE due to an hypervisor error 1830 */ 1831 void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc) 1832 { 1833 enum ctx_state prev_state = exception_enter(); 1834 1835 if (user_mode(regs)) { 1836 #ifdef CONFIG_PPC_SUBPAGE_PROT 1837 if (rc == -2) 1838 _exception(SIGSEGV, regs, SEGV_ACCERR, address); 1839 else 1840 #endif 1841 _exception(SIGBUS, regs, BUS_ADRERR, address); 1842 } else 1843 bad_page_fault(regs, address, SIGBUS); 1844 1845 exception_exit(prev_state); 1846 } 1847 1848 long hpte_insert_repeating(unsigned long hash, unsigned long vpn, 1849 unsigned long pa, unsigned long rflags, 1850 unsigned long vflags, int psize, int ssize) 1851 { 1852 unsigned long hpte_group; 1853 long slot; 1854 1855 repeat: 1856 hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP; 1857 1858 /* Insert into the hash table, primary slot */ 1859 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, vflags, 1860 psize, psize, ssize); 1861 1862 /* Primary is full, try the secondary */ 1863 if (unlikely(slot == -1)) { 1864 hpte_group = (~hash & htab_hash_mask) * HPTES_PER_GROUP; 1865 slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 1866 vflags | HPTE_V_SECONDARY, 1867 psize, psize, ssize); 1868 if (slot == -1) { 1869 if (mftb() & 0x1) 1870 hpte_group = (hash & htab_hash_mask) * 1871 HPTES_PER_GROUP; 1872 1873 mmu_hash_ops.hpte_remove(hpte_group); 1874 goto repeat; 1875 } 1876 } 1877 1878 return slot; 1879 } 1880 1881 #ifdef CONFIG_DEBUG_PAGEALLOC 1882 static void kernel_map_linear_page(unsigned long vaddr, unsigned long lmi) 1883 { 1884 unsigned long hash; 1885 unsigned long vsid = get_kernel_vsid(vaddr, mmu_kernel_ssize); 1886 unsigned long vpn = hpt_vpn(vaddr, vsid, mmu_kernel_ssize); 1887 unsigned long mode = htab_convert_pte_flags(pgprot_val(PAGE_KERNEL)); 1888 long ret; 1889 1890 hash = hpt_hash(vpn, PAGE_SHIFT, mmu_kernel_ssize); 1891 1892 /* Don't create HPTE entries for bad address */ 1893 if (!vsid) 1894 return; 1895 1896 ret = hpte_insert_repeating(hash, vpn, __pa(vaddr), mode, 1897 HPTE_V_BOLTED, 1898 mmu_linear_psize, mmu_kernel_ssize); 1899 1900 BUG_ON (ret < 0); 1901 spin_lock(&linear_map_hash_lock); 1902 BUG_ON(linear_map_hash_slots[lmi] & 0x80); 1903 linear_map_hash_slots[lmi] = ret | 0x80; 1904 spin_unlock(&linear_map_hash_lock); 1905 } 1906 1907 static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi) 1908 { 1909 unsigned long hash, hidx, slot; 1910 unsigned long vsid = get_kernel_vsid(vaddr, mmu_kernel_ssize); 1911 unsigned long vpn = hpt_vpn(vaddr, vsid, mmu_kernel_ssize); 1912 1913 hash = hpt_hash(vpn, PAGE_SHIFT, mmu_kernel_ssize); 1914 spin_lock(&linear_map_hash_lock); 1915 BUG_ON(!(linear_map_hash_slots[lmi] & 0x80)); 1916 hidx = linear_map_hash_slots[lmi] & 0x7f; 1917 linear_map_hash_slots[lmi] = 0; 1918 spin_unlock(&linear_map_hash_lock); 1919 if (hidx & _PTEIDX_SECONDARY) 1920 hash = ~hash; 1921 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 1922 slot += hidx & _PTEIDX_GROUP_IX; 1923 mmu_hash_ops.hpte_invalidate(slot, vpn, mmu_linear_psize, 1924 mmu_linear_psize, 1925 mmu_kernel_ssize, 0); 1926 } 1927 1928 void __kernel_map_pages(struct page *page, int numpages, int enable) 1929 { 1930 unsigned long flags, vaddr, lmi; 1931 int i; 1932 1933 local_irq_save(flags); 1934 for (i = 0; i < numpages; i++, page++) { 1935 vaddr = (unsigned long)page_address(page); 1936 lmi = __pa(vaddr) >> PAGE_SHIFT; 1937 if (lmi >= linear_map_hash_count) 1938 continue; 1939 if (enable) 1940 kernel_map_linear_page(vaddr, lmi); 1941 else 1942 kernel_unmap_linear_page(vaddr, lmi); 1943 } 1944 local_irq_restore(flags); 1945 } 1946 #endif /* CONFIG_DEBUG_PAGEALLOC */ 1947 1948 void hash__setup_initial_memory_limit(phys_addr_t first_memblock_base, 1949 phys_addr_t first_memblock_size) 1950 { 1951 /* 1952 * We don't currently support the first MEMBLOCK not mapping 0 1953 * physical on those processors 1954 */ 1955 BUG_ON(first_memblock_base != 0); 1956 1957 /* 1958 * On virtualized systems the first entry is our RMA region aka VRMA, 1959 * non-virtualized 64-bit hash MMU systems don't have a limitation 1960 * on real mode access. 1961 * 1962 * For guests on platforms before POWER9, we clamp the it limit to 1G 1963 * to avoid some funky things such as RTAS bugs etc... 1964 * 1965 * On POWER9 we limit to 1TB in case the host erroneously told us that 1966 * the RMA was >1TB. Effective address bits 0:23 are treated as zero 1967 * (meaning the access is aliased to zero i.e. addr = addr % 1TB) 1968 * for virtual real mode addressing and so it doesn't make sense to 1969 * have an area larger than 1TB as it can't be addressed. 1970 */ 1971 if (!early_cpu_has_feature(CPU_FTR_HVMODE)) { 1972 ppc64_rma_size = first_memblock_size; 1973 if (!early_cpu_has_feature(CPU_FTR_ARCH_300)) 1974 ppc64_rma_size = min_t(u64, ppc64_rma_size, 0x40000000); 1975 else 1976 ppc64_rma_size = min_t(u64, ppc64_rma_size, 1977 1UL << SID_SHIFT_1T); 1978 1979 /* Finally limit subsequent allocations */ 1980 memblock_set_current_limit(ppc64_rma_size); 1981 } else { 1982 ppc64_rma_size = ULONG_MAX; 1983 } 1984 } 1985 1986 #ifdef CONFIG_DEBUG_FS 1987 1988 static int hpt_order_get(void *data, u64 *val) 1989 { 1990 *val = ppc64_pft_size; 1991 return 0; 1992 } 1993 1994 static int hpt_order_set(void *data, u64 val) 1995 { 1996 int ret; 1997 1998 if (!mmu_hash_ops.resize_hpt) 1999 return -ENODEV; 2000 2001 cpus_read_lock(); 2002 ret = mmu_hash_ops.resize_hpt(val); 2003 cpus_read_unlock(); 2004 2005 return ret; 2006 } 2007 2008 DEFINE_DEBUGFS_ATTRIBUTE(fops_hpt_order, hpt_order_get, hpt_order_set, "%llu\n"); 2009 2010 static int __init hash64_debugfs(void) 2011 { 2012 if (!debugfs_create_file_unsafe("hpt_order", 0600, powerpc_debugfs_root, 2013 NULL, &fops_hpt_order)) { 2014 pr_err("lpar: unable to create hpt_order debugsfs file\n"); 2015 } 2016 2017 return 0; 2018 } 2019 machine_device_initcall(pseries, hash64_debugfs); 2020 #endif /* CONFIG_DEBUG_FS */ 2021 2022 void __init print_system_hash_info(void) 2023 { 2024 pr_info("ppc64_pft_size = 0x%llx\n", ppc64_pft_size); 2025 2026 if (htab_hash_mask) 2027 pr_info("htab_hash_mask = 0x%lx\n", htab_hash_mask); 2028 } 2029