1 /* 2 * pSeries_lpar.c 3 * Copyright (C) 2001 Todd Inglett, IBM Corporation 4 * 5 * pSeries LPAR support. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License as published by 9 * the Free Software Foundation; either version 2 of the License, or 10 * (at your option) any later version. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program; if not, write to the Free Software 19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 20 */ 21 22 /* Enables debugging of low-level hash table routines - careful! */ 23 #undef DEBUG 24 #define pr_fmt(fmt) "lpar: " fmt 25 26 #include <linux/kernel.h> 27 #include <linux/dma-mapping.h> 28 #include <linux/console.h> 29 #include <linux/export.h> 30 #include <linux/jump_label.h> 31 #include <linux/delay.h> 32 #include <linux/stop_machine.h> 33 #include <asm/processor.h> 34 #include <asm/mmu.h> 35 #include <asm/page.h> 36 #include <asm/pgtable.h> 37 #include <asm/machdep.h> 38 #include <asm/mmu_context.h> 39 #include <asm/iommu.h> 40 #include <asm/tlb.h> 41 #include <asm/prom.h> 42 #include <asm/cputable.h> 43 #include <asm/udbg.h> 44 #include <asm/smp.h> 45 #include <asm/trace.h> 46 #include <asm/firmware.h> 47 #include <asm/plpar_wrappers.h> 48 #include <asm/kexec.h> 49 #include <asm/fadump.h> 50 #include <asm/asm-prototypes.h> 51 #include <asm/debugfs.h> 52 53 #include "pseries.h" 54 55 /* Flag bits for H_BULK_REMOVE */ 56 #define HBR_REQUEST 0x4000000000000000UL 57 #define HBR_RESPONSE 0x8000000000000000UL 58 #define HBR_END 0xc000000000000000UL 59 #define HBR_AVPN 0x0200000000000000UL 60 #define HBR_ANDCOND 0x0100000000000000UL 61 62 63 /* in hvCall.S */ 64 EXPORT_SYMBOL(plpar_hcall); 65 EXPORT_SYMBOL(plpar_hcall9); 66 EXPORT_SYMBOL(plpar_hcall_norets); 67 68 void vpa_init(int cpu) 69 { 70 int hwcpu = get_hard_smp_processor_id(cpu); 71 unsigned long addr; 72 long ret; 73 struct paca_struct *pp; 74 struct dtl_entry *dtl; 75 76 /* 77 * The spec says it "may be problematic" if CPU x registers the VPA of 78 * CPU y. We should never do that, but wail if we ever do. 79 */ 80 WARN_ON(cpu != smp_processor_id()); 81 82 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 83 lppaca_of(cpu).vmxregs_in_use = 1; 84 85 if (cpu_has_feature(CPU_FTR_ARCH_207S)) 86 lppaca_of(cpu).ebb_regs_in_use = 1; 87 88 addr = __pa(&lppaca_of(cpu)); 89 ret = register_vpa(hwcpu, addr); 90 91 if (ret) { 92 pr_err("WARNING: VPA registration for cpu %d (hw %d) of area " 93 "%lx failed with %ld\n", cpu, hwcpu, addr, ret); 94 return; 95 } 96 97 #ifdef CONFIG_PPC_BOOK3S_64 98 /* 99 * PAPR says this feature is SLB-Buffer but firmware never 100 * reports that. All SPLPAR support SLB shadow buffer. 101 */ 102 if (!radix_enabled() && firmware_has_feature(FW_FEATURE_SPLPAR)) { 103 addr = __pa(paca_ptrs[cpu]->slb_shadow_ptr); 104 ret = register_slb_shadow(hwcpu, addr); 105 if (ret) 106 pr_err("WARNING: SLB shadow buffer registration for " 107 "cpu %d (hw %d) of area %lx failed with %ld\n", 108 cpu, hwcpu, addr, ret); 109 } 110 #endif /* CONFIG_PPC_BOOK3S_64 */ 111 112 /* 113 * Register dispatch trace log, if one has been allocated. 114 */ 115 pp = paca_ptrs[cpu]; 116 dtl = pp->dispatch_log; 117 if (dtl) { 118 pp->dtl_ridx = 0; 119 pp->dtl_curr = dtl; 120 lppaca_of(cpu).dtl_idx = 0; 121 122 /* hypervisor reads buffer length from this field */ 123 dtl->enqueue_to_dispatch_time = cpu_to_be32(DISPATCH_LOG_BYTES); 124 ret = register_dtl(hwcpu, __pa(dtl)); 125 if (ret) 126 pr_err("WARNING: DTL registration of cpu %d (hw %d) " 127 "failed with %ld\n", smp_processor_id(), 128 hwcpu, ret); 129 lppaca_of(cpu).dtl_enable_mask = 2; 130 } 131 } 132 133 #ifdef CONFIG_PPC_BOOK3S_64 134 135 static long pSeries_lpar_hpte_insert(unsigned long hpte_group, 136 unsigned long vpn, unsigned long pa, 137 unsigned long rflags, unsigned long vflags, 138 int psize, int apsize, int ssize) 139 { 140 unsigned long lpar_rc; 141 unsigned long flags; 142 unsigned long slot; 143 unsigned long hpte_v, hpte_r; 144 145 if (!(vflags & HPTE_V_BOLTED)) 146 pr_devel("hpte_insert(group=%lx, vpn=%016lx, " 147 "pa=%016lx, rflags=%lx, vflags=%lx, psize=%d)\n", 148 hpte_group, vpn, pa, rflags, vflags, psize); 149 150 hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID; 151 hpte_r = hpte_encode_r(pa, psize, apsize) | rflags; 152 153 if (!(vflags & HPTE_V_BOLTED)) 154 pr_devel(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r); 155 156 /* Now fill in the actual HPTE */ 157 /* Set CEC cookie to 0 */ 158 /* Zero page = 0 */ 159 /* I-cache Invalidate = 0 */ 160 /* I-cache synchronize = 0 */ 161 /* Exact = 0 */ 162 flags = 0; 163 164 if (firmware_has_feature(FW_FEATURE_XCMO) && !(hpte_r & HPTE_R_N)) 165 flags |= H_COALESCE_CAND; 166 167 lpar_rc = plpar_pte_enter(flags, hpte_group, hpte_v, hpte_r, &slot); 168 if (unlikely(lpar_rc == H_PTEG_FULL)) { 169 pr_devel("Hash table group is full\n"); 170 return -1; 171 } 172 173 /* 174 * Since we try and ioremap PHBs we don't own, the pte insert 175 * will fail. However we must catch the failure in hash_page 176 * or we will loop forever, so return -2 in this case. 177 */ 178 if (unlikely(lpar_rc != H_SUCCESS)) { 179 pr_err("Failed hash pte insert with error %ld\n", lpar_rc); 180 return -2; 181 } 182 if (!(vflags & HPTE_V_BOLTED)) 183 pr_devel(" -> slot: %lu\n", slot & 7); 184 185 /* Because of iSeries, we have to pass down the secondary 186 * bucket bit here as well 187 */ 188 return (slot & 7) | (!!(vflags & HPTE_V_SECONDARY) << 3); 189 } 190 191 static DEFINE_SPINLOCK(pSeries_lpar_tlbie_lock); 192 193 static long pSeries_lpar_hpte_remove(unsigned long hpte_group) 194 { 195 unsigned long slot_offset; 196 unsigned long lpar_rc; 197 int i; 198 unsigned long dummy1, dummy2; 199 200 /* pick a random slot to start at */ 201 slot_offset = mftb() & 0x7; 202 203 for (i = 0; i < HPTES_PER_GROUP; i++) { 204 205 /* don't remove a bolted entry */ 206 lpar_rc = plpar_pte_remove(H_ANDCOND, hpte_group + slot_offset, 207 (0x1UL << 4), &dummy1, &dummy2); 208 if (lpar_rc == H_SUCCESS) 209 return i; 210 211 /* 212 * The test for adjunct partition is performed before the 213 * ANDCOND test. H_RESOURCE may be returned, so we need to 214 * check for that as well. 215 */ 216 BUG_ON(lpar_rc != H_NOT_FOUND && lpar_rc != H_RESOURCE); 217 218 slot_offset++; 219 slot_offset &= 0x7; 220 } 221 222 return -1; 223 } 224 225 static void manual_hpte_clear_all(void) 226 { 227 unsigned long size_bytes = 1UL << ppc64_pft_size; 228 unsigned long hpte_count = size_bytes >> 4; 229 struct { 230 unsigned long pteh; 231 unsigned long ptel; 232 } ptes[4]; 233 long lpar_rc; 234 unsigned long i, j; 235 236 /* Read in batches of 4, 237 * invalidate only valid entries not in the VRMA 238 * hpte_count will be a multiple of 4 239 */ 240 for (i = 0; i < hpte_count; i += 4) { 241 lpar_rc = plpar_pte_read_4_raw(0, i, (void *)ptes); 242 if (lpar_rc != H_SUCCESS) { 243 pr_info("Failed to read hash page table at %ld err %ld\n", 244 i, lpar_rc); 245 continue; 246 } 247 for (j = 0; j < 4; j++){ 248 if ((ptes[j].pteh & HPTE_V_VRMA_MASK) == 249 HPTE_V_VRMA_MASK) 250 continue; 251 if (ptes[j].pteh & HPTE_V_VALID) 252 plpar_pte_remove_raw(0, i + j, 0, 253 &(ptes[j].pteh), &(ptes[j].ptel)); 254 } 255 } 256 } 257 258 static int hcall_hpte_clear_all(void) 259 { 260 int rc; 261 262 do { 263 rc = plpar_hcall_norets(H_CLEAR_HPT); 264 } while (rc == H_CONTINUE); 265 266 return rc; 267 } 268 269 static void pseries_hpte_clear_all(void) 270 { 271 int rc; 272 273 rc = hcall_hpte_clear_all(); 274 if (rc != H_SUCCESS) 275 manual_hpte_clear_all(); 276 277 #ifdef __LITTLE_ENDIAN__ 278 /* 279 * Reset exceptions to big endian. 280 * 281 * FIXME this is a hack for kexec, we need to reset the exception 282 * endian before starting the new kernel and this is a convenient place 283 * to do it. 284 * 285 * This is also called on boot when a fadump happens. In that case we 286 * must not change the exception endian mode. 287 */ 288 if (firmware_has_feature(FW_FEATURE_SET_MODE) && !is_fadump_active()) 289 pseries_big_endian_exceptions(); 290 #endif 291 } 292 293 /* 294 * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and 295 * the low 3 bits of flags happen to line up. So no transform is needed. 296 * We can probably optimize here and assume the high bits of newpp are 297 * already zero. For now I am paranoid. 298 */ 299 static long pSeries_lpar_hpte_updatepp(unsigned long slot, 300 unsigned long newpp, 301 unsigned long vpn, 302 int psize, int apsize, 303 int ssize, unsigned long inv_flags) 304 { 305 unsigned long lpar_rc; 306 unsigned long flags; 307 unsigned long want_v; 308 309 want_v = hpte_encode_avpn(vpn, psize, ssize); 310 311 flags = (newpp & 7) | H_AVPN; 312 if (mmu_has_feature(MMU_FTR_KERNEL_RO)) 313 /* Move pp0 into bit 8 (IBM 55) */ 314 flags |= (newpp & HPTE_R_PP0) >> 55; 315 316 pr_devel(" update: avpnv=%016lx, hash=%016lx, f=%lx, psize: %d ...", 317 want_v, slot, flags, psize); 318 319 lpar_rc = plpar_pte_protect(flags, slot, want_v); 320 321 if (lpar_rc == H_NOT_FOUND) { 322 pr_devel("not found !\n"); 323 return -1; 324 } 325 326 pr_devel("ok\n"); 327 328 BUG_ON(lpar_rc != H_SUCCESS); 329 330 return 0; 331 } 332 333 static long __pSeries_lpar_hpte_find(unsigned long want_v, unsigned long hpte_group) 334 { 335 long lpar_rc; 336 unsigned long i, j; 337 struct { 338 unsigned long pteh; 339 unsigned long ptel; 340 } ptes[4]; 341 342 for (i = 0; i < HPTES_PER_GROUP; i += 4, hpte_group += 4) { 343 344 lpar_rc = plpar_pte_read_4(0, hpte_group, (void *)ptes); 345 if (lpar_rc != H_SUCCESS) { 346 pr_info("Failed to read hash page table at %ld err %ld\n", 347 hpte_group, lpar_rc); 348 continue; 349 } 350 351 for (j = 0; j < 4; j++) { 352 if (HPTE_V_COMPARE(ptes[j].pteh, want_v) && 353 (ptes[j].pteh & HPTE_V_VALID)) 354 return i + j; 355 } 356 } 357 358 return -1; 359 } 360 361 static long pSeries_lpar_hpte_find(unsigned long vpn, int psize, int ssize) 362 { 363 long slot; 364 unsigned long hash; 365 unsigned long want_v; 366 unsigned long hpte_group; 367 368 hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, ssize); 369 want_v = hpte_encode_avpn(vpn, psize, ssize); 370 371 /* Bolted entries are always in the primary group */ 372 hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP; 373 slot = __pSeries_lpar_hpte_find(want_v, hpte_group); 374 if (slot < 0) 375 return -1; 376 return hpte_group + slot; 377 } 378 379 static void pSeries_lpar_hpte_updateboltedpp(unsigned long newpp, 380 unsigned long ea, 381 int psize, int ssize) 382 { 383 unsigned long vpn; 384 unsigned long lpar_rc, slot, vsid, flags; 385 386 vsid = get_kernel_vsid(ea, ssize); 387 vpn = hpt_vpn(ea, vsid, ssize); 388 389 slot = pSeries_lpar_hpte_find(vpn, psize, ssize); 390 BUG_ON(slot == -1); 391 392 flags = newpp & 7; 393 if (mmu_has_feature(MMU_FTR_KERNEL_RO)) 394 /* Move pp0 into bit 8 (IBM 55) */ 395 flags |= (newpp & HPTE_R_PP0) >> 55; 396 397 lpar_rc = plpar_pte_protect(flags, slot, 0); 398 399 BUG_ON(lpar_rc != H_SUCCESS); 400 } 401 402 static void pSeries_lpar_hpte_invalidate(unsigned long slot, unsigned long vpn, 403 int psize, int apsize, 404 int ssize, int local) 405 { 406 unsigned long want_v; 407 unsigned long lpar_rc; 408 unsigned long dummy1, dummy2; 409 410 pr_devel(" inval : slot=%lx, vpn=%016lx, psize: %d, local: %d\n", 411 slot, vpn, psize, local); 412 413 want_v = hpte_encode_avpn(vpn, psize, ssize); 414 lpar_rc = plpar_pte_remove(H_AVPN, slot, want_v, &dummy1, &dummy2); 415 if (lpar_rc == H_NOT_FOUND) 416 return; 417 418 BUG_ON(lpar_rc != H_SUCCESS); 419 } 420 421 422 /* 423 * As defined in the PAPR's section 14.5.4.1.8 424 * The control mask doesn't include the returned reference and change bit from 425 * the processed PTE. 426 */ 427 #define HBLKR_AVPN 0x0100000000000000UL 428 #define HBLKR_CTRL_MASK 0xf800000000000000UL 429 #define HBLKR_CTRL_SUCCESS 0x8000000000000000UL 430 #define HBLKR_CTRL_ERRNOTFOUND 0x8800000000000000UL 431 #define HBLKR_CTRL_ERRBUSY 0xa000000000000000UL 432 433 /** 434 * H_BLOCK_REMOVE caller. 435 * @idx should point to the latest @param entry set with a PTEX. 436 * If PTE cannot be processed because another CPUs has already locked that 437 * group, those entries are put back in @param starting at index 1. 438 * If entries has to be retried and @retry_busy is set to true, these entries 439 * are retried until success. If @retry_busy is set to false, the returned 440 * is the number of entries yet to process. 441 */ 442 static unsigned long call_block_remove(unsigned long idx, unsigned long *param, 443 bool retry_busy) 444 { 445 unsigned long i, rc, new_idx; 446 unsigned long retbuf[PLPAR_HCALL9_BUFSIZE]; 447 448 if (idx < 2) { 449 pr_warn("Unexpected empty call to H_BLOCK_REMOVE"); 450 return 0; 451 } 452 again: 453 new_idx = 0; 454 if (idx > PLPAR_HCALL9_BUFSIZE) { 455 pr_err("Too many PTEs (%lu) for H_BLOCK_REMOVE", idx); 456 idx = PLPAR_HCALL9_BUFSIZE; 457 } else if (idx < PLPAR_HCALL9_BUFSIZE) 458 param[idx] = HBR_END; 459 460 rc = plpar_hcall9(H_BLOCK_REMOVE, retbuf, 461 param[0], /* AVA */ 462 param[1], param[2], param[3], param[4], /* TS0-7 */ 463 param[5], param[6], param[7], param[8]); 464 if (rc == H_SUCCESS) 465 return 0; 466 467 BUG_ON(rc != H_PARTIAL); 468 469 /* Check that the unprocessed entries were 'not found' or 'busy' */ 470 for (i = 0; i < idx-1; i++) { 471 unsigned long ctrl = retbuf[i] & HBLKR_CTRL_MASK; 472 473 if (ctrl == HBLKR_CTRL_ERRBUSY) { 474 param[++new_idx] = param[i+1]; 475 continue; 476 } 477 478 BUG_ON(ctrl != HBLKR_CTRL_SUCCESS 479 && ctrl != HBLKR_CTRL_ERRNOTFOUND); 480 } 481 482 /* 483 * If there were entries found busy, retry these entries if requested, 484 * of if all the entries have to be retried. 485 */ 486 if (new_idx && (retry_busy || new_idx == (PLPAR_HCALL9_BUFSIZE-1))) { 487 idx = new_idx + 1; 488 goto again; 489 } 490 491 return new_idx; 492 } 493 494 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 495 /* 496 * Limit iterations holding pSeries_lpar_tlbie_lock to 3. We also need 497 * to make sure that we avoid bouncing the hypervisor tlbie lock. 498 */ 499 #define PPC64_HUGE_HPTE_BATCH 12 500 501 static void hugepage_block_invalidate(unsigned long *slot, unsigned long *vpn, 502 int count, int psize, int ssize) 503 { 504 unsigned long param[PLPAR_HCALL9_BUFSIZE]; 505 unsigned long shift, current_vpgb, vpgb; 506 int i, pix = 0; 507 508 shift = mmu_psize_defs[psize].shift; 509 510 for (i = 0; i < count; i++) { 511 /* 512 * Shifting 3 bits more on the right to get a 513 * 8 pages aligned virtual addresse. 514 */ 515 vpgb = (vpn[i] >> (shift - VPN_SHIFT + 3)); 516 if (!pix || vpgb != current_vpgb) { 517 /* 518 * Need to start a new 8 pages block, flush 519 * the current one if needed. 520 */ 521 if (pix) 522 (void)call_block_remove(pix, param, true); 523 current_vpgb = vpgb; 524 param[0] = hpte_encode_avpn(vpn[i], psize, ssize); 525 pix = 1; 526 } 527 528 param[pix++] = HBR_REQUEST | HBLKR_AVPN | slot[i]; 529 if (pix == PLPAR_HCALL9_BUFSIZE) { 530 pix = call_block_remove(pix, param, false); 531 /* 532 * pix = 0 means that all the entries were 533 * removed, we can start a new block. 534 * Otherwise, this means that there are entries 535 * to retry, and pix points to latest one, so 536 * we should increment it and try to continue 537 * the same block. 538 */ 539 if (pix) 540 pix++; 541 } 542 } 543 if (pix) 544 (void)call_block_remove(pix, param, true); 545 } 546 547 static void hugepage_bulk_invalidate(unsigned long *slot, unsigned long *vpn, 548 int count, int psize, int ssize) 549 { 550 unsigned long param[PLPAR_HCALL9_BUFSIZE]; 551 int i = 0, pix = 0, rc; 552 553 for (i = 0; i < count; i++) { 554 555 if (!firmware_has_feature(FW_FEATURE_BULK_REMOVE)) { 556 pSeries_lpar_hpte_invalidate(slot[i], vpn[i], psize, 0, 557 ssize, 0); 558 } else { 559 param[pix] = HBR_REQUEST | HBR_AVPN | slot[i]; 560 param[pix+1] = hpte_encode_avpn(vpn[i], psize, ssize); 561 pix += 2; 562 if (pix == 8) { 563 rc = plpar_hcall9(H_BULK_REMOVE, param, 564 param[0], param[1], param[2], 565 param[3], param[4], param[5], 566 param[6], param[7]); 567 BUG_ON(rc != H_SUCCESS); 568 pix = 0; 569 } 570 } 571 } 572 if (pix) { 573 param[pix] = HBR_END; 574 rc = plpar_hcall9(H_BULK_REMOVE, param, param[0], param[1], 575 param[2], param[3], param[4], param[5], 576 param[6], param[7]); 577 BUG_ON(rc != H_SUCCESS); 578 } 579 } 580 581 static inline void __pSeries_lpar_hugepage_invalidate(unsigned long *slot, 582 unsigned long *vpn, 583 int count, int psize, 584 int ssize) 585 { 586 unsigned long flags = 0; 587 int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); 588 589 if (lock_tlbie) 590 spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags); 591 592 if (firmware_has_feature(FW_FEATURE_BLOCK_REMOVE)) 593 hugepage_block_invalidate(slot, vpn, count, psize, ssize); 594 else 595 hugepage_bulk_invalidate(slot, vpn, count, psize, ssize); 596 597 if (lock_tlbie) 598 spin_unlock_irqrestore(&pSeries_lpar_tlbie_lock, flags); 599 } 600 601 static void pSeries_lpar_hugepage_invalidate(unsigned long vsid, 602 unsigned long addr, 603 unsigned char *hpte_slot_array, 604 int psize, int ssize, int local) 605 { 606 int i, index = 0; 607 unsigned long s_addr = addr; 608 unsigned int max_hpte_count, valid; 609 unsigned long vpn_array[PPC64_HUGE_HPTE_BATCH]; 610 unsigned long slot_array[PPC64_HUGE_HPTE_BATCH]; 611 unsigned long shift, hidx, vpn = 0, hash, slot; 612 613 shift = mmu_psize_defs[psize].shift; 614 max_hpte_count = 1U << (PMD_SHIFT - shift); 615 616 for (i = 0; i < max_hpte_count; i++) { 617 valid = hpte_valid(hpte_slot_array, i); 618 if (!valid) 619 continue; 620 hidx = hpte_hash_index(hpte_slot_array, i); 621 622 /* get the vpn */ 623 addr = s_addr + (i * (1ul << shift)); 624 vpn = hpt_vpn(addr, vsid, ssize); 625 hash = hpt_hash(vpn, shift, ssize); 626 if (hidx & _PTEIDX_SECONDARY) 627 hash = ~hash; 628 629 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 630 slot += hidx & _PTEIDX_GROUP_IX; 631 632 slot_array[index] = slot; 633 vpn_array[index] = vpn; 634 if (index == PPC64_HUGE_HPTE_BATCH - 1) { 635 /* 636 * Now do a bluk invalidate 637 */ 638 __pSeries_lpar_hugepage_invalidate(slot_array, 639 vpn_array, 640 PPC64_HUGE_HPTE_BATCH, 641 psize, ssize); 642 index = 0; 643 } else 644 index++; 645 } 646 if (index) 647 __pSeries_lpar_hugepage_invalidate(slot_array, vpn_array, 648 index, psize, ssize); 649 } 650 #else 651 static void pSeries_lpar_hugepage_invalidate(unsigned long vsid, 652 unsigned long addr, 653 unsigned char *hpte_slot_array, 654 int psize, int ssize, int local) 655 { 656 WARN(1, "%s called without THP support\n", __func__); 657 } 658 #endif 659 660 static int pSeries_lpar_hpte_removebolted(unsigned long ea, 661 int psize, int ssize) 662 { 663 unsigned long vpn; 664 unsigned long slot, vsid; 665 666 vsid = get_kernel_vsid(ea, ssize); 667 vpn = hpt_vpn(ea, vsid, ssize); 668 669 slot = pSeries_lpar_hpte_find(vpn, psize, ssize); 670 if (slot == -1) 671 return -ENOENT; 672 673 /* 674 * lpar doesn't use the passed actual page size 675 */ 676 pSeries_lpar_hpte_invalidate(slot, vpn, psize, 0, ssize, 0); 677 return 0; 678 } 679 680 681 static inline unsigned long compute_slot(real_pte_t pte, 682 unsigned long vpn, 683 unsigned long index, 684 unsigned long shift, 685 int ssize) 686 { 687 unsigned long slot, hash, hidx; 688 689 hash = hpt_hash(vpn, shift, ssize); 690 hidx = __rpte_to_hidx(pte, index); 691 if (hidx & _PTEIDX_SECONDARY) 692 hash = ~hash; 693 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 694 slot += hidx & _PTEIDX_GROUP_IX; 695 return slot; 696 } 697 698 /** 699 * The hcall H_BLOCK_REMOVE implies that the virtual pages to processed are 700 * "all within the same naturally aligned 8 page virtual address block". 701 */ 702 static void do_block_remove(unsigned long number, struct ppc64_tlb_batch *batch, 703 unsigned long *param) 704 { 705 unsigned long vpn; 706 unsigned long i, pix = 0; 707 unsigned long index, shift, slot, current_vpgb, vpgb; 708 real_pte_t pte; 709 int psize, ssize; 710 711 psize = batch->psize; 712 ssize = batch->ssize; 713 714 for (i = 0; i < number; i++) { 715 vpn = batch->vpn[i]; 716 pte = batch->pte[i]; 717 pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) { 718 /* 719 * Shifting 3 bits more on the right to get a 720 * 8 pages aligned virtual addresse. 721 */ 722 vpgb = (vpn >> (shift - VPN_SHIFT + 3)); 723 if (!pix || vpgb != current_vpgb) { 724 /* 725 * Need to start a new 8 pages block, flush 726 * the current one if needed. 727 */ 728 if (pix) 729 (void)call_block_remove(pix, param, 730 true); 731 current_vpgb = vpgb; 732 param[0] = hpte_encode_avpn(vpn, psize, 733 ssize); 734 pix = 1; 735 } 736 737 slot = compute_slot(pte, vpn, index, shift, ssize); 738 param[pix++] = HBR_REQUEST | HBLKR_AVPN | slot; 739 740 if (pix == PLPAR_HCALL9_BUFSIZE) { 741 pix = call_block_remove(pix, param, false); 742 /* 743 * pix = 0 means that all the entries were 744 * removed, we can start a new block. 745 * Otherwise, this means that there are entries 746 * to retry, and pix points to latest one, so 747 * we should increment it and try to continue 748 * the same block. 749 */ 750 if (pix) 751 pix++; 752 } 753 } pte_iterate_hashed_end(); 754 } 755 756 if (pix) 757 (void)call_block_remove(pix, param, true); 758 } 759 760 /* 761 * Take a spinlock around flushes to avoid bouncing the hypervisor tlbie 762 * lock. 763 */ 764 static void pSeries_lpar_flush_hash_range(unsigned long number, int local) 765 { 766 unsigned long vpn; 767 unsigned long i, pix, rc; 768 unsigned long flags = 0; 769 struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch); 770 int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); 771 unsigned long param[PLPAR_HCALL9_BUFSIZE]; 772 unsigned long index, shift, slot; 773 real_pte_t pte; 774 int psize, ssize; 775 776 if (lock_tlbie) 777 spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags); 778 779 if (firmware_has_feature(FW_FEATURE_BLOCK_REMOVE)) { 780 do_block_remove(number, batch, param); 781 goto out; 782 } 783 784 psize = batch->psize; 785 ssize = batch->ssize; 786 pix = 0; 787 for (i = 0; i < number; i++) { 788 vpn = batch->vpn[i]; 789 pte = batch->pte[i]; 790 pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) { 791 slot = compute_slot(pte, vpn, index, shift, ssize); 792 if (!firmware_has_feature(FW_FEATURE_BULK_REMOVE)) { 793 /* 794 * lpar doesn't use the passed actual page size 795 */ 796 pSeries_lpar_hpte_invalidate(slot, vpn, psize, 797 0, ssize, local); 798 } else { 799 param[pix] = HBR_REQUEST | HBR_AVPN | slot; 800 param[pix+1] = hpte_encode_avpn(vpn, psize, 801 ssize); 802 pix += 2; 803 if (pix == 8) { 804 rc = plpar_hcall9(H_BULK_REMOVE, param, 805 param[0], param[1], param[2], 806 param[3], param[4], param[5], 807 param[6], param[7]); 808 BUG_ON(rc != H_SUCCESS); 809 pix = 0; 810 } 811 } 812 } pte_iterate_hashed_end(); 813 } 814 if (pix) { 815 param[pix] = HBR_END; 816 rc = plpar_hcall9(H_BULK_REMOVE, param, param[0], param[1], 817 param[2], param[3], param[4], param[5], 818 param[6], param[7]); 819 BUG_ON(rc != H_SUCCESS); 820 } 821 822 out: 823 if (lock_tlbie) 824 spin_unlock_irqrestore(&pSeries_lpar_tlbie_lock, flags); 825 } 826 827 static int __init disable_bulk_remove(char *str) 828 { 829 if (strcmp(str, "off") == 0 && 830 firmware_has_feature(FW_FEATURE_BULK_REMOVE)) { 831 pr_info("Disabling BULK_REMOVE firmware feature"); 832 powerpc_firmware_features &= ~FW_FEATURE_BULK_REMOVE; 833 } 834 return 1; 835 } 836 837 __setup("bulk_remove=", disable_bulk_remove); 838 839 #define HPT_RESIZE_TIMEOUT 10000 /* ms */ 840 841 struct hpt_resize_state { 842 unsigned long shift; 843 int commit_rc; 844 }; 845 846 static int pseries_lpar_resize_hpt_commit(void *data) 847 { 848 struct hpt_resize_state *state = data; 849 850 state->commit_rc = plpar_resize_hpt_commit(0, state->shift); 851 if (state->commit_rc != H_SUCCESS) 852 return -EIO; 853 854 /* Hypervisor has transitioned the HTAB, update our globals */ 855 ppc64_pft_size = state->shift; 856 htab_size_bytes = 1UL << ppc64_pft_size; 857 htab_hash_mask = (htab_size_bytes >> 7) - 1; 858 859 return 0; 860 } 861 862 /* Must be called in user context */ 863 static int pseries_lpar_resize_hpt(unsigned long shift) 864 { 865 struct hpt_resize_state state = { 866 .shift = shift, 867 .commit_rc = H_FUNCTION, 868 }; 869 unsigned int delay, total_delay = 0; 870 int rc; 871 ktime_t t0, t1, t2; 872 873 might_sleep(); 874 875 if (!firmware_has_feature(FW_FEATURE_HPT_RESIZE)) 876 return -ENODEV; 877 878 pr_info("Attempting to resize HPT to shift %lu\n", shift); 879 880 t0 = ktime_get(); 881 882 rc = plpar_resize_hpt_prepare(0, shift); 883 while (H_IS_LONG_BUSY(rc)) { 884 delay = get_longbusy_msecs(rc); 885 total_delay += delay; 886 if (total_delay > HPT_RESIZE_TIMEOUT) { 887 /* prepare with shift==0 cancels an in-progress resize */ 888 rc = plpar_resize_hpt_prepare(0, 0); 889 if (rc != H_SUCCESS) 890 pr_warn("Unexpected error %d cancelling timed out HPT resize\n", 891 rc); 892 return -ETIMEDOUT; 893 } 894 msleep(delay); 895 rc = plpar_resize_hpt_prepare(0, shift); 896 }; 897 898 switch (rc) { 899 case H_SUCCESS: 900 /* Continue on */ 901 break; 902 903 case H_PARAMETER: 904 return -EINVAL; 905 case H_RESOURCE: 906 return -EPERM; 907 default: 908 pr_warn("Unexpected error %d from H_RESIZE_HPT_PREPARE\n", rc); 909 return -EIO; 910 } 911 912 t1 = ktime_get(); 913 914 rc = stop_machine(pseries_lpar_resize_hpt_commit, &state, NULL); 915 916 t2 = ktime_get(); 917 918 if (rc != 0) { 919 switch (state.commit_rc) { 920 case H_PTEG_FULL: 921 pr_warn("Hash collision while resizing HPT\n"); 922 return -ENOSPC; 923 924 default: 925 pr_warn("Unexpected error %d from H_RESIZE_HPT_COMMIT\n", 926 state.commit_rc); 927 return -EIO; 928 }; 929 } 930 931 pr_info("HPT resize to shift %lu complete (%lld ms / %lld ms)\n", 932 shift, (long long) ktime_ms_delta(t1, t0), 933 (long long) ktime_ms_delta(t2, t1)); 934 935 return 0; 936 } 937 938 static int pseries_lpar_register_process_table(unsigned long base, 939 unsigned long page_size, unsigned long table_size) 940 { 941 long rc; 942 unsigned long flags = 0; 943 944 if (table_size) 945 flags |= PROC_TABLE_NEW; 946 if (radix_enabled()) 947 flags |= PROC_TABLE_RADIX | PROC_TABLE_GTSE; 948 else 949 flags |= PROC_TABLE_HPT_SLB; 950 for (;;) { 951 rc = plpar_hcall_norets(H_REGISTER_PROC_TBL, flags, base, 952 page_size, table_size); 953 if (!H_IS_LONG_BUSY(rc)) 954 break; 955 mdelay(get_longbusy_msecs(rc)); 956 } 957 if (rc != H_SUCCESS) { 958 pr_err("Failed to register process table (rc=%ld)\n", rc); 959 BUG(); 960 } 961 return rc; 962 } 963 964 void __init hpte_init_pseries(void) 965 { 966 mmu_hash_ops.hpte_invalidate = pSeries_lpar_hpte_invalidate; 967 mmu_hash_ops.hpte_updatepp = pSeries_lpar_hpte_updatepp; 968 mmu_hash_ops.hpte_updateboltedpp = pSeries_lpar_hpte_updateboltedpp; 969 mmu_hash_ops.hpte_insert = pSeries_lpar_hpte_insert; 970 mmu_hash_ops.hpte_remove = pSeries_lpar_hpte_remove; 971 mmu_hash_ops.hpte_removebolted = pSeries_lpar_hpte_removebolted; 972 mmu_hash_ops.flush_hash_range = pSeries_lpar_flush_hash_range; 973 mmu_hash_ops.hpte_clear_all = pseries_hpte_clear_all; 974 mmu_hash_ops.hugepage_invalidate = pSeries_lpar_hugepage_invalidate; 975 register_process_table = pseries_lpar_register_process_table; 976 977 if (firmware_has_feature(FW_FEATURE_HPT_RESIZE)) 978 mmu_hash_ops.resize_hpt = pseries_lpar_resize_hpt; 979 } 980 981 void radix_init_pseries(void) 982 { 983 pr_info("Using radix MMU under hypervisor\n"); 984 register_process_table = pseries_lpar_register_process_table; 985 } 986 987 #ifdef CONFIG_PPC_SMLPAR 988 #define CMO_FREE_HINT_DEFAULT 1 989 static int cmo_free_hint_flag = CMO_FREE_HINT_DEFAULT; 990 991 static int __init cmo_free_hint(char *str) 992 { 993 char *parm; 994 parm = strstrip(str); 995 996 if (strcasecmp(parm, "no") == 0 || strcasecmp(parm, "off") == 0) { 997 pr_info("%s: CMO free page hinting is not active.\n", __func__); 998 cmo_free_hint_flag = 0; 999 return 1; 1000 } 1001 1002 cmo_free_hint_flag = 1; 1003 pr_info("%s: CMO free page hinting is active.\n", __func__); 1004 1005 if (strcasecmp(parm, "yes") == 0 || strcasecmp(parm, "on") == 0) 1006 return 1; 1007 1008 return 0; 1009 } 1010 1011 __setup("cmo_free_hint=", cmo_free_hint); 1012 1013 static void pSeries_set_page_state(struct page *page, int order, 1014 unsigned long state) 1015 { 1016 int i, j; 1017 unsigned long cmo_page_sz, addr; 1018 1019 cmo_page_sz = cmo_get_page_size(); 1020 addr = __pa((unsigned long)page_address(page)); 1021 1022 for (i = 0; i < (1 << order); i++, addr += PAGE_SIZE) { 1023 for (j = 0; j < PAGE_SIZE; j += cmo_page_sz) 1024 plpar_hcall_norets(H_PAGE_INIT, state, addr + j, 0); 1025 } 1026 } 1027 1028 void arch_free_page(struct page *page, int order) 1029 { 1030 if (radix_enabled()) 1031 return; 1032 if (!cmo_free_hint_flag || !firmware_has_feature(FW_FEATURE_CMO)) 1033 return; 1034 1035 pSeries_set_page_state(page, order, H_PAGE_SET_UNUSED); 1036 } 1037 EXPORT_SYMBOL(arch_free_page); 1038 1039 #endif /* CONFIG_PPC_SMLPAR */ 1040 #endif /* CONFIG_PPC_BOOK3S_64 */ 1041 1042 #ifdef CONFIG_TRACEPOINTS 1043 #ifdef HAVE_JUMP_LABEL 1044 struct static_key hcall_tracepoint_key = STATIC_KEY_INIT; 1045 1046 int hcall_tracepoint_regfunc(void) 1047 { 1048 static_key_slow_inc(&hcall_tracepoint_key); 1049 return 0; 1050 } 1051 1052 void hcall_tracepoint_unregfunc(void) 1053 { 1054 static_key_slow_dec(&hcall_tracepoint_key); 1055 } 1056 #else 1057 /* 1058 * We optimise our hcall path by placing hcall_tracepoint_refcount 1059 * directly in the TOC so we can check if the hcall tracepoints are 1060 * enabled via a single load. 1061 */ 1062 1063 /* NB: reg/unreg are called while guarded with the tracepoints_mutex */ 1064 extern long hcall_tracepoint_refcount; 1065 1066 int hcall_tracepoint_regfunc(void) 1067 { 1068 hcall_tracepoint_refcount++; 1069 return 0; 1070 } 1071 1072 void hcall_tracepoint_unregfunc(void) 1073 { 1074 hcall_tracepoint_refcount--; 1075 } 1076 #endif 1077 1078 /* 1079 * Since the tracing code might execute hcalls we need to guard against 1080 * recursion. One example of this are spinlocks calling H_YIELD on 1081 * shared processor partitions. 1082 */ 1083 static DEFINE_PER_CPU(unsigned int, hcall_trace_depth); 1084 1085 1086 void __trace_hcall_entry(unsigned long opcode, unsigned long *args) 1087 { 1088 unsigned long flags; 1089 unsigned int *depth; 1090 1091 /* 1092 * We cannot call tracepoints inside RCU idle regions which 1093 * means we must not trace H_CEDE. 1094 */ 1095 if (opcode == H_CEDE) 1096 return; 1097 1098 local_irq_save(flags); 1099 1100 depth = this_cpu_ptr(&hcall_trace_depth); 1101 1102 if (*depth) 1103 goto out; 1104 1105 (*depth)++; 1106 preempt_disable(); 1107 trace_hcall_entry(opcode, args); 1108 (*depth)--; 1109 1110 out: 1111 local_irq_restore(flags); 1112 } 1113 1114 void __trace_hcall_exit(long opcode, long retval, unsigned long *retbuf) 1115 { 1116 unsigned long flags; 1117 unsigned int *depth; 1118 1119 if (opcode == H_CEDE) 1120 return; 1121 1122 local_irq_save(flags); 1123 1124 depth = this_cpu_ptr(&hcall_trace_depth); 1125 1126 if (*depth) 1127 goto out; 1128 1129 (*depth)++; 1130 trace_hcall_exit(opcode, retval, retbuf); 1131 preempt_enable(); 1132 (*depth)--; 1133 1134 out: 1135 local_irq_restore(flags); 1136 } 1137 #endif 1138 1139 /** 1140 * h_get_mpp 1141 * H_GET_MPP hcall returns info in 7 parms 1142 */ 1143 int h_get_mpp(struct hvcall_mpp_data *mpp_data) 1144 { 1145 int rc; 1146 unsigned long retbuf[PLPAR_HCALL9_BUFSIZE]; 1147 1148 rc = plpar_hcall9(H_GET_MPP, retbuf); 1149 1150 mpp_data->entitled_mem = retbuf[0]; 1151 mpp_data->mapped_mem = retbuf[1]; 1152 1153 mpp_data->group_num = (retbuf[2] >> 2 * 8) & 0xffff; 1154 mpp_data->pool_num = retbuf[2] & 0xffff; 1155 1156 mpp_data->mem_weight = (retbuf[3] >> 7 * 8) & 0xff; 1157 mpp_data->unallocated_mem_weight = (retbuf[3] >> 6 * 8) & 0xff; 1158 mpp_data->unallocated_entitlement = retbuf[3] & 0xffffffffffffUL; 1159 1160 mpp_data->pool_size = retbuf[4]; 1161 mpp_data->loan_request = retbuf[5]; 1162 mpp_data->backing_mem = retbuf[6]; 1163 1164 return rc; 1165 } 1166 EXPORT_SYMBOL(h_get_mpp); 1167 1168 int h_get_mpp_x(struct hvcall_mpp_x_data *mpp_x_data) 1169 { 1170 int rc; 1171 unsigned long retbuf[PLPAR_HCALL9_BUFSIZE] = { 0 }; 1172 1173 rc = plpar_hcall9(H_GET_MPP_X, retbuf); 1174 1175 mpp_x_data->coalesced_bytes = retbuf[0]; 1176 mpp_x_data->pool_coalesced_bytes = retbuf[1]; 1177 mpp_x_data->pool_purr_cycles = retbuf[2]; 1178 mpp_x_data->pool_spurr_cycles = retbuf[3]; 1179 1180 return rc; 1181 } 1182 1183 static unsigned long vsid_unscramble(unsigned long vsid, int ssize) 1184 { 1185 unsigned long protovsid; 1186 unsigned long va_bits = VA_BITS; 1187 unsigned long modinv, vsid_modulus; 1188 unsigned long max_mod_inv, tmp_modinv; 1189 1190 if (!mmu_has_feature(MMU_FTR_68_BIT_VA)) 1191 va_bits = 65; 1192 1193 if (ssize == MMU_SEGSIZE_256M) { 1194 modinv = VSID_MULINV_256M; 1195 vsid_modulus = ((1UL << (va_bits - SID_SHIFT)) - 1); 1196 } else { 1197 modinv = VSID_MULINV_1T; 1198 vsid_modulus = ((1UL << (va_bits - SID_SHIFT_1T)) - 1); 1199 } 1200 1201 /* 1202 * vsid outside our range. 1203 */ 1204 if (vsid >= vsid_modulus) 1205 return 0; 1206 1207 /* 1208 * If modinv is the modular multiplicate inverse of (x % vsid_modulus) 1209 * and vsid = (protovsid * x) % vsid_modulus, then we say: 1210 * protovsid = (vsid * modinv) % vsid_modulus 1211 */ 1212 1213 /* Check if (vsid * modinv) overflow (63 bits) */ 1214 max_mod_inv = 0x7fffffffffffffffull / vsid; 1215 if (modinv < max_mod_inv) 1216 return (vsid * modinv) % vsid_modulus; 1217 1218 tmp_modinv = modinv/max_mod_inv; 1219 modinv %= max_mod_inv; 1220 1221 protovsid = (((vsid * max_mod_inv) % vsid_modulus) * tmp_modinv) % vsid_modulus; 1222 protovsid = (protovsid + vsid * modinv) % vsid_modulus; 1223 1224 return protovsid; 1225 } 1226 1227 static int __init reserve_vrma_context_id(void) 1228 { 1229 unsigned long protovsid; 1230 1231 /* 1232 * Reserve context ids which map to reserved virtual addresses. For now 1233 * we only reserve the context id which maps to the VRMA VSID. We ignore 1234 * the addresses in "ibm,adjunct-virtual-addresses" because we don't 1235 * enable adjunct support via the "ibm,client-architecture-support" 1236 * interface. 1237 */ 1238 protovsid = vsid_unscramble(VRMA_VSID, MMU_SEGSIZE_1T); 1239 hash__reserve_context_id(protovsid >> ESID_BITS_1T); 1240 return 0; 1241 } 1242 machine_device_initcall(pseries, reserve_vrma_context_id); 1243 1244 #ifdef CONFIG_DEBUG_FS 1245 /* debugfs file interface for vpa data */ 1246 static ssize_t vpa_file_read(struct file *filp, char __user *buf, size_t len, 1247 loff_t *pos) 1248 { 1249 int cpu = (long)filp->private_data; 1250 struct lppaca *lppaca = &lppaca_of(cpu); 1251 1252 return simple_read_from_buffer(buf, len, pos, lppaca, 1253 sizeof(struct lppaca)); 1254 } 1255 1256 static const struct file_operations vpa_fops = { 1257 .open = simple_open, 1258 .read = vpa_file_read, 1259 .llseek = default_llseek, 1260 }; 1261 1262 static int __init vpa_debugfs_init(void) 1263 { 1264 char name[16]; 1265 long i; 1266 static struct dentry *vpa_dir; 1267 1268 if (!firmware_has_feature(FW_FEATURE_SPLPAR)) 1269 return 0; 1270 1271 vpa_dir = debugfs_create_dir("vpa", powerpc_debugfs_root); 1272 if (!vpa_dir) { 1273 pr_warn("%s: can't create vpa root dir\n", __func__); 1274 return -ENOMEM; 1275 } 1276 1277 /* set up the per-cpu vpa file*/ 1278 for_each_possible_cpu(i) { 1279 struct dentry *d; 1280 1281 sprintf(name, "cpu-%ld", i); 1282 1283 d = debugfs_create_file(name, 0400, vpa_dir, (void *)i, 1284 &vpa_fops); 1285 if (!d) { 1286 pr_warn("%s: can't create per-cpu vpa file\n", 1287 __func__); 1288 return -ENOMEM; 1289 } 1290 } 1291 1292 return 0; 1293 } 1294 machine_arch_initcall(pseries, vpa_debugfs_init); 1295 #endif /* CONFIG_DEBUG_FS */ 1296