1 /* 2 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 3 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved. 4 * 5 * Authors: 6 * Paul Mackerras <paulus@au1.ibm.com> 7 * Alexander Graf <agraf@suse.de> 8 * Kevin Wolf <mail@kevin-wolf.de> 9 * 10 * Description: KVM functions specific to running on Book 3S 11 * processors in hypervisor mode (specifically POWER7 and later). 12 * 13 * This file is derived from arch/powerpc/kvm/book3s.c, 14 * by Alexander Graf <agraf@suse.de>. 15 * 16 * This program is free software; you can redistribute it and/or modify 17 * it under the terms of the GNU General Public License, version 2, as 18 * published by the Free Software Foundation. 19 */ 20 21 #include <linux/kvm_host.h> 22 #include <linux/err.h> 23 #include <linux/slab.h> 24 #include <linux/preempt.h> 25 #include <linux/sched.h> 26 #include <linux/delay.h> 27 #include <linux/export.h> 28 #include <linux/fs.h> 29 #include <linux/anon_inodes.h> 30 #include <linux/cpu.h> 31 #include <linux/cpumask.h> 32 #include <linux/spinlock.h> 33 #include <linux/page-flags.h> 34 #include <linux/srcu.h> 35 #include <linux/miscdevice.h> 36 #include <linux/debugfs.h> 37 38 #include <asm/reg.h> 39 #include <asm/cputable.h> 40 #include <asm/cacheflush.h> 41 #include <asm/tlbflush.h> 42 #include <asm/uaccess.h> 43 #include <asm/io.h> 44 #include <asm/kvm_ppc.h> 45 #include <asm/kvm_book3s.h> 46 #include <asm/mmu_context.h> 47 #include <asm/lppaca.h> 48 #include <asm/processor.h> 49 #include <asm/cputhreads.h> 50 #include <asm/page.h> 51 #include <asm/hvcall.h> 52 #include <asm/switch_to.h> 53 #include <asm/smp.h> 54 #include <asm/dbell.h> 55 #include <asm/hmi.h> 56 #include <asm/pnv-pci.h> 57 #include <asm/mmu.h> 58 #include <asm/opal.h> 59 #include <asm/xics.h> 60 #include <linux/gfp.h> 61 #include <linux/vmalloc.h> 62 #include <linux/highmem.h> 63 #include <linux/hugetlb.h> 64 #include <linux/kvm_irqfd.h> 65 #include <linux/irqbypass.h> 66 #include <linux/module.h> 67 #include <linux/compiler.h> 68 #include <linux/of.h> 69 70 #include "book3s.h" 71 72 #define CREATE_TRACE_POINTS 73 #include "trace_hv.h" 74 75 /* #define EXIT_DEBUG */ 76 /* #define EXIT_DEBUG_SIMPLE */ 77 /* #define EXIT_DEBUG_INT */ 78 79 /* Used to indicate that a guest page fault needs to be handled */ 80 #define RESUME_PAGE_FAULT (RESUME_GUEST | RESUME_FLAG_ARCH1) 81 /* Used to indicate that a guest passthrough interrupt needs to be handled */ 82 #define RESUME_PASSTHROUGH (RESUME_GUEST | RESUME_FLAG_ARCH2) 83 84 /* Used as a "null" value for timebase values */ 85 #define TB_NIL (~(u64)0) 86 87 static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1); 88 89 static int dynamic_mt_modes = 6; 90 module_param(dynamic_mt_modes, int, S_IRUGO | S_IWUSR); 91 MODULE_PARM_DESC(dynamic_mt_modes, "Set of allowed dynamic micro-threading modes: 0 (= none), 2, 4, or 6 (= 2 or 4)"); 92 static int target_smt_mode; 93 module_param(target_smt_mode, int, S_IRUGO | S_IWUSR); 94 MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)"); 95 96 #ifdef CONFIG_KVM_XICS 97 static struct kernel_param_ops module_param_ops = { 98 .set = param_set_int, 99 .get = param_get_int, 100 }; 101 102 module_param_cb(kvm_irq_bypass, &module_param_ops, &kvm_irq_bypass, 103 S_IRUGO | S_IWUSR); 104 MODULE_PARM_DESC(kvm_irq_bypass, "Bypass passthrough interrupt optimization"); 105 106 module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect, 107 S_IRUGO | S_IWUSR); 108 MODULE_PARM_DESC(h_ipi_redirect, "Redirect H_IPI wakeup to a free host core"); 109 #endif 110 111 static void kvmppc_end_cede(struct kvm_vcpu *vcpu); 112 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu); 113 114 static inline struct kvm_vcpu *next_runnable_thread(struct kvmppc_vcore *vc, 115 int *ip) 116 { 117 int i = *ip; 118 struct kvm_vcpu *vcpu; 119 120 while (++i < MAX_SMT_THREADS) { 121 vcpu = READ_ONCE(vc->runnable_threads[i]); 122 if (vcpu) { 123 *ip = i; 124 return vcpu; 125 } 126 } 127 return NULL; 128 } 129 130 /* Used to traverse the list of runnable threads for a given vcore */ 131 #define for_each_runnable_thread(i, vcpu, vc) \ 132 for (i = -1; (vcpu = next_runnable_thread(vc, &i)); ) 133 134 static bool kvmppc_ipi_thread(int cpu) 135 { 136 unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER); 137 138 /* On POWER9 we can use msgsnd to IPI any cpu */ 139 if (cpu_has_feature(CPU_FTR_ARCH_300)) { 140 msg |= get_hard_smp_processor_id(cpu); 141 smp_mb(); 142 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg)); 143 return true; 144 } 145 146 /* On POWER8 for IPIs to threads in the same core, use msgsnd */ 147 if (cpu_has_feature(CPU_FTR_ARCH_207S)) { 148 preempt_disable(); 149 if (cpu_first_thread_sibling(cpu) == 150 cpu_first_thread_sibling(smp_processor_id())) { 151 msg |= cpu_thread_in_core(cpu); 152 smp_mb(); 153 __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg)); 154 preempt_enable(); 155 return true; 156 } 157 preempt_enable(); 158 } 159 160 #if defined(CONFIG_PPC_ICP_NATIVE) && defined(CONFIG_SMP) 161 if (cpu >= 0 && cpu < nr_cpu_ids) { 162 if (paca[cpu].kvm_hstate.xics_phys) { 163 xics_wake_cpu(cpu); 164 return true; 165 } 166 opal_int_set_mfrr(get_hard_smp_processor_id(cpu), IPI_PRIORITY); 167 return true; 168 } 169 #endif 170 171 return false; 172 } 173 174 static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) 175 { 176 int cpu; 177 struct swait_queue_head *wqp; 178 179 wqp = kvm_arch_vcpu_wq(vcpu); 180 if (swait_active(wqp)) { 181 swake_up(wqp); 182 ++vcpu->stat.halt_wakeup; 183 } 184 185 if (kvmppc_ipi_thread(vcpu->arch.thread_cpu)) 186 return; 187 188 /* CPU points to the first thread of the core */ 189 cpu = vcpu->cpu; 190 if (cpu >= 0 && cpu < nr_cpu_ids && cpu_online(cpu)) 191 smp_send_reschedule(cpu); 192 } 193 194 /* 195 * We use the vcpu_load/put functions to measure stolen time. 196 * Stolen time is counted as time when either the vcpu is able to 197 * run as part of a virtual core, but the task running the vcore 198 * is preempted or sleeping, or when the vcpu needs something done 199 * in the kernel by the task running the vcpu, but that task is 200 * preempted or sleeping. Those two things have to be counted 201 * separately, since one of the vcpu tasks will take on the job 202 * of running the core, and the other vcpu tasks in the vcore will 203 * sleep waiting for it to do that, but that sleep shouldn't count 204 * as stolen time. 205 * 206 * Hence we accumulate stolen time when the vcpu can run as part of 207 * a vcore using vc->stolen_tb, and the stolen time when the vcpu 208 * needs its task to do other things in the kernel (for example, 209 * service a page fault) in busy_stolen. We don't accumulate 210 * stolen time for a vcore when it is inactive, or for a vcpu 211 * when it is in state RUNNING or NOTREADY. NOTREADY is a bit of 212 * a misnomer; it means that the vcpu task is not executing in 213 * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in 214 * the kernel. We don't have any way of dividing up that time 215 * between time that the vcpu is genuinely stopped, time that 216 * the task is actively working on behalf of the vcpu, and time 217 * that the task is preempted, so we don't count any of it as 218 * stolen. 219 * 220 * Updates to busy_stolen are protected by arch.tbacct_lock; 221 * updates to vc->stolen_tb are protected by the vcore->stoltb_lock 222 * lock. The stolen times are measured in units of timebase ticks. 223 * (Note that the != TB_NIL checks below are purely defensive; 224 * they should never fail.) 225 */ 226 227 static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc) 228 { 229 unsigned long flags; 230 231 spin_lock_irqsave(&vc->stoltb_lock, flags); 232 vc->preempt_tb = mftb(); 233 spin_unlock_irqrestore(&vc->stoltb_lock, flags); 234 } 235 236 static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc) 237 { 238 unsigned long flags; 239 240 spin_lock_irqsave(&vc->stoltb_lock, flags); 241 if (vc->preempt_tb != TB_NIL) { 242 vc->stolen_tb += mftb() - vc->preempt_tb; 243 vc->preempt_tb = TB_NIL; 244 } 245 spin_unlock_irqrestore(&vc->stoltb_lock, flags); 246 } 247 248 static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu) 249 { 250 struct kvmppc_vcore *vc = vcpu->arch.vcore; 251 unsigned long flags; 252 253 /* 254 * We can test vc->runner without taking the vcore lock, 255 * because only this task ever sets vc->runner to this 256 * vcpu, and once it is set to this vcpu, only this task 257 * ever sets it to NULL. 258 */ 259 if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING) 260 kvmppc_core_end_stolen(vc); 261 262 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); 263 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST && 264 vcpu->arch.busy_preempt != TB_NIL) { 265 vcpu->arch.busy_stolen += mftb() - vcpu->arch.busy_preempt; 266 vcpu->arch.busy_preempt = TB_NIL; 267 } 268 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags); 269 } 270 271 static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu) 272 { 273 struct kvmppc_vcore *vc = vcpu->arch.vcore; 274 unsigned long flags; 275 276 if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING) 277 kvmppc_core_start_stolen(vc); 278 279 spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); 280 if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST) 281 vcpu->arch.busy_preempt = mftb(); 282 spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags); 283 } 284 285 static void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr) 286 { 287 /* 288 * Check for illegal transactional state bit combination 289 * and if we find it, force the TS field to a safe state. 290 */ 291 if ((msr & MSR_TS_MASK) == MSR_TS_MASK) 292 msr &= ~MSR_TS_MASK; 293 vcpu->arch.shregs.msr = msr; 294 kvmppc_end_cede(vcpu); 295 } 296 297 static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr) 298 { 299 vcpu->arch.pvr = pvr; 300 } 301 302 /* Dummy value used in computing PCR value below */ 303 #define PCR_ARCH_300 (PCR_ARCH_207 << 1) 304 305 static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) 306 { 307 unsigned long host_pcr_bit = 0, guest_pcr_bit = 0; 308 struct kvmppc_vcore *vc = vcpu->arch.vcore; 309 310 /* We can (emulate) our own architecture version and anything older */ 311 if (cpu_has_feature(CPU_FTR_ARCH_300)) 312 host_pcr_bit = PCR_ARCH_300; 313 else if (cpu_has_feature(CPU_FTR_ARCH_207S)) 314 host_pcr_bit = PCR_ARCH_207; 315 else if (cpu_has_feature(CPU_FTR_ARCH_206)) 316 host_pcr_bit = PCR_ARCH_206; 317 else 318 host_pcr_bit = PCR_ARCH_205; 319 320 /* Determine lowest PCR bit needed to run guest in given PVR level */ 321 guest_pcr_bit = host_pcr_bit; 322 if (arch_compat) { 323 switch (arch_compat) { 324 case PVR_ARCH_205: 325 guest_pcr_bit = PCR_ARCH_205; 326 break; 327 case PVR_ARCH_206: 328 case PVR_ARCH_206p: 329 guest_pcr_bit = PCR_ARCH_206; 330 break; 331 case PVR_ARCH_207: 332 guest_pcr_bit = PCR_ARCH_207; 333 break; 334 case PVR_ARCH_300: 335 guest_pcr_bit = PCR_ARCH_300; 336 break; 337 default: 338 return -EINVAL; 339 } 340 } 341 342 /* Check requested PCR bits don't exceed our capabilities */ 343 if (guest_pcr_bit > host_pcr_bit) 344 return -EINVAL; 345 346 spin_lock(&vc->lock); 347 vc->arch_compat = arch_compat; 348 /* Set all PCR bits for which guest_pcr_bit <= bit < host_pcr_bit */ 349 vc->pcr = host_pcr_bit - guest_pcr_bit; 350 spin_unlock(&vc->lock); 351 352 return 0; 353 } 354 355 static void kvmppc_dump_regs(struct kvm_vcpu *vcpu) 356 { 357 int r; 358 359 pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id); 360 pr_err("pc = %.16lx msr = %.16llx trap = %x\n", 361 vcpu->arch.pc, vcpu->arch.shregs.msr, vcpu->arch.trap); 362 for (r = 0; r < 16; ++r) 363 pr_err("r%2d = %.16lx r%d = %.16lx\n", 364 r, kvmppc_get_gpr(vcpu, r), 365 r+16, kvmppc_get_gpr(vcpu, r+16)); 366 pr_err("ctr = %.16lx lr = %.16lx\n", 367 vcpu->arch.ctr, vcpu->arch.lr); 368 pr_err("srr0 = %.16llx srr1 = %.16llx\n", 369 vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1); 370 pr_err("sprg0 = %.16llx sprg1 = %.16llx\n", 371 vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1); 372 pr_err("sprg2 = %.16llx sprg3 = %.16llx\n", 373 vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3); 374 pr_err("cr = %.8x xer = %.16lx dsisr = %.8x\n", 375 vcpu->arch.cr, vcpu->arch.xer, vcpu->arch.shregs.dsisr); 376 pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar); 377 pr_err("fault dar = %.16lx dsisr = %.8x\n", 378 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr); 379 pr_err("SLB (%d entries):\n", vcpu->arch.slb_max); 380 for (r = 0; r < vcpu->arch.slb_max; ++r) 381 pr_err(" ESID = %.16llx VSID = %.16llx\n", 382 vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv); 383 pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n", 384 vcpu->arch.vcore->lpcr, vcpu->kvm->arch.sdr1, 385 vcpu->arch.last_inst); 386 } 387 388 static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id) 389 { 390 struct kvm_vcpu *ret; 391 392 mutex_lock(&kvm->lock); 393 ret = kvm_get_vcpu_by_id(kvm, id); 394 mutex_unlock(&kvm->lock); 395 return ret; 396 } 397 398 static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa) 399 { 400 vpa->__old_status |= LPPACA_OLD_SHARED_PROC; 401 vpa->yield_count = cpu_to_be32(1); 402 } 403 404 static int set_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *v, 405 unsigned long addr, unsigned long len) 406 { 407 /* check address is cacheline aligned */ 408 if (addr & (L1_CACHE_BYTES - 1)) 409 return -EINVAL; 410 spin_lock(&vcpu->arch.vpa_update_lock); 411 if (v->next_gpa != addr || v->len != len) { 412 v->next_gpa = addr; 413 v->len = addr ? len : 0; 414 v->update_pending = 1; 415 } 416 spin_unlock(&vcpu->arch.vpa_update_lock); 417 return 0; 418 } 419 420 /* Length for a per-processor buffer is passed in at offset 4 in the buffer */ 421 struct reg_vpa { 422 u32 dummy; 423 union { 424 __be16 hword; 425 __be32 word; 426 } length; 427 }; 428 429 static int vpa_is_registered(struct kvmppc_vpa *vpap) 430 { 431 if (vpap->update_pending) 432 return vpap->next_gpa != 0; 433 return vpap->pinned_addr != NULL; 434 } 435 436 static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu, 437 unsigned long flags, 438 unsigned long vcpuid, unsigned long vpa) 439 { 440 struct kvm *kvm = vcpu->kvm; 441 unsigned long len, nb; 442 void *va; 443 struct kvm_vcpu *tvcpu; 444 int err; 445 int subfunc; 446 struct kvmppc_vpa *vpap; 447 448 tvcpu = kvmppc_find_vcpu(kvm, vcpuid); 449 if (!tvcpu) 450 return H_PARAMETER; 451 452 subfunc = (flags >> H_VPA_FUNC_SHIFT) & H_VPA_FUNC_MASK; 453 if (subfunc == H_VPA_REG_VPA || subfunc == H_VPA_REG_DTL || 454 subfunc == H_VPA_REG_SLB) { 455 /* Registering new area - address must be cache-line aligned */ 456 if ((vpa & (L1_CACHE_BYTES - 1)) || !vpa) 457 return H_PARAMETER; 458 459 /* convert logical addr to kernel addr and read length */ 460 va = kvmppc_pin_guest_page(kvm, vpa, &nb); 461 if (va == NULL) 462 return H_PARAMETER; 463 if (subfunc == H_VPA_REG_VPA) 464 len = be16_to_cpu(((struct reg_vpa *)va)->length.hword); 465 else 466 len = be32_to_cpu(((struct reg_vpa *)va)->length.word); 467 kvmppc_unpin_guest_page(kvm, va, vpa, false); 468 469 /* Check length */ 470 if (len > nb || len < sizeof(struct reg_vpa)) 471 return H_PARAMETER; 472 } else { 473 vpa = 0; 474 len = 0; 475 } 476 477 err = H_PARAMETER; 478 vpap = NULL; 479 spin_lock(&tvcpu->arch.vpa_update_lock); 480 481 switch (subfunc) { 482 case H_VPA_REG_VPA: /* register VPA */ 483 if (len < sizeof(struct lppaca)) 484 break; 485 vpap = &tvcpu->arch.vpa; 486 err = 0; 487 break; 488 489 case H_VPA_REG_DTL: /* register DTL */ 490 if (len < sizeof(struct dtl_entry)) 491 break; 492 len -= len % sizeof(struct dtl_entry); 493 494 /* Check that they have previously registered a VPA */ 495 err = H_RESOURCE; 496 if (!vpa_is_registered(&tvcpu->arch.vpa)) 497 break; 498 499 vpap = &tvcpu->arch.dtl; 500 err = 0; 501 break; 502 503 case H_VPA_REG_SLB: /* register SLB shadow buffer */ 504 /* Check that they have previously registered a VPA */ 505 err = H_RESOURCE; 506 if (!vpa_is_registered(&tvcpu->arch.vpa)) 507 break; 508 509 vpap = &tvcpu->arch.slb_shadow; 510 err = 0; 511 break; 512 513 case H_VPA_DEREG_VPA: /* deregister VPA */ 514 /* Check they don't still have a DTL or SLB buf registered */ 515 err = H_RESOURCE; 516 if (vpa_is_registered(&tvcpu->arch.dtl) || 517 vpa_is_registered(&tvcpu->arch.slb_shadow)) 518 break; 519 520 vpap = &tvcpu->arch.vpa; 521 err = 0; 522 break; 523 524 case H_VPA_DEREG_DTL: /* deregister DTL */ 525 vpap = &tvcpu->arch.dtl; 526 err = 0; 527 break; 528 529 case H_VPA_DEREG_SLB: /* deregister SLB shadow buffer */ 530 vpap = &tvcpu->arch.slb_shadow; 531 err = 0; 532 break; 533 } 534 535 if (vpap) { 536 vpap->next_gpa = vpa; 537 vpap->len = len; 538 vpap->update_pending = 1; 539 } 540 541 spin_unlock(&tvcpu->arch.vpa_update_lock); 542 543 return err; 544 } 545 546 static void kvmppc_update_vpa(struct kvm_vcpu *vcpu, struct kvmppc_vpa *vpap) 547 { 548 struct kvm *kvm = vcpu->kvm; 549 void *va; 550 unsigned long nb; 551 unsigned long gpa; 552 553 /* 554 * We need to pin the page pointed to by vpap->next_gpa, 555 * but we can't call kvmppc_pin_guest_page under the lock 556 * as it does get_user_pages() and down_read(). So we 557 * have to drop the lock, pin the page, then get the lock 558 * again and check that a new area didn't get registered 559 * in the meantime. 560 */ 561 for (;;) { 562 gpa = vpap->next_gpa; 563 spin_unlock(&vcpu->arch.vpa_update_lock); 564 va = NULL; 565 nb = 0; 566 if (gpa) 567 va = kvmppc_pin_guest_page(kvm, gpa, &nb); 568 spin_lock(&vcpu->arch.vpa_update_lock); 569 if (gpa == vpap->next_gpa) 570 break; 571 /* sigh... unpin that one and try again */ 572 if (va) 573 kvmppc_unpin_guest_page(kvm, va, gpa, false); 574 } 575 576 vpap->update_pending = 0; 577 if (va && nb < vpap->len) { 578 /* 579 * If it's now too short, it must be that userspace 580 * has changed the mappings underlying guest memory, 581 * so unregister the region. 582 */ 583 kvmppc_unpin_guest_page(kvm, va, gpa, false); 584 va = NULL; 585 } 586 if (vpap->pinned_addr) 587 kvmppc_unpin_guest_page(kvm, vpap->pinned_addr, vpap->gpa, 588 vpap->dirty); 589 vpap->gpa = gpa; 590 vpap->pinned_addr = va; 591 vpap->dirty = false; 592 if (va) 593 vpap->pinned_end = va + vpap->len; 594 } 595 596 static void kvmppc_update_vpas(struct kvm_vcpu *vcpu) 597 { 598 if (!(vcpu->arch.vpa.update_pending || 599 vcpu->arch.slb_shadow.update_pending || 600 vcpu->arch.dtl.update_pending)) 601 return; 602 603 spin_lock(&vcpu->arch.vpa_update_lock); 604 if (vcpu->arch.vpa.update_pending) { 605 kvmppc_update_vpa(vcpu, &vcpu->arch.vpa); 606 if (vcpu->arch.vpa.pinned_addr) 607 init_vpa(vcpu, vcpu->arch.vpa.pinned_addr); 608 } 609 if (vcpu->arch.dtl.update_pending) { 610 kvmppc_update_vpa(vcpu, &vcpu->arch.dtl); 611 vcpu->arch.dtl_ptr = vcpu->arch.dtl.pinned_addr; 612 vcpu->arch.dtl_index = 0; 613 } 614 if (vcpu->arch.slb_shadow.update_pending) 615 kvmppc_update_vpa(vcpu, &vcpu->arch.slb_shadow); 616 spin_unlock(&vcpu->arch.vpa_update_lock); 617 } 618 619 /* 620 * Return the accumulated stolen time for the vcore up until `now'. 621 * The caller should hold the vcore lock. 622 */ 623 static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now) 624 { 625 u64 p; 626 unsigned long flags; 627 628 spin_lock_irqsave(&vc->stoltb_lock, flags); 629 p = vc->stolen_tb; 630 if (vc->vcore_state != VCORE_INACTIVE && 631 vc->preempt_tb != TB_NIL) 632 p += now - vc->preempt_tb; 633 spin_unlock_irqrestore(&vc->stoltb_lock, flags); 634 return p; 635 } 636 637 static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, 638 struct kvmppc_vcore *vc) 639 { 640 struct dtl_entry *dt; 641 struct lppaca *vpa; 642 unsigned long stolen; 643 unsigned long core_stolen; 644 u64 now; 645 646 dt = vcpu->arch.dtl_ptr; 647 vpa = vcpu->arch.vpa.pinned_addr; 648 now = mftb(); 649 core_stolen = vcore_stolen_time(vc, now); 650 stolen = core_stolen - vcpu->arch.stolen_logged; 651 vcpu->arch.stolen_logged = core_stolen; 652 spin_lock_irq(&vcpu->arch.tbacct_lock); 653 stolen += vcpu->arch.busy_stolen; 654 vcpu->arch.busy_stolen = 0; 655 spin_unlock_irq(&vcpu->arch.tbacct_lock); 656 if (!dt || !vpa) 657 return; 658 memset(dt, 0, sizeof(struct dtl_entry)); 659 dt->dispatch_reason = 7; 660 dt->processor_id = cpu_to_be16(vc->pcpu + vcpu->arch.ptid); 661 dt->timebase = cpu_to_be64(now + vc->tb_offset); 662 dt->enqueue_to_dispatch_time = cpu_to_be32(stolen); 663 dt->srr0 = cpu_to_be64(kvmppc_get_pc(vcpu)); 664 dt->srr1 = cpu_to_be64(vcpu->arch.shregs.msr); 665 ++dt; 666 if (dt == vcpu->arch.dtl.pinned_end) 667 dt = vcpu->arch.dtl.pinned_addr; 668 vcpu->arch.dtl_ptr = dt; 669 /* order writing *dt vs. writing vpa->dtl_idx */ 670 smp_wmb(); 671 vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index); 672 vcpu->arch.dtl.dirty = true; 673 } 674 675 static bool kvmppc_power8_compatible(struct kvm_vcpu *vcpu) 676 { 677 if (vcpu->arch.vcore->arch_compat >= PVR_ARCH_207) 678 return true; 679 if ((!vcpu->arch.vcore->arch_compat) && 680 cpu_has_feature(CPU_FTR_ARCH_207S)) 681 return true; 682 return false; 683 } 684 685 static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags, 686 unsigned long resource, unsigned long value1, 687 unsigned long value2) 688 { 689 switch (resource) { 690 case H_SET_MODE_RESOURCE_SET_CIABR: 691 if (!kvmppc_power8_compatible(vcpu)) 692 return H_P2; 693 if (value2) 694 return H_P4; 695 if (mflags) 696 return H_UNSUPPORTED_FLAG_START; 697 /* Guests can't breakpoint the hypervisor */ 698 if ((value1 & CIABR_PRIV) == CIABR_PRIV_HYPER) 699 return H_P3; 700 vcpu->arch.ciabr = value1; 701 return H_SUCCESS; 702 case H_SET_MODE_RESOURCE_SET_DAWR: 703 if (!kvmppc_power8_compatible(vcpu)) 704 return H_P2; 705 if (mflags) 706 return H_UNSUPPORTED_FLAG_START; 707 if (value2 & DABRX_HYP) 708 return H_P4; 709 vcpu->arch.dawr = value1; 710 vcpu->arch.dawrx = value2; 711 return H_SUCCESS; 712 default: 713 return H_TOO_HARD; 714 } 715 } 716 717 static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target) 718 { 719 struct kvmppc_vcore *vcore = target->arch.vcore; 720 721 /* 722 * We expect to have been called by the real mode handler 723 * (kvmppc_rm_h_confer()) which would have directly returned 724 * H_SUCCESS if the source vcore wasn't idle (e.g. if it may 725 * have useful work to do and should not confer) so we don't 726 * recheck that here. 727 */ 728 729 spin_lock(&vcore->lock); 730 if (target->arch.state == KVMPPC_VCPU_RUNNABLE && 731 vcore->vcore_state != VCORE_INACTIVE && 732 vcore->runner) 733 target = vcore->runner; 734 spin_unlock(&vcore->lock); 735 736 return kvm_vcpu_yield_to(target); 737 } 738 739 static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu) 740 { 741 int yield_count = 0; 742 struct lppaca *lppaca; 743 744 spin_lock(&vcpu->arch.vpa_update_lock); 745 lppaca = (struct lppaca *)vcpu->arch.vpa.pinned_addr; 746 if (lppaca) 747 yield_count = be32_to_cpu(lppaca->yield_count); 748 spin_unlock(&vcpu->arch.vpa_update_lock); 749 return yield_count; 750 } 751 752 int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) 753 { 754 unsigned long req = kvmppc_get_gpr(vcpu, 3); 755 unsigned long target, ret = H_SUCCESS; 756 int yield_count; 757 struct kvm_vcpu *tvcpu; 758 int idx, rc; 759 760 if (req <= MAX_HCALL_OPCODE && 761 !test_bit(req/4, vcpu->kvm->arch.enabled_hcalls)) 762 return RESUME_HOST; 763 764 switch (req) { 765 case H_CEDE: 766 break; 767 case H_PROD: 768 target = kvmppc_get_gpr(vcpu, 4); 769 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target); 770 if (!tvcpu) { 771 ret = H_PARAMETER; 772 break; 773 } 774 tvcpu->arch.prodded = 1; 775 smp_mb(); 776 if (vcpu->arch.ceded) { 777 if (swait_active(&vcpu->wq)) { 778 swake_up(&vcpu->wq); 779 vcpu->stat.halt_wakeup++; 780 } 781 } 782 break; 783 case H_CONFER: 784 target = kvmppc_get_gpr(vcpu, 4); 785 if (target == -1) 786 break; 787 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target); 788 if (!tvcpu) { 789 ret = H_PARAMETER; 790 break; 791 } 792 yield_count = kvmppc_get_gpr(vcpu, 5); 793 if (kvmppc_get_yield_count(tvcpu) != yield_count) 794 break; 795 kvm_arch_vcpu_yield_to(tvcpu); 796 break; 797 case H_REGISTER_VPA: 798 ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4), 799 kvmppc_get_gpr(vcpu, 5), 800 kvmppc_get_gpr(vcpu, 6)); 801 break; 802 case H_RTAS: 803 if (list_empty(&vcpu->kvm->arch.rtas_tokens)) 804 return RESUME_HOST; 805 806 idx = srcu_read_lock(&vcpu->kvm->srcu); 807 rc = kvmppc_rtas_hcall(vcpu); 808 srcu_read_unlock(&vcpu->kvm->srcu, idx); 809 810 if (rc == -ENOENT) 811 return RESUME_HOST; 812 else if (rc == 0) 813 break; 814 815 /* Send the error out to userspace via KVM_RUN */ 816 return rc; 817 case H_LOGICAL_CI_LOAD: 818 ret = kvmppc_h_logical_ci_load(vcpu); 819 if (ret == H_TOO_HARD) 820 return RESUME_HOST; 821 break; 822 case H_LOGICAL_CI_STORE: 823 ret = kvmppc_h_logical_ci_store(vcpu); 824 if (ret == H_TOO_HARD) 825 return RESUME_HOST; 826 break; 827 case H_SET_MODE: 828 ret = kvmppc_h_set_mode(vcpu, kvmppc_get_gpr(vcpu, 4), 829 kvmppc_get_gpr(vcpu, 5), 830 kvmppc_get_gpr(vcpu, 6), 831 kvmppc_get_gpr(vcpu, 7)); 832 if (ret == H_TOO_HARD) 833 return RESUME_HOST; 834 break; 835 case H_XIRR: 836 case H_CPPR: 837 case H_EOI: 838 case H_IPI: 839 case H_IPOLL: 840 case H_XIRR_X: 841 if (kvmppc_xics_enabled(vcpu)) { 842 ret = kvmppc_xics_hcall(vcpu, req); 843 break; 844 } 845 return RESUME_HOST; 846 case H_PUT_TCE: 847 ret = kvmppc_h_put_tce(vcpu, kvmppc_get_gpr(vcpu, 4), 848 kvmppc_get_gpr(vcpu, 5), 849 kvmppc_get_gpr(vcpu, 6)); 850 if (ret == H_TOO_HARD) 851 return RESUME_HOST; 852 break; 853 case H_PUT_TCE_INDIRECT: 854 ret = kvmppc_h_put_tce_indirect(vcpu, kvmppc_get_gpr(vcpu, 4), 855 kvmppc_get_gpr(vcpu, 5), 856 kvmppc_get_gpr(vcpu, 6), 857 kvmppc_get_gpr(vcpu, 7)); 858 if (ret == H_TOO_HARD) 859 return RESUME_HOST; 860 break; 861 case H_STUFF_TCE: 862 ret = kvmppc_h_stuff_tce(vcpu, kvmppc_get_gpr(vcpu, 4), 863 kvmppc_get_gpr(vcpu, 5), 864 kvmppc_get_gpr(vcpu, 6), 865 kvmppc_get_gpr(vcpu, 7)); 866 if (ret == H_TOO_HARD) 867 return RESUME_HOST; 868 break; 869 default: 870 return RESUME_HOST; 871 } 872 kvmppc_set_gpr(vcpu, 3, ret); 873 vcpu->arch.hcall_needed = 0; 874 return RESUME_GUEST; 875 } 876 877 static int kvmppc_hcall_impl_hv(unsigned long cmd) 878 { 879 switch (cmd) { 880 case H_CEDE: 881 case H_PROD: 882 case H_CONFER: 883 case H_REGISTER_VPA: 884 case H_SET_MODE: 885 case H_LOGICAL_CI_LOAD: 886 case H_LOGICAL_CI_STORE: 887 #ifdef CONFIG_KVM_XICS 888 case H_XIRR: 889 case H_CPPR: 890 case H_EOI: 891 case H_IPI: 892 case H_IPOLL: 893 case H_XIRR_X: 894 #endif 895 return 1; 896 } 897 898 /* See if it's in the real-mode table */ 899 return kvmppc_hcall_impl_hv_realmode(cmd); 900 } 901 902 static int kvmppc_emulate_debug_inst(struct kvm_run *run, 903 struct kvm_vcpu *vcpu) 904 { 905 u32 last_inst; 906 907 if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) != 908 EMULATE_DONE) { 909 /* 910 * Fetch failed, so return to guest and 911 * try executing it again. 912 */ 913 return RESUME_GUEST; 914 } 915 916 if (last_inst == KVMPPC_INST_SW_BREAKPOINT) { 917 run->exit_reason = KVM_EXIT_DEBUG; 918 run->debug.arch.address = kvmppc_get_pc(vcpu); 919 return RESUME_HOST; 920 } else { 921 kvmppc_core_queue_program(vcpu, SRR1_PROGILL); 922 return RESUME_GUEST; 923 } 924 } 925 926 static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu, 927 struct task_struct *tsk) 928 { 929 int r = RESUME_HOST; 930 931 vcpu->stat.sum_exits++; 932 933 /* 934 * This can happen if an interrupt occurs in the last stages 935 * of guest entry or the first stages of guest exit (i.e. after 936 * setting paca->kvm_hstate.in_guest to KVM_GUEST_MODE_GUEST_HV 937 * and before setting it to KVM_GUEST_MODE_HOST_HV). 938 * That can happen due to a bug, or due to a machine check 939 * occurring at just the wrong time. 940 */ 941 if (vcpu->arch.shregs.msr & MSR_HV) { 942 printk(KERN_EMERG "KVM trap in HV mode!\n"); 943 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n", 944 vcpu->arch.trap, kvmppc_get_pc(vcpu), 945 vcpu->arch.shregs.msr); 946 kvmppc_dump_regs(vcpu); 947 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 948 run->hw.hardware_exit_reason = vcpu->arch.trap; 949 return RESUME_HOST; 950 } 951 run->exit_reason = KVM_EXIT_UNKNOWN; 952 run->ready_for_interrupt_injection = 1; 953 switch (vcpu->arch.trap) { 954 /* We're good on these - the host merely wanted to get our attention */ 955 case BOOK3S_INTERRUPT_HV_DECREMENTER: 956 vcpu->stat.dec_exits++; 957 r = RESUME_GUEST; 958 break; 959 case BOOK3S_INTERRUPT_EXTERNAL: 960 case BOOK3S_INTERRUPT_H_DOORBELL: 961 case BOOK3S_INTERRUPT_H_VIRT: 962 vcpu->stat.ext_intr_exits++; 963 r = RESUME_GUEST; 964 break; 965 /* HMI is hypervisor interrupt and host has handled it. Resume guest.*/ 966 case BOOK3S_INTERRUPT_HMI: 967 case BOOK3S_INTERRUPT_PERFMON: 968 r = RESUME_GUEST; 969 break; 970 case BOOK3S_INTERRUPT_MACHINE_CHECK: 971 /* 972 * Deliver a machine check interrupt to the guest. 973 * We have to do this, even if the host has handled the 974 * machine check, because machine checks use SRR0/1 and 975 * the interrupt might have trashed guest state in them. 976 */ 977 kvmppc_book3s_queue_irqprio(vcpu, 978 BOOK3S_INTERRUPT_MACHINE_CHECK); 979 r = RESUME_GUEST; 980 break; 981 case BOOK3S_INTERRUPT_PROGRAM: 982 { 983 ulong flags; 984 /* 985 * Normally program interrupts are delivered directly 986 * to the guest by the hardware, but we can get here 987 * as a result of a hypervisor emulation interrupt 988 * (e40) getting turned into a 700 by BML RTAS. 989 */ 990 flags = vcpu->arch.shregs.msr & 0x1f0000ull; 991 kvmppc_core_queue_program(vcpu, flags); 992 r = RESUME_GUEST; 993 break; 994 } 995 case BOOK3S_INTERRUPT_SYSCALL: 996 { 997 /* hcall - punt to userspace */ 998 int i; 999 1000 /* hypercall with MSR_PR has already been handled in rmode, 1001 * and never reaches here. 1002 */ 1003 1004 run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3); 1005 for (i = 0; i < 9; ++i) 1006 run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i); 1007 run->exit_reason = KVM_EXIT_PAPR_HCALL; 1008 vcpu->arch.hcall_needed = 1; 1009 r = RESUME_HOST; 1010 break; 1011 } 1012 /* 1013 * We get these next two if the guest accesses a page which it thinks 1014 * it has mapped but which is not actually present, either because 1015 * it is for an emulated I/O device or because the corresonding 1016 * host page has been paged out. Any other HDSI/HISI interrupts 1017 * have been handled already. 1018 */ 1019 case BOOK3S_INTERRUPT_H_DATA_STORAGE: 1020 r = RESUME_PAGE_FAULT; 1021 break; 1022 case BOOK3S_INTERRUPT_H_INST_STORAGE: 1023 vcpu->arch.fault_dar = kvmppc_get_pc(vcpu); 1024 vcpu->arch.fault_dsisr = 0; 1025 r = RESUME_PAGE_FAULT; 1026 break; 1027 /* 1028 * This occurs if the guest executes an illegal instruction. 1029 * If the guest debug is disabled, generate a program interrupt 1030 * to the guest. If guest debug is enabled, we need to check 1031 * whether the instruction is a software breakpoint instruction. 1032 * Accordingly return to Guest or Host. 1033 */ 1034 case BOOK3S_INTERRUPT_H_EMUL_ASSIST: 1035 if (vcpu->arch.emul_inst != KVM_INST_FETCH_FAILED) 1036 vcpu->arch.last_inst = kvmppc_need_byteswap(vcpu) ? 1037 swab32(vcpu->arch.emul_inst) : 1038 vcpu->arch.emul_inst; 1039 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) { 1040 r = kvmppc_emulate_debug_inst(run, vcpu); 1041 } else { 1042 kvmppc_core_queue_program(vcpu, SRR1_PROGILL); 1043 r = RESUME_GUEST; 1044 } 1045 break; 1046 /* 1047 * This occurs if the guest (kernel or userspace), does something that 1048 * is prohibited by HFSCR. We just generate a program interrupt to 1049 * the guest. 1050 */ 1051 case BOOK3S_INTERRUPT_H_FAC_UNAVAIL: 1052 kvmppc_core_queue_program(vcpu, SRR1_PROGILL); 1053 r = RESUME_GUEST; 1054 break; 1055 case BOOK3S_INTERRUPT_HV_RM_HARD: 1056 r = RESUME_PASSTHROUGH; 1057 break; 1058 default: 1059 kvmppc_dump_regs(vcpu); 1060 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n", 1061 vcpu->arch.trap, kvmppc_get_pc(vcpu), 1062 vcpu->arch.shregs.msr); 1063 run->hw.hardware_exit_reason = vcpu->arch.trap; 1064 r = RESUME_HOST; 1065 break; 1066 } 1067 1068 return r; 1069 } 1070 1071 static int kvm_arch_vcpu_ioctl_get_sregs_hv(struct kvm_vcpu *vcpu, 1072 struct kvm_sregs *sregs) 1073 { 1074 int i; 1075 1076 memset(sregs, 0, sizeof(struct kvm_sregs)); 1077 sregs->pvr = vcpu->arch.pvr; 1078 for (i = 0; i < vcpu->arch.slb_max; i++) { 1079 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige; 1080 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv; 1081 } 1082 1083 return 0; 1084 } 1085 1086 static int kvm_arch_vcpu_ioctl_set_sregs_hv(struct kvm_vcpu *vcpu, 1087 struct kvm_sregs *sregs) 1088 { 1089 int i, j; 1090 1091 /* Only accept the same PVR as the host's, since we can't spoof it */ 1092 if (sregs->pvr != vcpu->arch.pvr) 1093 return -EINVAL; 1094 1095 j = 0; 1096 for (i = 0; i < vcpu->arch.slb_nr; i++) { 1097 if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) { 1098 vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe; 1099 vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv; 1100 ++j; 1101 } 1102 } 1103 vcpu->arch.slb_max = j; 1104 1105 return 0; 1106 } 1107 1108 static void kvmppc_set_lpcr(struct kvm_vcpu *vcpu, u64 new_lpcr, 1109 bool preserve_top32) 1110 { 1111 struct kvm *kvm = vcpu->kvm; 1112 struct kvmppc_vcore *vc = vcpu->arch.vcore; 1113 u64 mask; 1114 1115 mutex_lock(&kvm->lock); 1116 spin_lock(&vc->lock); 1117 /* 1118 * If ILE (interrupt little-endian) has changed, update the 1119 * MSR_LE bit in the intr_msr for each vcpu in this vcore. 1120 */ 1121 if ((new_lpcr & LPCR_ILE) != (vc->lpcr & LPCR_ILE)) { 1122 struct kvm_vcpu *vcpu; 1123 int i; 1124 1125 kvm_for_each_vcpu(i, vcpu, kvm) { 1126 if (vcpu->arch.vcore != vc) 1127 continue; 1128 if (new_lpcr & LPCR_ILE) 1129 vcpu->arch.intr_msr |= MSR_LE; 1130 else 1131 vcpu->arch.intr_msr &= ~MSR_LE; 1132 } 1133 } 1134 1135 /* 1136 * Userspace can only modify DPFD (default prefetch depth), 1137 * ILE (interrupt little-endian) and TC (translation control). 1138 * On POWER8 userspace can also modify AIL (alt. interrupt loc.) 1139 */ 1140 mask = LPCR_DPFD | LPCR_ILE | LPCR_TC; 1141 if (cpu_has_feature(CPU_FTR_ARCH_207S)) 1142 mask |= LPCR_AIL; 1143 1144 /* Broken 32-bit version of LPCR must not clear top bits */ 1145 if (preserve_top32) 1146 mask &= 0xFFFFFFFF; 1147 vc->lpcr = (vc->lpcr & ~mask) | (new_lpcr & mask); 1148 spin_unlock(&vc->lock); 1149 mutex_unlock(&kvm->lock); 1150 } 1151 1152 static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, 1153 union kvmppc_one_reg *val) 1154 { 1155 int r = 0; 1156 long int i; 1157 1158 switch (id) { 1159 case KVM_REG_PPC_DEBUG_INST: 1160 *val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT); 1161 break; 1162 case KVM_REG_PPC_HIOR: 1163 *val = get_reg_val(id, 0); 1164 break; 1165 case KVM_REG_PPC_DABR: 1166 *val = get_reg_val(id, vcpu->arch.dabr); 1167 break; 1168 case KVM_REG_PPC_DABRX: 1169 *val = get_reg_val(id, vcpu->arch.dabrx); 1170 break; 1171 case KVM_REG_PPC_DSCR: 1172 *val = get_reg_val(id, vcpu->arch.dscr); 1173 break; 1174 case KVM_REG_PPC_PURR: 1175 *val = get_reg_val(id, vcpu->arch.purr); 1176 break; 1177 case KVM_REG_PPC_SPURR: 1178 *val = get_reg_val(id, vcpu->arch.spurr); 1179 break; 1180 case KVM_REG_PPC_AMR: 1181 *val = get_reg_val(id, vcpu->arch.amr); 1182 break; 1183 case KVM_REG_PPC_UAMOR: 1184 *val = get_reg_val(id, vcpu->arch.uamor); 1185 break; 1186 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS: 1187 i = id - KVM_REG_PPC_MMCR0; 1188 *val = get_reg_val(id, vcpu->arch.mmcr[i]); 1189 break; 1190 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8: 1191 i = id - KVM_REG_PPC_PMC1; 1192 *val = get_reg_val(id, vcpu->arch.pmc[i]); 1193 break; 1194 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2: 1195 i = id - KVM_REG_PPC_SPMC1; 1196 *val = get_reg_val(id, vcpu->arch.spmc[i]); 1197 break; 1198 case KVM_REG_PPC_SIAR: 1199 *val = get_reg_val(id, vcpu->arch.siar); 1200 break; 1201 case KVM_REG_PPC_SDAR: 1202 *val = get_reg_val(id, vcpu->arch.sdar); 1203 break; 1204 case KVM_REG_PPC_SIER: 1205 *val = get_reg_val(id, vcpu->arch.sier); 1206 break; 1207 case KVM_REG_PPC_IAMR: 1208 *val = get_reg_val(id, vcpu->arch.iamr); 1209 break; 1210 case KVM_REG_PPC_PSPB: 1211 *val = get_reg_val(id, vcpu->arch.pspb); 1212 break; 1213 case KVM_REG_PPC_DPDES: 1214 *val = get_reg_val(id, vcpu->arch.vcore->dpdes); 1215 break; 1216 case KVM_REG_PPC_VTB: 1217 *val = get_reg_val(id, vcpu->arch.vcore->vtb); 1218 break; 1219 case KVM_REG_PPC_DAWR: 1220 *val = get_reg_val(id, vcpu->arch.dawr); 1221 break; 1222 case KVM_REG_PPC_DAWRX: 1223 *val = get_reg_val(id, vcpu->arch.dawrx); 1224 break; 1225 case KVM_REG_PPC_CIABR: 1226 *val = get_reg_val(id, vcpu->arch.ciabr); 1227 break; 1228 case KVM_REG_PPC_CSIGR: 1229 *val = get_reg_val(id, vcpu->arch.csigr); 1230 break; 1231 case KVM_REG_PPC_TACR: 1232 *val = get_reg_val(id, vcpu->arch.tacr); 1233 break; 1234 case KVM_REG_PPC_TCSCR: 1235 *val = get_reg_val(id, vcpu->arch.tcscr); 1236 break; 1237 case KVM_REG_PPC_PID: 1238 *val = get_reg_val(id, vcpu->arch.pid); 1239 break; 1240 case KVM_REG_PPC_ACOP: 1241 *val = get_reg_val(id, vcpu->arch.acop); 1242 break; 1243 case KVM_REG_PPC_WORT: 1244 *val = get_reg_val(id, vcpu->arch.wort); 1245 break; 1246 case KVM_REG_PPC_TIDR: 1247 *val = get_reg_val(id, vcpu->arch.tid); 1248 break; 1249 case KVM_REG_PPC_PSSCR: 1250 *val = get_reg_val(id, vcpu->arch.psscr); 1251 break; 1252 case KVM_REG_PPC_VPA_ADDR: 1253 spin_lock(&vcpu->arch.vpa_update_lock); 1254 *val = get_reg_val(id, vcpu->arch.vpa.next_gpa); 1255 spin_unlock(&vcpu->arch.vpa_update_lock); 1256 break; 1257 case KVM_REG_PPC_VPA_SLB: 1258 spin_lock(&vcpu->arch.vpa_update_lock); 1259 val->vpaval.addr = vcpu->arch.slb_shadow.next_gpa; 1260 val->vpaval.length = vcpu->arch.slb_shadow.len; 1261 spin_unlock(&vcpu->arch.vpa_update_lock); 1262 break; 1263 case KVM_REG_PPC_VPA_DTL: 1264 spin_lock(&vcpu->arch.vpa_update_lock); 1265 val->vpaval.addr = vcpu->arch.dtl.next_gpa; 1266 val->vpaval.length = vcpu->arch.dtl.len; 1267 spin_unlock(&vcpu->arch.vpa_update_lock); 1268 break; 1269 case KVM_REG_PPC_TB_OFFSET: 1270 *val = get_reg_val(id, vcpu->arch.vcore->tb_offset); 1271 break; 1272 case KVM_REG_PPC_LPCR: 1273 case KVM_REG_PPC_LPCR_64: 1274 *val = get_reg_val(id, vcpu->arch.vcore->lpcr); 1275 break; 1276 case KVM_REG_PPC_PPR: 1277 *val = get_reg_val(id, vcpu->arch.ppr); 1278 break; 1279 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1280 case KVM_REG_PPC_TFHAR: 1281 *val = get_reg_val(id, vcpu->arch.tfhar); 1282 break; 1283 case KVM_REG_PPC_TFIAR: 1284 *val = get_reg_val(id, vcpu->arch.tfiar); 1285 break; 1286 case KVM_REG_PPC_TEXASR: 1287 *val = get_reg_val(id, vcpu->arch.texasr); 1288 break; 1289 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31: 1290 i = id - KVM_REG_PPC_TM_GPR0; 1291 *val = get_reg_val(id, vcpu->arch.gpr_tm[i]); 1292 break; 1293 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63: 1294 { 1295 int j; 1296 i = id - KVM_REG_PPC_TM_VSR0; 1297 if (i < 32) 1298 for (j = 0; j < TS_FPRWIDTH; j++) 1299 val->vsxval[j] = vcpu->arch.fp_tm.fpr[i][j]; 1300 else { 1301 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 1302 val->vval = vcpu->arch.vr_tm.vr[i-32]; 1303 else 1304 r = -ENXIO; 1305 } 1306 break; 1307 } 1308 case KVM_REG_PPC_TM_CR: 1309 *val = get_reg_val(id, vcpu->arch.cr_tm); 1310 break; 1311 case KVM_REG_PPC_TM_XER: 1312 *val = get_reg_val(id, vcpu->arch.xer_tm); 1313 break; 1314 case KVM_REG_PPC_TM_LR: 1315 *val = get_reg_val(id, vcpu->arch.lr_tm); 1316 break; 1317 case KVM_REG_PPC_TM_CTR: 1318 *val = get_reg_val(id, vcpu->arch.ctr_tm); 1319 break; 1320 case KVM_REG_PPC_TM_FPSCR: 1321 *val = get_reg_val(id, vcpu->arch.fp_tm.fpscr); 1322 break; 1323 case KVM_REG_PPC_TM_AMR: 1324 *val = get_reg_val(id, vcpu->arch.amr_tm); 1325 break; 1326 case KVM_REG_PPC_TM_PPR: 1327 *val = get_reg_val(id, vcpu->arch.ppr_tm); 1328 break; 1329 case KVM_REG_PPC_TM_VRSAVE: 1330 *val = get_reg_val(id, vcpu->arch.vrsave_tm); 1331 break; 1332 case KVM_REG_PPC_TM_VSCR: 1333 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 1334 *val = get_reg_val(id, vcpu->arch.vr_tm.vscr.u[3]); 1335 else 1336 r = -ENXIO; 1337 break; 1338 case KVM_REG_PPC_TM_DSCR: 1339 *val = get_reg_val(id, vcpu->arch.dscr_tm); 1340 break; 1341 case KVM_REG_PPC_TM_TAR: 1342 *val = get_reg_val(id, vcpu->arch.tar_tm); 1343 break; 1344 #endif 1345 case KVM_REG_PPC_ARCH_COMPAT: 1346 *val = get_reg_val(id, vcpu->arch.vcore->arch_compat); 1347 break; 1348 default: 1349 r = -EINVAL; 1350 break; 1351 } 1352 1353 return r; 1354 } 1355 1356 static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, 1357 union kvmppc_one_reg *val) 1358 { 1359 int r = 0; 1360 long int i; 1361 unsigned long addr, len; 1362 1363 switch (id) { 1364 case KVM_REG_PPC_HIOR: 1365 /* Only allow this to be set to zero */ 1366 if (set_reg_val(id, *val)) 1367 r = -EINVAL; 1368 break; 1369 case KVM_REG_PPC_DABR: 1370 vcpu->arch.dabr = set_reg_val(id, *val); 1371 break; 1372 case KVM_REG_PPC_DABRX: 1373 vcpu->arch.dabrx = set_reg_val(id, *val) & ~DABRX_HYP; 1374 break; 1375 case KVM_REG_PPC_DSCR: 1376 vcpu->arch.dscr = set_reg_val(id, *val); 1377 break; 1378 case KVM_REG_PPC_PURR: 1379 vcpu->arch.purr = set_reg_val(id, *val); 1380 break; 1381 case KVM_REG_PPC_SPURR: 1382 vcpu->arch.spurr = set_reg_val(id, *val); 1383 break; 1384 case KVM_REG_PPC_AMR: 1385 vcpu->arch.amr = set_reg_val(id, *val); 1386 break; 1387 case KVM_REG_PPC_UAMOR: 1388 vcpu->arch.uamor = set_reg_val(id, *val); 1389 break; 1390 case KVM_REG_PPC_MMCR0 ... KVM_REG_PPC_MMCRS: 1391 i = id - KVM_REG_PPC_MMCR0; 1392 vcpu->arch.mmcr[i] = set_reg_val(id, *val); 1393 break; 1394 case KVM_REG_PPC_PMC1 ... KVM_REG_PPC_PMC8: 1395 i = id - KVM_REG_PPC_PMC1; 1396 vcpu->arch.pmc[i] = set_reg_val(id, *val); 1397 break; 1398 case KVM_REG_PPC_SPMC1 ... KVM_REG_PPC_SPMC2: 1399 i = id - KVM_REG_PPC_SPMC1; 1400 vcpu->arch.spmc[i] = set_reg_val(id, *val); 1401 break; 1402 case KVM_REG_PPC_SIAR: 1403 vcpu->arch.siar = set_reg_val(id, *val); 1404 break; 1405 case KVM_REG_PPC_SDAR: 1406 vcpu->arch.sdar = set_reg_val(id, *val); 1407 break; 1408 case KVM_REG_PPC_SIER: 1409 vcpu->arch.sier = set_reg_val(id, *val); 1410 break; 1411 case KVM_REG_PPC_IAMR: 1412 vcpu->arch.iamr = set_reg_val(id, *val); 1413 break; 1414 case KVM_REG_PPC_PSPB: 1415 vcpu->arch.pspb = set_reg_val(id, *val); 1416 break; 1417 case KVM_REG_PPC_DPDES: 1418 vcpu->arch.vcore->dpdes = set_reg_val(id, *val); 1419 break; 1420 case KVM_REG_PPC_VTB: 1421 vcpu->arch.vcore->vtb = set_reg_val(id, *val); 1422 break; 1423 case KVM_REG_PPC_DAWR: 1424 vcpu->arch.dawr = set_reg_val(id, *val); 1425 break; 1426 case KVM_REG_PPC_DAWRX: 1427 vcpu->arch.dawrx = set_reg_val(id, *val) & ~DAWRX_HYP; 1428 break; 1429 case KVM_REG_PPC_CIABR: 1430 vcpu->arch.ciabr = set_reg_val(id, *val); 1431 /* Don't allow setting breakpoints in hypervisor code */ 1432 if ((vcpu->arch.ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER) 1433 vcpu->arch.ciabr &= ~CIABR_PRIV; /* disable */ 1434 break; 1435 case KVM_REG_PPC_CSIGR: 1436 vcpu->arch.csigr = set_reg_val(id, *val); 1437 break; 1438 case KVM_REG_PPC_TACR: 1439 vcpu->arch.tacr = set_reg_val(id, *val); 1440 break; 1441 case KVM_REG_PPC_TCSCR: 1442 vcpu->arch.tcscr = set_reg_val(id, *val); 1443 break; 1444 case KVM_REG_PPC_PID: 1445 vcpu->arch.pid = set_reg_val(id, *val); 1446 break; 1447 case KVM_REG_PPC_ACOP: 1448 vcpu->arch.acop = set_reg_val(id, *val); 1449 break; 1450 case KVM_REG_PPC_WORT: 1451 vcpu->arch.wort = set_reg_val(id, *val); 1452 break; 1453 case KVM_REG_PPC_TIDR: 1454 vcpu->arch.tid = set_reg_val(id, *val); 1455 break; 1456 case KVM_REG_PPC_PSSCR: 1457 vcpu->arch.psscr = set_reg_val(id, *val) & PSSCR_GUEST_VIS; 1458 break; 1459 case KVM_REG_PPC_VPA_ADDR: 1460 addr = set_reg_val(id, *val); 1461 r = -EINVAL; 1462 if (!addr && (vcpu->arch.slb_shadow.next_gpa || 1463 vcpu->arch.dtl.next_gpa)) 1464 break; 1465 r = set_vpa(vcpu, &vcpu->arch.vpa, addr, sizeof(struct lppaca)); 1466 break; 1467 case KVM_REG_PPC_VPA_SLB: 1468 addr = val->vpaval.addr; 1469 len = val->vpaval.length; 1470 r = -EINVAL; 1471 if (addr && !vcpu->arch.vpa.next_gpa) 1472 break; 1473 r = set_vpa(vcpu, &vcpu->arch.slb_shadow, addr, len); 1474 break; 1475 case KVM_REG_PPC_VPA_DTL: 1476 addr = val->vpaval.addr; 1477 len = val->vpaval.length; 1478 r = -EINVAL; 1479 if (addr && (len < sizeof(struct dtl_entry) || 1480 !vcpu->arch.vpa.next_gpa)) 1481 break; 1482 len -= len % sizeof(struct dtl_entry); 1483 r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len); 1484 break; 1485 case KVM_REG_PPC_TB_OFFSET: 1486 /* round up to multiple of 2^24 */ 1487 vcpu->arch.vcore->tb_offset = 1488 ALIGN(set_reg_val(id, *val), 1UL << 24); 1489 break; 1490 case KVM_REG_PPC_LPCR: 1491 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true); 1492 break; 1493 case KVM_REG_PPC_LPCR_64: 1494 kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), false); 1495 break; 1496 case KVM_REG_PPC_PPR: 1497 vcpu->arch.ppr = set_reg_val(id, *val); 1498 break; 1499 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 1500 case KVM_REG_PPC_TFHAR: 1501 vcpu->arch.tfhar = set_reg_val(id, *val); 1502 break; 1503 case KVM_REG_PPC_TFIAR: 1504 vcpu->arch.tfiar = set_reg_val(id, *val); 1505 break; 1506 case KVM_REG_PPC_TEXASR: 1507 vcpu->arch.texasr = set_reg_val(id, *val); 1508 break; 1509 case KVM_REG_PPC_TM_GPR0 ... KVM_REG_PPC_TM_GPR31: 1510 i = id - KVM_REG_PPC_TM_GPR0; 1511 vcpu->arch.gpr_tm[i] = set_reg_val(id, *val); 1512 break; 1513 case KVM_REG_PPC_TM_VSR0 ... KVM_REG_PPC_TM_VSR63: 1514 { 1515 int j; 1516 i = id - KVM_REG_PPC_TM_VSR0; 1517 if (i < 32) 1518 for (j = 0; j < TS_FPRWIDTH; j++) 1519 vcpu->arch.fp_tm.fpr[i][j] = val->vsxval[j]; 1520 else 1521 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 1522 vcpu->arch.vr_tm.vr[i-32] = val->vval; 1523 else 1524 r = -ENXIO; 1525 break; 1526 } 1527 case KVM_REG_PPC_TM_CR: 1528 vcpu->arch.cr_tm = set_reg_val(id, *val); 1529 break; 1530 case KVM_REG_PPC_TM_XER: 1531 vcpu->arch.xer_tm = set_reg_val(id, *val); 1532 break; 1533 case KVM_REG_PPC_TM_LR: 1534 vcpu->arch.lr_tm = set_reg_val(id, *val); 1535 break; 1536 case KVM_REG_PPC_TM_CTR: 1537 vcpu->arch.ctr_tm = set_reg_val(id, *val); 1538 break; 1539 case KVM_REG_PPC_TM_FPSCR: 1540 vcpu->arch.fp_tm.fpscr = set_reg_val(id, *val); 1541 break; 1542 case KVM_REG_PPC_TM_AMR: 1543 vcpu->arch.amr_tm = set_reg_val(id, *val); 1544 break; 1545 case KVM_REG_PPC_TM_PPR: 1546 vcpu->arch.ppr_tm = set_reg_val(id, *val); 1547 break; 1548 case KVM_REG_PPC_TM_VRSAVE: 1549 vcpu->arch.vrsave_tm = set_reg_val(id, *val); 1550 break; 1551 case KVM_REG_PPC_TM_VSCR: 1552 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 1553 vcpu->arch.vr.vscr.u[3] = set_reg_val(id, *val); 1554 else 1555 r = - ENXIO; 1556 break; 1557 case KVM_REG_PPC_TM_DSCR: 1558 vcpu->arch.dscr_tm = set_reg_val(id, *val); 1559 break; 1560 case KVM_REG_PPC_TM_TAR: 1561 vcpu->arch.tar_tm = set_reg_val(id, *val); 1562 break; 1563 #endif 1564 case KVM_REG_PPC_ARCH_COMPAT: 1565 r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val)); 1566 break; 1567 default: 1568 r = -EINVAL; 1569 break; 1570 } 1571 1572 return r; 1573 } 1574 1575 /* 1576 * On POWER9, threads are independent and can be in different partitions. 1577 * Therefore we consider each thread to be a subcore. 1578 * There is a restriction that all threads have to be in the same 1579 * MMU mode (radix or HPT), unfortunately, but since we only support 1580 * HPT guests on a HPT host so far, that isn't an impediment yet. 1581 */ 1582 static int threads_per_vcore(void) 1583 { 1584 if (cpu_has_feature(CPU_FTR_ARCH_300)) 1585 return 1; 1586 return threads_per_subcore; 1587 } 1588 1589 static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core) 1590 { 1591 struct kvmppc_vcore *vcore; 1592 1593 vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL); 1594 1595 if (vcore == NULL) 1596 return NULL; 1597 1598 spin_lock_init(&vcore->lock); 1599 spin_lock_init(&vcore->stoltb_lock); 1600 init_swait_queue_head(&vcore->wq); 1601 vcore->preempt_tb = TB_NIL; 1602 vcore->lpcr = kvm->arch.lpcr; 1603 vcore->first_vcpuid = core * threads_per_vcore(); 1604 vcore->kvm = kvm; 1605 INIT_LIST_HEAD(&vcore->preempt_list); 1606 1607 return vcore; 1608 } 1609 1610 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING 1611 static struct debugfs_timings_element { 1612 const char *name; 1613 size_t offset; 1614 } timings[] = { 1615 {"rm_entry", offsetof(struct kvm_vcpu, arch.rm_entry)}, 1616 {"rm_intr", offsetof(struct kvm_vcpu, arch.rm_intr)}, 1617 {"rm_exit", offsetof(struct kvm_vcpu, arch.rm_exit)}, 1618 {"guest", offsetof(struct kvm_vcpu, arch.guest_time)}, 1619 {"cede", offsetof(struct kvm_vcpu, arch.cede_time)}, 1620 }; 1621 1622 #define N_TIMINGS (sizeof(timings) / sizeof(timings[0])) 1623 1624 struct debugfs_timings_state { 1625 struct kvm_vcpu *vcpu; 1626 unsigned int buflen; 1627 char buf[N_TIMINGS * 100]; 1628 }; 1629 1630 static int debugfs_timings_open(struct inode *inode, struct file *file) 1631 { 1632 struct kvm_vcpu *vcpu = inode->i_private; 1633 struct debugfs_timings_state *p; 1634 1635 p = kzalloc(sizeof(*p), GFP_KERNEL); 1636 if (!p) 1637 return -ENOMEM; 1638 1639 kvm_get_kvm(vcpu->kvm); 1640 p->vcpu = vcpu; 1641 file->private_data = p; 1642 1643 return nonseekable_open(inode, file); 1644 } 1645 1646 static int debugfs_timings_release(struct inode *inode, struct file *file) 1647 { 1648 struct debugfs_timings_state *p = file->private_data; 1649 1650 kvm_put_kvm(p->vcpu->kvm); 1651 kfree(p); 1652 return 0; 1653 } 1654 1655 static ssize_t debugfs_timings_read(struct file *file, char __user *buf, 1656 size_t len, loff_t *ppos) 1657 { 1658 struct debugfs_timings_state *p = file->private_data; 1659 struct kvm_vcpu *vcpu = p->vcpu; 1660 char *s, *buf_end; 1661 struct kvmhv_tb_accumulator tb; 1662 u64 count; 1663 loff_t pos; 1664 ssize_t n; 1665 int i, loops; 1666 bool ok; 1667 1668 if (!p->buflen) { 1669 s = p->buf; 1670 buf_end = s + sizeof(p->buf); 1671 for (i = 0; i < N_TIMINGS; ++i) { 1672 struct kvmhv_tb_accumulator *acc; 1673 1674 acc = (struct kvmhv_tb_accumulator *) 1675 ((unsigned long)vcpu + timings[i].offset); 1676 ok = false; 1677 for (loops = 0; loops < 1000; ++loops) { 1678 count = acc->seqcount; 1679 if (!(count & 1)) { 1680 smp_rmb(); 1681 tb = *acc; 1682 smp_rmb(); 1683 if (count == acc->seqcount) { 1684 ok = true; 1685 break; 1686 } 1687 } 1688 udelay(1); 1689 } 1690 if (!ok) 1691 snprintf(s, buf_end - s, "%s: stuck\n", 1692 timings[i].name); 1693 else 1694 snprintf(s, buf_end - s, 1695 "%s: %llu %llu %llu %llu\n", 1696 timings[i].name, count / 2, 1697 tb_to_ns(tb.tb_total), 1698 tb_to_ns(tb.tb_min), 1699 tb_to_ns(tb.tb_max)); 1700 s += strlen(s); 1701 } 1702 p->buflen = s - p->buf; 1703 } 1704 1705 pos = *ppos; 1706 if (pos >= p->buflen) 1707 return 0; 1708 if (len > p->buflen - pos) 1709 len = p->buflen - pos; 1710 n = copy_to_user(buf, p->buf + pos, len); 1711 if (n) { 1712 if (n == len) 1713 return -EFAULT; 1714 len -= n; 1715 } 1716 *ppos = pos + len; 1717 return len; 1718 } 1719 1720 static ssize_t debugfs_timings_write(struct file *file, const char __user *buf, 1721 size_t len, loff_t *ppos) 1722 { 1723 return -EACCES; 1724 } 1725 1726 static const struct file_operations debugfs_timings_ops = { 1727 .owner = THIS_MODULE, 1728 .open = debugfs_timings_open, 1729 .release = debugfs_timings_release, 1730 .read = debugfs_timings_read, 1731 .write = debugfs_timings_write, 1732 .llseek = generic_file_llseek, 1733 }; 1734 1735 /* Create a debugfs directory for the vcpu */ 1736 static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id) 1737 { 1738 char buf[16]; 1739 struct kvm *kvm = vcpu->kvm; 1740 1741 snprintf(buf, sizeof(buf), "vcpu%u", id); 1742 if (IS_ERR_OR_NULL(kvm->arch.debugfs_dir)) 1743 return; 1744 vcpu->arch.debugfs_dir = debugfs_create_dir(buf, kvm->arch.debugfs_dir); 1745 if (IS_ERR_OR_NULL(vcpu->arch.debugfs_dir)) 1746 return; 1747 vcpu->arch.debugfs_timings = 1748 debugfs_create_file("timings", 0444, vcpu->arch.debugfs_dir, 1749 vcpu, &debugfs_timings_ops); 1750 } 1751 1752 #else /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */ 1753 static void debugfs_vcpu_init(struct kvm_vcpu *vcpu, unsigned int id) 1754 { 1755 } 1756 #endif /* CONFIG_KVM_BOOK3S_HV_EXIT_TIMING */ 1757 1758 static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm, 1759 unsigned int id) 1760 { 1761 struct kvm_vcpu *vcpu; 1762 int err = -EINVAL; 1763 int core; 1764 struct kvmppc_vcore *vcore; 1765 1766 core = id / threads_per_vcore(); 1767 if (core >= KVM_MAX_VCORES) 1768 goto out; 1769 1770 err = -ENOMEM; 1771 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); 1772 if (!vcpu) 1773 goto out; 1774 1775 err = kvm_vcpu_init(vcpu, kvm, id); 1776 if (err) 1777 goto free_vcpu; 1778 1779 vcpu->arch.shared = &vcpu->arch.shregs; 1780 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE 1781 /* 1782 * The shared struct is never shared on HV, 1783 * so we can always use host endianness 1784 */ 1785 #ifdef __BIG_ENDIAN__ 1786 vcpu->arch.shared_big_endian = true; 1787 #else 1788 vcpu->arch.shared_big_endian = false; 1789 #endif 1790 #endif 1791 vcpu->arch.mmcr[0] = MMCR0_FC; 1792 vcpu->arch.ctrl = CTRL_RUNLATCH; 1793 /* default to host PVR, since we can't spoof it */ 1794 kvmppc_set_pvr_hv(vcpu, mfspr(SPRN_PVR)); 1795 spin_lock_init(&vcpu->arch.vpa_update_lock); 1796 spin_lock_init(&vcpu->arch.tbacct_lock); 1797 vcpu->arch.busy_preempt = TB_NIL; 1798 vcpu->arch.intr_msr = MSR_SF | MSR_ME; 1799 1800 kvmppc_mmu_book3s_hv_init(vcpu); 1801 1802 vcpu->arch.state = KVMPPC_VCPU_NOTREADY; 1803 1804 init_waitqueue_head(&vcpu->arch.cpu_run); 1805 1806 mutex_lock(&kvm->lock); 1807 vcore = kvm->arch.vcores[core]; 1808 if (!vcore) { 1809 vcore = kvmppc_vcore_create(kvm, core); 1810 kvm->arch.vcores[core] = vcore; 1811 kvm->arch.online_vcores++; 1812 } 1813 mutex_unlock(&kvm->lock); 1814 1815 if (!vcore) 1816 goto free_vcpu; 1817 1818 spin_lock(&vcore->lock); 1819 ++vcore->num_threads; 1820 spin_unlock(&vcore->lock); 1821 vcpu->arch.vcore = vcore; 1822 vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid; 1823 vcpu->arch.thread_cpu = -1; 1824 1825 vcpu->arch.cpu_type = KVM_CPU_3S_64; 1826 kvmppc_sanity_check(vcpu); 1827 1828 debugfs_vcpu_init(vcpu, id); 1829 1830 return vcpu; 1831 1832 free_vcpu: 1833 kmem_cache_free(kvm_vcpu_cache, vcpu); 1834 out: 1835 return ERR_PTR(err); 1836 } 1837 1838 static void unpin_vpa(struct kvm *kvm, struct kvmppc_vpa *vpa) 1839 { 1840 if (vpa->pinned_addr) 1841 kvmppc_unpin_guest_page(kvm, vpa->pinned_addr, vpa->gpa, 1842 vpa->dirty); 1843 } 1844 1845 static void kvmppc_core_vcpu_free_hv(struct kvm_vcpu *vcpu) 1846 { 1847 spin_lock(&vcpu->arch.vpa_update_lock); 1848 unpin_vpa(vcpu->kvm, &vcpu->arch.dtl); 1849 unpin_vpa(vcpu->kvm, &vcpu->arch.slb_shadow); 1850 unpin_vpa(vcpu->kvm, &vcpu->arch.vpa); 1851 spin_unlock(&vcpu->arch.vpa_update_lock); 1852 kvm_vcpu_uninit(vcpu); 1853 kmem_cache_free(kvm_vcpu_cache, vcpu); 1854 } 1855 1856 static int kvmppc_core_check_requests_hv(struct kvm_vcpu *vcpu) 1857 { 1858 /* Indicate we want to get back into the guest */ 1859 return 1; 1860 } 1861 1862 static void kvmppc_set_timer(struct kvm_vcpu *vcpu) 1863 { 1864 unsigned long dec_nsec, now; 1865 1866 now = get_tb(); 1867 if (now > vcpu->arch.dec_expires) { 1868 /* decrementer has already gone negative */ 1869 kvmppc_core_queue_dec(vcpu); 1870 kvmppc_core_prepare_to_enter(vcpu); 1871 return; 1872 } 1873 dec_nsec = (vcpu->arch.dec_expires - now) * NSEC_PER_SEC 1874 / tb_ticks_per_sec; 1875 hrtimer_start(&vcpu->arch.dec_timer, ktime_set(0, dec_nsec), 1876 HRTIMER_MODE_REL); 1877 vcpu->arch.timer_running = 1; 1878 } 1879 1880 static void kvmppc_end_cede(struct kvm_vcpu *vcpu) 1881 { 1882 vcpu->arch.ceded = 0; 1883 if (vcpu->arch.timer_running) { 1884 hrtimer_try_to_cancel(&vcpu->arch.dec_timer); 1885 vcpu->arch.timer_running = 0; 1886 } 1887 } 1888 1889 extern void __kvmppc_vcore_entry(void); 1890 1891 static void kvmppc_remove_runnable(struct kvmppc_vcore *vc, 1892 struct kvm_vcpu *vcpu) 1893 { 1894 u64 now; 1895 1896 if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE) 1897 return; 1898 spin_lock_irq(&vcpu->arch.tbacct_lock); 1899 now = mftb(); 1900 vcpu->arch.busy_stolen += vcore_stolen_time(vc, now) - 1901 vcpu->arch.stolen_logged; 1902 vcpu->arch.busy_preempt = now; 1903 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST; 1904 spin_unlock_irq(&vcpu->arch.tbacct_lock); 1905 --vc->n_runnable; 1906 WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], NULL); 1907 } 1908 1909 static int kvmppc_grab_hwthread(int cpu) 1910 { 1911 struct paca_struct *tpaca; 1912 long timeout = 10000; 1913 1914 tpaca = &paca[cpu]; 1915 1916 /* Ensure the thread won't go into the kernel if it wakes */ 1917 tpaca->kvm_hstate.kvm_vcpu = NULL; 1918 tpaca->kvm_hstate.kvm_vcore = NULL; 1919 tpaca->kvm_hstate.napping = 0; 1920 smp_wmb(); 1921 tpaca->kvm_hstate.hwthread_req = 1; 1922 1923 /* 1924 * If the thread is already executing in the kernel (e.g. handling 1925 * a stray interrupt), wait for it to get back to nap mode. 1926 * The smp_mb() is to ensure that our setting of hwthread_req 1927 * is visible before we look at hwthread_state, so if this 1928 * races with the code at system_reset_pSeries and the thread 1929 * misses our setting of hwthread_req, we are sure to see its 1930 * setting of hwthread_state, and vice versa. 1931 */ 1932 smp_mb(); 1933 while (tpaca->kvm_hstate.hwthread_state == KVM_HWTHREAD_IN_KERNEL) { 1934 if (--timeout <= 0) { 1935 pr_err("KVM: couldn't grab cpu %d\n", cpu); 1936 return -EBUSY; 1937 } 1938 udelay(1); 1939 } 1940 return 0; 1941 } 1942 1943 static void kvmppc_release_hwthread(int cpu) 1944 { 1945 struct paca_struct *tpaca; 1946 1947 tpaca = &paca[cpu]; 1948 tpaca->kvm_hstate.hwthread_req = 0; 1949 tpaca->kvm_hstate.kvm_vcpu = NULL; 1950 tpaca->kvm_hstate.kvm_vcore = NULL; 1951 tpaca->kvm_hstate.kvm_split_mode = NULL; 1952 } 1953 1954 static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc) 1955 { 1956 int cpu; 1957 struct paca_struct *tpaca; 1958 struct kvmppc_vcore *mvc = vc->master_vcore; 1959 1960 cpu = vc->pcpu; 1961 if (vcpu) { 1962 if (vcpu->arch.timer_running) { 1963 hrtimer_try_to_cancel(&vcpu->arch.dec_timer); 1964 vcpu->arch.timer_running = 0; 1965 } 1966 cpu += vcpu->arch.ptid; 1967 vcpu->cpu = mvc->pcpu; 1968 vcpu->arch.thread_cpu = cpu; 1969 } 1970 tpaca = &paca[cpu]; 1971 tpaca->kvm_hstate.kvm_vcpu = vcpu; 1972 tpaca->kvm_hstate.ptid = cpu - mvc->pcpu; 1973 /* Order stores to hstate.kvm_vcpu etc. before store to kvm_vcore */ 1974 smp_wmb(); 1975 tpaca->kvm_hstate.kvm_vcore = mvc; 1976 if (cpu != smp_processor_id()) 1977 kvmppc_ipi_thread(cpu); 1978 } 1979 1980 static void kvmppc_wait_for_nap(void) 1981 { 1982 int cpu = smp_processor_id(); 1983 int i, loops; 1984 int n_threads = threads_per_vcore(); 1985 1986 if (n_threads <= 1) 1987 return; 1988 for (loops = 0; loops < 1000000; ++loops) { 1989 /* 1990 * Check if all threads are finished. 1991 * We set the vcore pointer when starting a thread 1992 * and the thread clears it when finished, so we look 1993 * for any threads that still have a non-NULL vcore ptr. 1994 */ 1995 for (i = 1; i < n_threads; ++i) 1996 if (paca[cpu + i].kvm_hstate.kvm_vcore) 1997 break; 1998 if (i == n_threads) { 1999 HMT_medium(); 2000 return; 2001 } 2002 HMT_low(); 2003 } 2004 HMT_medium(); 2005 for (i = 1; i < n_threads; ++i) 2006 if (paca[cpu + i].kvm_hstate.kvm_vcore) 2007 pr_err("KVM: CPU %d seems to be stuck\n", cpu + i); 2008 } 2009 2010 /* 2011 * Check that we are on thread 0 and that any other threads in 2012 * this core are off-line. Then grab the threads so they can't 2013 * enter the kernel. 2014 */ 2015 static int on_primary_thread(void) 2016 { 2017 int cpu = smp_processor_id(); 2018 int thr; 2019 2020 /* Are we on a primary subcore? */ 2021 if (cpu_thread_in_subcore(cpu)) 2022 return 0; 2023 2024 thr = 0; 2025 while (++thr < threads_per_subcore) 2026 if (cpu_online(cpu + thr)) 2027 return 0; 2028 2029 /* Grab all hw threads so they can't go into the kernel */ 2030 for (thr = 1; thr < threads_per_subcore; ++thr) { 2031 if (kvmppc_grab_hwthread(cpu + thr)) { 2032 /* Couldn't grab one; let the others go */ 2033 do { 2034 kvmppc_release_hwthread(cpu + thr); 2035 } while (--thr > 0); 2036 return 0; 2037 } 2038 } 2039 return 1; 2040 } 2041 2042 /* 2043 * A list of virtual cores for each physical CPU. 2044 * These are vcores that could run but their runner VCPU tasks are 2045 * (or may be) preempted. 2046 */ 2047 struct preempted_vcore_list { 2048 struct list_head list; 2049 spinlock_t lock; 2050 }; 2051 2052 static DEFINE_PER_CPU(struct preempted_vcore_list, preempted_vcores); 2053 2054 static void init_vcore_lists(void) 2055 { 2056 int cpu; 2057 2058 for_each_possible_cpu(cpu) { 2059 struct preempted_vcore_list *lp = &per_cpu(preempted_vcores, cpu); 2060 spin_lock_init(&lp->lock); 2061 INIT_LIST_HEAD(&lp->list); 2062 } 2063 } 2064 2065 static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc) 2066 { 2067 struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores); 2068 2069 vc->vcore_state = VCORE_PREEMPT; 2070 vc->pcpu = smp_processor_id(); 2071 if (vc->num_threads < threads_per_vcore()) { 2072 spin_lock(&lp->lock); 2073 list_add_tail(&vc->preempt_list, &lp->list); 2074 spin_unlock(&lp->lock); 2075 } 2076 2077 /* Start accumulating stolen time */ 2078 kvmppc_core_start_stolen(vc); 2079 } 2080 2081 static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc) 2082 { 2083 struct preempted_vcore_list *lp; 2084 2085 kvmppc_core_end_stolen(vc); 2086 if (!list_empty(&vc->preempt_list)) { 2087 lp = &per_cpu(preempted_vcores, vc->pcpu); 2088 spin_lock(&lp->lock); 2089 list_del_init(&vc->preempt_list); 2090 spin_unlock(&lp->lock); 2091 } 2092 vc->vcore_state = VCORE_INACTIVE; 2093 } 2094 2095 /* 2096 * This stores information about the virtual cores currently 2097 * assigned to a physical core. 2098 */ 2099 struct core_info { 2100 int n_subcores; 2101 int max_subcore_threads; 2102 int total_threads; 2103 int subcore_threads[MAX_SUBCORES]; 2104 struct kvm *subcore_vm[MAX_SUBCORES]; 2105 struct list_head vcs[MAX_SUBCORES]; 2106 }; 2107 2108 /* 2109 * This mapping means subcores 0 and 1 can use threads 0-3 and 4-7 2110 * respectively in 2-way micro-threading (split-core) mode. 2111 */ 2112 static int subcore_thread_map[MAX_SUBCORES] = { 0, 4, 2, 6 }; 2113 2114 static void init_core_info(struct core_info *cip, struct kvmppc_vcore *vc) 2115 { 2116 int sub; 2117 2118 memset(cip, 0, sizeof(*cip)); 2119 cip->n_subcores = 1; 2120 cip->max_subcore_threads = vc->num_threads; 2121 cip->total_threads = vc->num_threads; 2122 cip->subcore_threads[0] = vc->num_threads; 2123 cip->subcore_vm[0] = vc->kvm; 2124 for (sub = 0; sub < MAX_SUBCORES; ++sub) 2125 INIT_LIST_HEAD(&cip->vcs[sub]); 2126 list_add_tail(&vc->preempt_list, &cip->vcs[0]); 2127 } 2128 2129 static bool subcore_config_ok(int n_subcores, int n_threads) 2130 { 2131 /* Can only dynamically split if unsplit to begin with */ 2132 if (n_subcores > 1 && threads_per_subcore < MAX_SMT_THREADS) 2133 return false; 2134 if (n_subcores > MAX_SUBCORES) 2135 return false; 2136 if (n_subcores > 1) { 2137 if (!(dynamic_mt_modes & 2)) 2138 n_subcores = 4; 2139 if (n_subcores > 2 && !(dynamic_mt_modes & 4)) 2140 return false; 2141 } 2142 2143 return n_subcores * roundup_pow_of_two(n_threads) <= MAX_SMT_THREADS; 2144 } 2145 2146 static void init_master_vcore(struct kvmppc_vcore *vc) 2147 { 2148 vc->master_vcore = vc; 2149 vc->entry_exit_map = 0; 2150 vc->in_guest = 0; 2151 vc->napping_threads = 0; 2152 vc->conferring_threads = 0; 2153 } 2154 2155 static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip) 2156 { 2157 int n_threads = vc->num_threads; 2158 int sub; 2159 2160 if (!cpu_has_feature(CPU_FTR_ARCH_207S)) 2161 return false; 2162 2163 if (n_threads < cip->max_subcore_threads) 2164 n_threads = cip->max_subcore_threads; 2165 if (!subcore_config_ok(cip->n_subcores + 1, n_threads)) 2166 return false; 2167 cip->max_subcore_threads = n_threads; 2168 2169 sub = cip->n_subcores; 2170 ++cip->n_subcores; 2171 cip->total_threads += vc->num_threads; 2172 cip->subcore_threads[sub] = vc->num_threads; 2173 cip->subcore_vm[sub] = vc->kvm; 2174 init_master_vcore(vc); 2175 list_move_tail(&vc->preempt_list, &cip->vcs[sub]); 2176 2177 return true; 2178 } 2179 2180 /* 2181 * Work out whether it is possible to piggyback the execution of 2182 * vcore *pvc onto the execution of the other vcores described in *cip. 2183 */ 2184 static bool can_piggyback(struct kvmppc_vcore *pvc, struct core_info *cip, 2185 int target_threads) 2186 { 2187 if (cip->total_threads + pvc->num_threads > target_threads) 2188 return false; 2189 2190 return can_dynamic_split(pvc, cip); 2191 } 2192 2193 static void prepare_threads(struct kvmppc_vcore *vc) 2194 { 2195 int i; 2196 struct kvm_vcpu *vcpu; 2197 2198 for_each_runnable_thread(i, vcpu, vc) { 2199 if (signal_pending(vcpu->arch.run_task)) 2200 vcpu->arch.ret = -EINTR; 2201 else if (vcpu->arch.vpa.update_pending || 2202 vcpu->arch.slb_shadow.update_pending || 2203 vcpu->arch.dtl.update_pending) 2204 vcpu->arch.ret = RESUME_GUEST; 2205 else 2206 continue; 2207 kvmppc_remove_runnable(vc, vcpu); 2208 wake_up(&vcpu->arch.cpu_run); 2209 } 2210 } 2211 2212 static void collect_piggybacks(struct core_info *cip, int target_threads) 2213 { 2214 struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores); 2215 struct kvmppc_vcore *pvc, *vcnext; 2216 2217 spin_lock(&lp->lock); 2218 list_for_each_entry_safe(pvc, vcnext, &lp->list, preempt_list) { 2219 if (!spin_trylock(&pvc->lock)) 2220 continue; 2221 prepare_threads(pvc); 2222 if (!pvc->n_runnable) { 2223 list_del_init(&pvc->preempt_list); 2224 if (pvc->runner == NULL) { 2225 pvc->vcore_state = VCORE_INACTIVE; 2226 kvmppc_core_end_stolen(pvc); 2227 } 2228 spin_unlock(&pvc->lock); 2229 continue; 2230 } 2231 if (!can_piggyback(pvc, cip, target_threads)) { 2232 spin_unlock(&pvc->lock); 2233 continue; 2234 } 2235 kvmppc_core_end_stolen(pvc); 2236 pvc->vcore_state = VCORE_PIGGYBACK; 2237 if (cip->total_threads >= target_threads) 2238 break; 2239 } 2240 spin_unlock(&lp->lock); 2241 } 2242 2243 static void post_guest_process(struct kvmppc_vcore *vc, bool is_master) 2244 { 2245 int still_running = 0, i; 2246 u64 now; 2247 long ret; 2248 struct kvm_vcpu *vcpu; 2249 2250 spin_lock(&vc->lock); 2251 now = get_tb(); 2252 for_each_runnable_thread(i, vcpu, vc) { 2253 /* cancel pending dec exception if dec is positive */ 2254 if (now < vcpu->arch.dec_expires && 2255 kvmppc_core_pending_dec(vcpu)) 2256 kvmppc_core_dequeue_dec(vcpu); 2257 2258 trace_kvm_guest_exit(vcpu); 2259 2260 ret = RESUME_GUEST; 2261 if (vcpu->arch.trap) 2262 ret = kvmppc_handle_exit_hv(vcpu->arch.kvm_run, vcpu, 2263 vcpu->arch.run_task); 2264 2265 vcpu->arch.ret = ret; 2266 vcpu->arch.trap = 0; 2267 2268 if (is_kvmppc_resume_guest(vcpu->arch.ret)) { 2269 if (vcpu->arch.pending_exceptions) 2270 kvmppc_core_prepare_to_enter(vcpu); 2271 if (vcpu->arch.ceded) 2272 kvmppc_set_timer(vcpu); 2273 else 2274 ++still_running; 2275 } else { 2276 kvmppc_remove_runnable(vc, vcpu); 2277 wake_up(&vcpu->arch.cpu_run); 2278 } 2279 } 2280 list_del_init(&vc->preempt_list); 2281 if (!is_master) { 2282 if (still_running > 0) { 2283 kvmppc_vcore_preempt(vc); 2284 } else if (vc->runner) { 2285 vc->vcore_state = VCORE_PREEMPT; 2286 kvmppc_core_start_stolen(vc); 2287 } else { 2288 vc->vcore_state = VCORE_INACTIVE; 2289 } 2290 if (vc->n_runnable > 0 && vc->runner == NULL) { 2291 /* make sure there's a candidate runner awake */ 2292 i = -1; 2293 vcpu = next_runnable_thread(vc, &i); 2294 wake_up(&vcpu->arch.cpu_run); 2295 } 2296 } 2297 spin_unlock(&vc->lock); 2298 } 2299 2300 /* 2301 * Clear core from the list of active host cores as we are about to 2302 * enter the guest. Only do this if it is the primary thread of the 2303 * core (not if a subcore) that is entering the guest. 2304 */ 2305 static inline int kvmppc_clear_host_core(unsigned int cpu) 2306 { 2307 int core; 2308 2309 if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu)) 2310 return 0; 2311 /* 2312 * Memory barrier can be omitted here as we will do a smp_wmb() 2313 * later in kvmppc_start_thread and we need ensure that state is 2314 * visible to other CPUs only after we enter guest. 2315 */ 2316 core = cpu >> threads_shift; 2317 kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 0; 2318 return 0; 2319 } 2320 2321 /* 2322 * Advertise this core as an active host core since we exited the guest 2323 * Only need to do this if it is the primary thread of the core that is 2324 * exiting. 2325 */ 2326 static inline int kvmppc_set_host_core(unsigned int cpu) 2327 { 2328 int core; 2329 2330 if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu)) 2331 return 0; 2332 2333 /* 2334 * Memory barrier can be omitted here because we do a spin_unlock 2335 * immediately after this which provides the memory barrier. 2336 */ 2337 core = cpu >> threads_shift; 2338 kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 1; 2339 return 0; 2340 } 2341 2342 /* 2343 * Run a set of guest threads on a physical core. 2344 * Called with vc->lock held. 2345 */ 2346 static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) 2347 { 2348 struct kvm_vcpu *vcpu; 2349 int i; 2350 int srcu_idx; 2351 struct core_info core_info; 2352 struct kvmppc_vcore *pvc, *vcnext; 2353 struct kvm_split_mode split_info, *sip; 2354 int split, subcore_size, active; 2355 int sub; 2356 bool thr0_done; 2357 unsigned long cmd_bit, stat_bit; 2358 int pcpu, thr; 2359 int target_threads; 2360 int controlled_threads; 2361 2362 /* 2363 * Remove from the list any threads that have a signal pending 2364 * or need a VPA update done 2365 */ 2366 prepare_threads(vc); 2367 2368 /* if the runner is no longer runnable, let the caller pick a new one */ 2369 if (vc->runner->arch.state != KVMPPC_VCPU_RUNNABLE) 2370 return; 2371 2372 /* 2373 * Initialize *vc. 2374 */ 2375 init_master_vcore(vc); 2376 vc->preempt_tb = TB_NIL; 2377 2378 /* 2379 * Number of threads that we will be controlling: the same as 2380 * the number of threads per subcore, except on POWER9, 2381 * where it's 1 because the threads are (mostly) independent. 2382 */ 2383 controlled_threads = threads_per_vcore(); 2384 2385 /* 2386 * Make sure we are running on primary threads, and that secondary 2387 * threads are offline. Also check if the number of threads in this 2388 * guest are greater than the current system threads per guest. 2389 */ 2390 if ((controlled_threads > 1) && 2391 ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) { 2392 for_each_runnable_thread(i, vcpu, vc) { 2393 vcpu->arch.ret = -EBUSY; 2394 kvmppc_remove_runnable(vc, vcpu); 2395 wake_up(&vcpu->arch.cpu_run); 2396 } 2397 goto out; 2398 } 2399 2400 /* 2401 * See if we could run any other vcores on the physical core 2402 * along with this one. 2403 */ 2404 init_core_info(&core_info, vc); 2405 pcpu = smp_processor_id(); 2406 target_threads = controlled_threads; 2407 if (target_smt_mode && target_smt_mode < target_threads) 2408 target_threads = target_smt_mode; 2409 if (vc->num_threads < target_threads) 2410 collect_piggybacks(&core_info, target_threads); 2411 2412 /* Decide on micro-threading (split-core) mode */ 2413 subcore_size = threads_per_subcore; 2414 cmd_bit = stat_bit = 0; 2415 split = core_info.n_subcores; 2416 sip = NULL; 2417 if (split > 1) { 2418 /* threads_per_subcore must be MAX_SMT_THREADS (8) here */ 2419 if (split == 2 && (dynamic_mt_modes & 2)) { 2420 cmd_bit = HID0_POWER8_1TO2LPAR; 2421 stat_bit = HID0_POWER8_2LPARMODE; 2422 } else { 2423 split = 4; 2424 cmd_bit = HID0_POWER8_1TO4LPAR; 2425 stat_bit = HID0_POWER8_4LPARMODE; 2426 } 2427 subcore_size = MAX_SMT_THREADS / split; 2428 sip = &split_info; 2429 memset(&split_info, 0, sizeof(split_info)); 2430 split_info.rpr = mfspr(SPRN_RPR); 2431 split_info.pmmar = mfspr(SPRN_PMMAR); 2432 split_info.ldbar = mfspr(SPRN_LDBAR); 2433 split_info.subcore_size = subcore_size; 2434 for (sub = 0; sub < core_info.n_subcores; ++sub) 2435 split_info.master_vcs[sub] = 2436 list_first_entry(&core_info.vcs[sub], 2437 struct kvmppc_vcore, preempt_list); 2438 /* order writes to split_info before kvm_split_mode pointer */ 2439 smp_wmb(); 2440 } 2441 pcpu = smp_processor_id(); 2442 for (thr = 0; thr < controlled_threads; ++thr) 2443 paca[pcpu + thr].kvm_hstate.kvm_split_mode = sip; 2444 2445 /* Initiate micro-threading (split-core) if required */ 2446 if (cmd_bit) { 2447 unsigned long hid0 = mfspr(SPRN_HID0); 2448 2449 hid0 |= cmd_bit | HID0_POWER8_DYNLPARDIS; 2450 mb(); 2451 mtspr(SPRN_HID0, hid0); 2452 isync(); 2453 for (;;) { 2454 hid0 = mfspr(SPRN_HID0); 2455 if (hid0 & stat_bit) 2456 break; 2457 cpu_relax(); 2458 } 2459 } 2460 2461 kvmppc_clear_host_core(pcpu); 2462 2463 /* Start all the threads */ 2464 active = 0; 2465 for (sub = 0; sub < core_info.n_subcores; ++sub) { 2466 thr = subcore_thread_map[sub]; 2467 thr0_done = false; 2468 active |= 1 << thr; 2469 list_for_each_entry(pvc, &core_info.vcs[sub], preempt_list) { 2470 pvc->pcpu = pcpu + thr; 2471 for_each_runnable_thread(i, vcpu, pvc) { 2472 kvmppc_start_thread(vcpu, pvc); 2473 kvmppc_create_dtl_entry(vcpu, pvc); 2474 trace_kvm_guest_enter(vcpu); 2475 if (!vcpu->arch.ptid) 2476 thr0_done = true; 2477 active |= 1 << (thr + vcpu->arch.ptid); 2478 } 2479 /* 2480 * We need to start the first thread of each subcore 2481 * even if it doesn't have a vcpu. 2482 */ 2483 if (pvc->master_vcore == pvc && !thr0_done) 2484 kvmppc_start_thread(NULL, pvc); 2485 thr += pvc->num_threads; 2486 } 2487 } 2488 2489 /* 2490 * Ensure that split_info.do_nap is set after setting 2491 * the vcore pointer in the PACA of the secondaries. 2492 */ 2493 smp_mb(); 2494 if (cmd_bit) 2495 split_info.do_nap = 1; /* ask secondaries to nap when done */ 2496 2497 /* 2498 * When doing micro-threading, poke the inactive threads as well. 2499 * This gets them to the nap instruction after kvm_do_nap, 2500 * which reduces the time taken to unsplit later. 2501 */ 2502 if (split > 1) 2503 for (thr = 1; thr < threads_per_subcore; ++thr) 2504 if (!(active & (1 << thr))) 2505 kvmppc_ipi_thread(pcpu + thr); 2506 2507 vc->vcore_state = VCORE_RUNNING; 2508 preempt_disable(); 2509 2510 trace_kvmppc_run_core(vc, 0); 2511 2512 for (sub = 0; sub < core_info.n_subcores; ++sub) 2513 list_for_each_entry(pvc, &core_info.vcs[sub], preempt_list) 2514 spin_unlock(&pvc->lock); 2515 2516 guest_enter(); 2517 2518 srcu_idx = srcu_read_lock(&vc->kvm->srcu); 2519 2520 __kvmppc_vcore_entry(); 2521 2522 srcu_read_unlock(&vc->kvm->srcu, srcu_idx); 2523 2524 spin_lock(&vc->lock); 2525 /* prevent other vcpu threads from doing kvmppc_start_thread() now */ 2526 vc->vcore_state = VCORE_EXITING; 2527 2528 /* wait for secondary threads to finish writing their state to memory */ 2529 kvmppc_wait_for_nap(); 2530 2531 /* Return to whole-core mode if we split the core earlier */ 2532 if (split > 1) { 2533 unsigned long hid0 = mfspr(SPRN_HID0); 2534 unsigned long loops = 0; 2535 2536 hid0 &= ~HID0_POWER8_DYNLPARDIS; 2537 stat_bit = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE; 2538 mb(); 2539 mtspr(SPRN_HID0, hid0); 2540 isync(); 2541 for (;;) { 2542 hid0 = mfspr(SPRN_HID0); 2543 if (!(hid0 & stat_bit)) 2544 break; 2545 cpu_relax(); 2546 ++loops; 2547 } 2548 split_info.do_nap = 0; 2549 } 2550 2551 /* Let secondaries go back to the offline loop */ 2552 for (i = 0; i < controlled_threads; ++i) { 2553 kvmppc_release_hwthread(pcpu + i); 2554 if (sip && sip->napped[i]) 2555 kvmppc_ipi_thread(pcpu + i); 2556 } 2557 2558 kvmppc_set_host_core(pcpu); 2559 2560 spin_unlock(&vc->lock); 2561 2562 /* make sure updates to secondary vcpu structs are visible now */ 2563 smp_mb(); 2564 guest_exit(); 2565 2566 for (sub = 0; sub < core_info.n_subcores; ++sub) 2567 list_for_each_entry_safe(pvc, vcnext, &core_info.vcs[sub], 2568 preempt_list) 2569 post_guest_process(pvc, pvc == vc); 2570 2571 spin_lock(&vc->lock); 2572 preempt_enable(); 2573 2574 out: 2575 vc->vcore_state = VCORE_INACTIVE; 2576 trace_kvmppc_run_core(vc, 1); 2577 } 2578 2579 /* 2580 * Wait for some other vcpu thread to execute us, and 2581 * wake us up when we need to handle something in the host. 2582 */ 2583 static void kvmppc_wait_for_exec(struct kvmppc_vcore *vc, 2584 struct kvm_vcpu *vcpu, int wait_state) 2585 { 2586 DEFINE_WAIT(wait); 2587 2588 prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state); 2589 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) { 2590 spin_unlock(&vc->lock); 2591 schedule(); 2592 spin_lock(&vc->lock); 2593 } 2594 finish_wait(&vcpu->arch.cpu_run, &wait); 2595 } 2596 2597 static void grow_halt_poll_ns(struct kvmppc_vcore *vc) 2598 { 2599 /* 10us base */ 2600 if (vc->halt_poll_ns == 0 && halt_poll_ns_grow) 2601 vc->halt_poll_ns = 10000; 2602 else 2603 vc->halt_poll_ns *= halt_poll_ns_grow; 2604 } 2605 2606 static void shrink_halt_poll_ns(struct kvmppc_vcore *vc) 2607 { 2608 if (halt_poll_ns_shrink == 0) 2609 vc->halt_poll_ns = 0; 2610 else 2611 vc->halt_poll_ns /= halt_poll_ns_shrink; 2612 } 2613 2614 /* 2615 * Check to see if any of the runnable vcpus on the vcore have pending 2616 * exceptions or are no longer ceded 2617 */ 2618 static int kvmppc_vcore_check_block(struct kvmppc_vcore *vc) 2619 { 2620 struct kvm_vcpu *vcpu; 2621 int i; 2622 2623 for_each_runnable_thread(i, vcpu, vc) { 2624 if (vcpu->arch.pending_exceptions || !vcpu->arch.ceded) 2625 return 1; 2626 } 2627 2628 return 0; 2629 } 2630 2631 /* 2632 * All the vcpus in this vcore are idle, so wait for a decrementer 2633 * or external interrupt to one of the vcpus. vc->lock is held. 2634 */ 2635 static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) 2636 { 2637 ktime_t cur, start_poll, start_wait; 2638 int do_sleep = 1; 2639 u64 block_ns; 2640 DECLARE_SWAITQUEUE(wait); 2641 2642 /* Poll for pending exceptions and ceded state */ 2643 cur = start_poll = ktime_get(); 2644 if (vc->halt_poll_ns) { 2645 ktime_t stop = ktime_add_ns(start_poll, vc->halt_poll_ns); 2646 ++vc->runner->stat.halt_attempted_poll; 2647 2648 vc->vcore_state = VCORE_POLLING; 2649 spin_unlock(&vc->lock); 2650 2651 do { 2652 if (kvmppc_vcore_check_block(vc)) { 2653 do_sleep = 0; 2654 break; 2655 } 2656 cur = ktime_get(); 2657 } while (single_task_running() && ktime_before(cur, stop)); 2658 2659 spin_lock(&vc->lock); 2660 vc->vcore_state = VCORE_INACTIVE; 2661 2662 if (!do_sleep) { 2663 ++vc->runner->stat.halt_successful_poll; 2664 goto out; 2665 } 2666 } 2667 2668 prepare_to_swait(&vc->wq, &wait, TASK_INTERRUPTIBLE); 2669 2670 if (kvmppc_vcore_check_block(vc)) { 2671 finish_swait(&vc->wq, &wait); 2672 do_sleep = 0; 2673 /* If we polled, count this as a successful poll */ 2674 if (vc->halt_poll_ns) 2675 ++vc->runner->stat.halt_successful_poll; 2676 goto out; 2677 } 2678 2679 start_wait = ktime_get(); 2680 2681 vc->vcore_state = VCORE_SLEEPING; 2682 trace_kvmppc_vcore_blocked(vc, 0); 2683 spin_unlock(&vc->lock); 2684 schedule(); 2685 finish_swait(&vc->wq, &wait); 2686 spin_lock(&vc->lock); 2687 vc->vcore_state = VCORE_INACTIVE; 2688 trace_kvmppc_vcore_blocked(vc, 1); 2689 ++vc->runner->stat.halt_successful_wait; 2690 2691 cur = ktime_get(); 2692 2693 out: 2694 block_ns = ktime_to_ns(cur) - ktime_to_ns(start_poll); 2695 2696 /* Attribute wait time */ 2697 if (do_sleep) { 2698 vc->runner->stat.halt_wait_ns += 2699 ktime_to_ns(cur) - ktime_to_ns(start_wait); 2700 /* Attribute failed poll time */ 2701 if (vc->halt_poll_ns) 2702 vc->runner->stat.halt_poll_fail_ns += 2703 ktime_to_ns(start_wait) - 2704 ktime_to_ns(start_poll); 2705 } else { 2706 /* Attribute successful poll time */ 2707 if (vc->halt_poll_ns) 2708 vc->runner->stat.halt_poll_success_ns += 2709 ktime_to_ns(cur) - 2710 ktime_to_ns(start_poll); 2711 } 2712 2713 /* Adjust poll time */ 2714 if (halt_poll_ns) { 2715 if (block_ns <= vc->halt_poll_ns) 2716 ; 2717 /* We slept and blocked for longer than the max halt time */ 2718 else if (vc->halt_poll_ns && block_ns > halt_poll_ns) 2719 shrink_halt_poll_ns(vc); 2720 /* We slept and our poll time is too small */ 2721 else if (vc->halt_poll_ns < halt_poll_ns && 2722 block_ns < halt_poll_ns) 2723 grow_halt_poll_ns(vc); 2724 if (vc->halt_poll_ns > halt_poll_ns) 2725 vc->halt_poll_ns = halt_poll_ns; 2726 } else 2727 vc->halt_poll_ns = 0; 2728 2729 trace_kvmppc_vcore_wakeup(do_sleep, block_ns); 2730 } 2731 2732 static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) 2733 { 2734 int n_ceded, i; 2735 struct kvmppc_vcore *vc; 2736 struct kvm_vcpu *v; 2737 2738 trace_kvmppc_run_vcpu_enter(vcpu); 2739 2740 kvm_run->exit_reason = 0; 2741 vcpu->arch.ret = RESUME_GUEST; 2742 vcpu->arch.trap = 0; 2743 kvmppc_update_vpas(vcpu); 2744 2745 /* 2746 * Synchronize with other threads in this virtual core 2747 */ 2748 vc = vcpu->arch.vcore; 2749 spin_lock(&vc->lock); 2750 vcpu->arch.ceded = 0; 2751 vcpu->arch.run_task = current; 2752 vcpu->arch.kvm_run = kvm_run; 2753 vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb()); 2754 vcpu->arch.state = KVMPPC_VCPU_RUNNABLE; 2755 vcpu->arch.busy_preempt = TB_NIL; 2756 WRITE_ONCE(vc->runnable_threads[vcpu->arch.ptid], vcpu); 2757 ++vc->n_runnable; 2758 2759 /* 2760 * This happens the first time this is called for a vcpu. 2761 * If the vcore is already running, we may be able to start 2762 * this thread straight away and have it join in. 2763 */ 2764 if (!signal_pending(current)) { 2765 if (vc->vcore_state == VCORE_PIGGYBACK) { 2766 struct kvmppc_vcore *mvc = vc->master_vcore; 2767 if (spin_trylock(&mvc->lock)) { 2768 if (mvc->vcore_state == VCORE_RUNNING && 2769 !VCORE_IS_EXITING(mvc)) { 2770 kvmppc_create_dtl_entry(vcpu, vc); 2771 kvmppc_start_thread(vcpu, vc); 2772 trace_kvm_guest_enter(vcpu); 2773 } 2774 spin_unlock(&mvc->lock); 2775 } 2776 } else if (vc->vcore_state == VCORE_RUNNING && 2777 !VCORE_IS_EXITING(vc)) { 2778 kvmppc_create_dtl_entry(vcpu, vc); 2779 kvmppc_start_thread(vcpu, vc); 2780 trace_kvm_guest_enter(vcpu); 2781 } else if (vc->vcore_state == VCORE_SLEEPING) { 2782 swake_up(&vc->wq); 2783 } 2784 2785 } 2786 2787 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE && 2788 !signal_pending(current)) { 2789 if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL) 2790 kvmppc_vcore_end_preempt(vc); 2791 2792 if (vc->vcore_state != VCORE_INACTIVE) { 2793 kvmppc_wait_for_exec(vc, vcpu, TASK_INTERRUPTIBLE); 2794 continue; 2795 } 2796 for_each_runnable_thread(i, v, vc) { 2797 kvmppc_core_prepare_to_enter(v); 2798 if (signal_pending(v->arch.run_task)) { 2799 kvmppc_remove_runnable(vc, v); 2800 v->stat.signal_exits++; 2801 v->arch.kvm_run->exit_reason = KVM_EXIT_INTR; 2802 v->arch.ret = -EINTR; 2803 wake_up(&v->arch.cpu_run); 2804 } 2805 } 2806 if (!vc->n_runnable || vcpu->arch.state != KVMPPC_VCPU_RUNNABLE) 2807 break; 2808 n_ceded = 0; 2809 for_each_runnable_thread(i, v, vc) { 2810 if (!v->arch.pending_exceptions) 2811 n_ceded += v->arch.ceded; 2812 else 2813 v->arch.ceded = 0; 2814 } 2815 vc->runner = vcpu; 2816 if (n_ceded == vc->n_runnable) { 2817 kvmppc_vcore_blocked(vc); 2818 } else if (need_resched()) { 2819 kvmppc_vcore_preempt(vc); 2820 /* Let something else run */ 2821 cond_resched_lock(&vc->lock); 2822 if (vc->vcore_state == VCORE_PREEMPT) 2823 kvmppc_vcore_end_preempt(vc); 2824 } else { 2825 kvmppc_run_core(vc); 2826 } 2827 vc->runner = NULL; 2828 } 2829 2830 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE && 2831 (vc->vcore_state == VCORE_RUNNING || 2832 vc->vcore_state == VCORE_EXITING || 2833 vc->vcore_state == VCORE_PIGGYBACK)) 2834 kvmppc_wait_for_exec(vc, vcpu, TASK_UNINTERRUPTIBLE); 2835 2836 if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL) 2837 kvmppc_vcore_end_preempt(vc); 2838 2839 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) { 2840 kvmppc_remove_runnable(vc, vcpu); 2841 vcpu->stat.signal_exits++; 2842 kvm_run->exit_reason = KVM_EXIT_INTR; 2843 vcpu->arch.ret = -EINTR; 2844 } 2845 2846 if (vc->n_runnable && vc->vcore_state == VCORE_INACTIVE) { 2847 /* Wake up some vcpu to run the core */ 2848 i = -1; 2849 v = next_runnable_thread(vc, &i); 2850 wake_up(&v->arch.cpu_run); 2851 } 2852 2853 trace_kvmppc_run_vcpu_exit(vcpu, kvm_run); 2854 spin_unlock(&vc->lock); 2855 return vcpu->arch.ret; 2856 } 2857 2858 static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu) 2859 { 2860 int r; 2861 int srcu_idx; 2862 2863 if (!vcpu->arch.sane) { 2864 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 2865 return -EINVAL; 2866 } 2867 2868 kvmppc_core_prepare_to_enter(vcpu); 2869 2870 /* No need to go into the guest when all we'll do is come back out */ 2871 if (signal_pending(current)) { 2872 run->exit_reason = KVM_EXIT_INTR; 2873 return -EINTR; 2874 } 2875 2876 atomic_inc(&vcpu->kvm->arch.vcpus_running); 2877 /* Order vcpus_running vs. hpte_setup_done, see kvmppc_alloc_reset_hpt */ 2878 smp_mb(); 2879 2880 /* On the first time here, set up HTAB and VRMA */ 2881 if (!vcpu->kvm->arch.hpte_setup_done) { 2882 r = kvmppc_hv_setup_htab_rma(vcpu); 2883 if (r) 2884 goto out; 2885 } 2886 2887 flush_all_to_thread(current); 2888 2889 vcpu->arch.wqp = &vcpu->arch.vcore->wq; 2890 vcpu->arch.pgdir = current->mm->pgd; 2891 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST; 2892 2893 do { 2894 r = kvmppc_run_vcpu(run, vcpu); 2895 2896 if (run->exit_reason == KVM_EXIT_PAPR_HCALL && 2897 !(vcpu->arch.shregs.msr & MSR_PR)) { 2898 trace_kvm_hcall_enter(vcpu); 2899 r = kvmppc_pseries_do_hcall(vcpu); 2900 trace_kvm_hcall_exit(vcpu, r); 2901 kvmppc_core_prepare_to_enter(vcpu); 2902 } else if (r == RESUME_PAGE_FAULT) { 2903 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 2904 r = kvmppc_book3s_hv_page_fault(run, vcpu, 2905 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr); 2906 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); 2907 } else if (r == RESUME_PASSTHROUGH) 2908 r = kvmppc_xics_rm_complete(vcpu, 0); 2909 } while (is_kvmppc_resume_guest(r)); 2910 2911 out: 2912 vcpu->arch.state = KVMPPC_VCPU_NOTREADY; 2913 atomic_dec(&vcpu->kvm->arch.vcpus_running); 2914 return r; 2915 } 2916 2917 static void kvmppc_add_seg_page_size(struct kvm_ppc_one_seg_page_size **sps, 2918 int linux_psize) 2919 { 2920 struct mmu_psize_def *def = &mmu_psize_defs[linux_psize]; 2921 2922 if (!def->shift) 2923 return; 2924 (*sps)->page_shift = def->shift; 2925 (*sps)->slb_enc = def->sllp; 2926 (*sps)->enc[0].page_shift = def->shift; 2927 (*sps)->enc[0].pte_enc = def->penc[linux_psize]; 2928 /* 2929 * Add 16MB MPSS support if host supports it 2930 */ 2931 if (linux_psize != MMU_PAGE_16M && def->penc[MMU_PAGE_16M] != -1) { 2932 (*sps)->enc[1].page_shift = 24; 2933 (*sps)->enc[1].pte_enc = def->penc[MMU_PAGE_16M]; 2934 } 2935 (*sps)++; 2936 } 2937 2938 static int kvm_vm_ioctl_get_smmu_info_hv(struct kvm *kvm, 2939 struct kvm_ppc_smmu_info *info) 2940 { 2941 struct kvm_ppc_one_seg_page_size *sps; 2942 2943 info->flags = KVM_PPC_PAGE_SIZES_REAL; 2944 if (mmu_has_feature(MMU_FTR_1T_SEGMENT)) 2945 info->flags |= KVM_PPC_1T_SEGMENTS; 2946 info->slb_size = mmu_slb_size; 2947 2948 /* We only support these sizes for now, and no muti-size segments */ 2949 sps = &info->sps[0]; 2950 kvmppc_add_seg_page_size(&sps, MMU_PAGE_4K); 2951 kvmppc_add_seg_page_size(&sps, MMU_PAGE_64K); 2952 kvmppc_add_seg_page_size(&sps, MMU_PAGE_16M); 2953 2954 return 0; 2955 } 2956 2957 /* 2958 * Get (and clear) the dirty memory log for a memory slot. 2959 */ 2960 static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm, 2961 struct kvm_dirty_log *log) 2962 { 2963 struct kvm_memslots *slots; 2964 struct kvm_memory_slot *memslot; 2965 int r; 2966 unsigned long n; 2967 2968 mutex_lock(&kvm->slots_lock); 2969 2970 r = -EINVAL; 2971 if (log->slot >= KVM_USER_MEM_SLOTS) 2972 goto out; 2973 2974 slots = kvm_memslots(kvm); 2975 memslot = id_to_memslot(slots, log->slot); 2976 r = -ENOENT; 2977 if (!memslot->dirty_bitmap) 2978 goto out; 2979 2980 n = kvm_dirty_bitmap_bytes(memslot); 2981 memset(memslot->dirty_bitmap, 0, n); 2982 2983 r = kvmppc_hv_get_dirty_log(kvm, memslot, memslot->dirty_bitmap); 2984 if (r) 2985 goto out; 2986 2987 r = -EFAULT; 2988 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) 2989 goto out; 2990 2991 r = 0; 2992 out: 2993 mutex_unlock(&kvm->slots_lock); 2994 return r; 2995 } 2996 2997 static void kvmppc_core_free_memslot_hv(struct kvm_memory_slot *free, 2998 struct kvm_memory_slot *dont) 2999 { 3000 if (!dont || free->arch.rmap != dont->arch.rmap) { 3001 vfree(free->arch.rmap); 3002 free->arch.rmap = NULL; 3003 } 3004 } 3005 3006 static int kvmppc_core_create_memslot_hv(struct kvm_memory_slot *slot, 3007 unsigned long npages) 3008 { 3009 slot->arch.rmap = vzalloc(npages * sizeof(*slot->arch.rmap)); 3010 if (!slot->arch.rmap) 3011 return -ENOMEM; 3012 3013 return 0; 3014 } 3015 3016 static int kvmppc_core_prepare_memory_region_hv(struct kvm *kvm, 3017 struct kvm_memory_slot *memslot, 3018 const struct kvm_userspace_memory_region *mem) 3019 { 3020 return 0; 3021 } 3022 3023 static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm, 3024 const struct kvm_userspace_memory_region *mem, 3025 const struct kvm_memory_slot *old, 3026 const struct kvm_memory_slot *new) 3027 { 3028 unsigned long npages = mem->memory_size >> PAGE_SHIFT; 3029 struct kvm_memslots *slots; 3030 struct kvm_memory_slot *memslot; 3031 3032 /* 3033 * If we are making a new memslot, it might make 3034 * some address that was previously cached as emulated 3035 * MMIO be no longer emulated MMIO, so invalidate 3036 * all the caches of emulated MMIO translations. 3037 */ 3038 if (npages) 3039 atomic64_inc(&kvm->arch.mmio_update); 3040 3041 if (npages && old->npages) { 3042 /* 3043 * If modifying a memslot, reset all the rmap dirty bits. 3044 * If this is a new memslot, we don't need to do anything 3045 * since the rmap array starts out as all zeroes, 3046 * i.e. no pages are dirty. 3047 */ 3048 slots = kvm_memslots(kvm); 3049 memslot = id_to_memslot(slots, mem->slot); 3050 kvmppc_hv_get_dirty_log(kvm, memslot, NULL); 3051 } 3052 } 3053 3054 /* 3055 * Update LPCR values in kvm->arch and in vcores. 3056 * Caller must hold kvm->lock. 3057 */ 3058 void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr, unsigned long mask) 3059 { 3060 long int i; 3061 u32 cores_done = 0; 3062 3063 if ((kvm->arch.lpcr & mask) == lpcr) 3064 return; 3065 3066 kvm->arch.lpcr = (kvm->arch.lpcr & ~mask) | lpcr; 3067 3068 for (i = 0; i < KVM_MAX_VCORES; ++i) { 3069 struct kvmppc_vcore *vc = kvm->arch.vcores[i]; 3070 if (!vc) 3071 continue; 3072 spin_lock(&vc->lock); 3073 vc->lpcr = (vc->lpcr & ~mask) | lpcr; 3074 spin_unlock(&vc->lock); 3075 if (++cores_done >= kvm->arch.online_vcores) 3076 break; 3077 } 3078 } 3079 3080 static void kvmppc_mmu_destroy_hv(struct kvm_vcpu *vcpu) 3081 { 3082 return; 3083 } 3084 3085 static void kvmppc_setup_partition_table(struct kvm *kvm) 3086 { 3087 unsigned long dw0, dw1; 3088 3089 /* PS field - page size for VRMA */ 3090 dw0 = ((kvm->arch.vrma_slb_v & SLB_VSID_L) >> 1) | 3091 ((kvm->arch.vrma_slb_v & SLB_VSID_LP) << 1); 3092 /* HTABSIZE and HTABORG fields */ 3093 dw0 |= kvm->arch.sdr1; 3094 3095 /* Second dword has GR=0; other fields are unused since UPRT=0 */ 3096 dw1 = 0; 3097 3098 mmu_partition_table_set_entry(kvm->arch.lpid, dw0, dw1); 3099 } 3100 3101 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu) 3102 { 3103 int err = 0; 3104 struct kvm *kvm = vcpu->kvm; 3105 unsigned long hva; 3106 struct kvm_memory_slot *memslot; 3107 struct vm_area_struct *vma; 3108 unsigned long lpcr = 0, senc; 3109 unsigned long psize, porder; 3110 int srcu_idx; 3111 3112 mutex_lock(&kvm->lock); 3113 if (kvm->arch.hpte_setup_done) 3114 goto out; /* another vcpu beat us to it */ 3115 3116 /* Allocate hashed page table (if not done already) and reset it */ 3117 if (!kvm->arch.hpt_virt) { 3118 err = kvmppc_alloc_hpt(kvm, NULL); 3119 if (err) { 3120 pr_err("KVM: Couldn't alloc HPT\n"); 3121 goto out; 3122 } 3123 } 3124 3125 /* Look up the memslot for guest physical address 0 */ 3126 srcu_idx = srcu_read_lock(&kvm->srcu); 3127 memslot = gfn_to_memslot(kvm, 0); 3128 3129 /* We must have some memory at 0 by now */ 3130 err = -EINVAL; 3131 if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) 3132 goto out_srcu; 3133 3134 /* Look up the VMA for the start of this memory slot */ 3135 hva = memslot->userspace_addr; 3136 down_read(¤t->mm->mmap_sem); 3137 vma = find_vma(current->mm, hva); 3138 if (!vma || vma->vm_start > hva || (vma->vm_flags & VM_IO)) 3139 goto up_out; 3140 3141 psize = vma_kernel_pagesize(vma); 3142 porder = __ilog2(psize); 3143 3144 up_read(¤t->mm->mmap_sem); 3145 3146 /* We can handle 4k, 64k or 16M pages in the VRMA */ 3147 err = -EINVAL; 3148 if (!(psize == 0x1000 || psize == 0x10000 || 3149 psize == 0x1000000)) 3150 goto out_srcu; 3151 3152 senc = slb_pgsize_encoding(psize); 3153 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T | 3154 (VRMA_VSID << SLB_VSID_SHIFT_1T); 3155 /* Create HPTEs in the hash page table for the VRMA */ 3156 kvmppc_map_vrma(vcpu, memslot, porder); 3157 3158 /* Update VRMASD field in the LPCR */ 3159 if (!cpu_has_feature(CPU_FTR_ARCH_300)) { 3160 /* the -4 is to account for senc values starting at 0x10 */ 3161 lpcr = senc << (LPCR_VRMASD_SH - 4); 3162 kvmppc_update_lpcr(kvm, lpcr, LPCR_VRMASD); 3163 } else { 3164 kvmppc_setup_partition_table(kvm); 3165 } 3166 3167 /* Order updates to kvm->arch.lpcr etc. vs. hpte_setup_done */ 3168 smp_wmb(); 3169 kvm->arch.hpte_setup_done = 1; 3170 err = 0; 3171 out_srcu: 3172 srcu_read_unlock(&kvm->srcu, srcu_idx); 3173 out: 3174 mutex_unlock(&kvm->lock); 3175 return err; 3176 3177 up_out: 3178 up_read(¤t->mm->mmap_sem); 3179 goto out_srcu; 3180 } 3181 3182 #ifdef CONFIG_KVM_XICS 3183 /* 3184 * Allocate a per-core structure for managing state about which cores are 3185 * running in the host versus the guest and for exchanging data between 3186 * real mode KVM and CPU running in the host. 3187 * This is only done for the first VM. 3188 * The allocated structure stays even if all VMs have stopped. 3189 * It is only freed when the kvm-hv module is unloaded. 3190 * It's OK for this routine to fail, we just don't support host 3191 * core operations like redirecting H_IPI wakeups. 3192 */ 3193 void kvmppc_alloc_host_rm_ops(void) 3194 { 3195 struct kvmppc_host_rm_ops *ops; 3196 unsigned long l_ops; 3197 int cpu, core; 3198 int size; 3199 3200 /* Not the first time here ? */ 3201 if (kvmppc_host_rm_ops_hv != NULL) 3202 return; 3203 3204 ops = kzalloc(sizeof(struct kvmppc_host_rm_ops), GFP_KERNEL); 3205 if (!ops) 3206 return; 3207 3208 size = cpu_nr_cores() * sizeof(struct kvmppc_host_rm_core); 3209 ops->rm_core = kzalloc(size, GFP_KERNEL); 3210 3211 if (!ops->rm_core) { 3212 kfree(ops); 3213 return; 3214 } 3215 3216 get_online_cpus(); 3217 3218 for (cpu = 0; cpu < nr_cpu_ids; cpu += threads_per_core) { 3219 if (!cpu_online(cpu)) 3220 continue; 3221 3222 core = cpu >> threads_shift; 3223 ops->rm_core[core].rm_state.in_host = 1; 3224 } 3225 3226 ops->vcpu_kick = kvmppc_fast_vcpu_kick_hv; 3227 3228 /* 3229 * Make the contents of the kvmppc_host_rm_ops structure visible 3230 * to other CPUs before we assign it to the global variable. 3231 * Do an atomic assignment (no locks used here), but if someone 3232 * beats us to it, just free our copy and return. 3233 */ 3234 smp_wmb(); 3235 l_ops = (unsigned long) ops; 3236 3237 if (cmpxchg64((unsigned long *)&kvmppc_host_rm_ops_hv, 0, l_ops)) { 3238 put_online_cpus(); 3239 kfree(ops->rm_core); 3240 kfree(ops); 3241 return; 3242 } 3243 3244 cpuhp_setup_state_nocalls(CPUHP_KVM_PPC_BOOK3S_PREPARE, 3245 "ppc/kvm_book3s:prepare", 3246 kvmppc_set_host_core, 3247 kvmppc_clear_host_core); 3248 put_online_cpus(); 3249 } 3250 3251 void kvmppc_free_host_rm_ops(void) 3252 { 3253 if (kvmppc_host_rm_ops_hv) { 3254 cpuhp_remove_state_nocalls(CPUHP_KVM_PPC_BOOK3S_PREPARE); 3255 kfree(kvmppc_host_rm_ops_hv->rm_core); 3256 kfree(kvmppc_host_rm_ops_hv); 3257 kvmppc_host_rm_ops_hv = NULL; 3258 } 3259 } 3260 #endif 3261 3262 static int kvmppc_core_init_vm_hv(struct kvm *kvm) 3263 { 3264 unsigned long lpcr, lpid; 3265 char buf[32]; 3266 3267 /* Allocate the guest's logical partition ID */ 3268 3269 lpid = kvmppc_alloc_lpid(); 3270 if ((long)lpid < 0) 3271 return -ENOMEM; 3272 kvm->arch.lpid = lpid; 3273 3274 kvmppc_alloc_host_rm_ops(); 3275 3276 /* 3277 * Since we don't flush the TLB when tearing down a VM, 3278 * and this lpid might have previously been used, 3279 * make sure we flush on each core before running the new VM. 3280 * On POWER9, the tlbie in mmu_partition_table_set_entry() 3281 * does this flush for us. 3282 */ 3283 if (!cpu_has_feature(CPU_FTR_ARCH_300)) 3284 cpumask_setall(&kvm->arch.need_tlb_flush); 3285 3286 /* Start out with the default set of hcalls enabled */ 3287 memcpy(kvm->arch.enabled_hcalls, default_enabled_hcalls, 3288 sizeof(kvm->arch.enabled_hcalls)); 3289 3290 if (!cpu_has_feature(CPU_FTR_ARCH_300)) 3291 kvm->arch.host_sdr1 = mfspr(SPRN_SDR1); 3292 3293 /* Init LPCR for virtual RMA mode */ 3294 kvm->arch.host_lpid = mfspr(SPRN_LPID); 3295 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR); 3296 lpcr &= LPCR_PECE | LPCR_LPES; 3297 lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE | 3298 LPCR_VPM0 | LPCR_VPM1; 3299 kvm->arch.vrma_slb_v = SLB_VSID_B_1T | 3300 (VRMA_VSID << SLB_VSID_SHIFT_1T); 3301 /* On POWER8 turn on online bit to enable PURR/SPURR */ 3302 if (cpu_has_feature(CPU_FTR_ARCH_207S)) 3303 lpcr |= LPCR_ONL; 3304 /* 3305 * On POWER9, VPM0 bit is reserved (VPM0=1 behaviour is assumed) 3306 * Set HVICE bit to enable hypervisor virtualization interrupts. 3307 */ 3308 if (cpu_has_feature(CPU_FTR_ARCH_300)) { 3309 lpcr &= ~LPCR_VPM0; 3310 lpcr |= LPCR_HVICE; 3311 } 3312 3313 kvm->arch.lpcr = lpcr; 3314 3315 /* 3316 * Work out how many sets the TLB has, for the use of 3317 * the TLB invalidation loop in book3s_hv_rmhandlers.S. 3318 */ 3319 if (cpu_has_feature(CPU_FTR_ARCH_300)) 3320 kvm->arch.tlb_sets = POWER9_TLB_SETS_HASH; /* 256 */ 3321 else if (cpu_has_feature(CPU_FTR_ARCH_207S)) 3322 kvm->arch.tlb_sets = POWER8_TLB_SETS; /* 512 */ 3323 else 3324 kvm->arch.tlb_sets = POWER7_TLB_SETS; /* 128 */ 3325 3326 /* 3327 * Track that we now have a HV mode VM active. This blocks secondary 3328 * CPU threads from coming online. 3329 */ 3330 kvm_hv_vm_activated(); 3331 3332 /* 3333 * Create a debugfs directory for the VM 3334 */ 3335 snprintf(buf, sizeof(buf), "vm%d", current->pid); 3336 kvm->arch.debugfs_dir = debugfs_create_dir(buf, kvm_debugfs_dir); 3337 if (!IS_ERR_OR_NULL(kvm->arch.debugfs_dir)) 3338 kvmppc_mmu_debugfs_init(kvm); 3339 3340 return 0; 3341 } 3342 3343 static void kvmppc_free_vcores(struct kvm *kvm) 3344 { 3345 long int i; 3346 3347 for (i = 0; i < KVM_MAX_VCORES; ++i) 3348 kfree(kvm->arch.vcores[i]); 3349 kvm->arch.online_vcores = 0; 3350 } 3351 3352 static void kvmppc_core_destroy_vm_hv(struct kvm *kvm) 3353 { 3354 debugfs_remove_recursive(kvm->arch.debugfs_dir); 3355 3356 kvm_hv_vm_deactivated(); 3357 3358 kvmppc_free_vcores(kvm); 3359 3360 kvmppc_free_hpt(kvm); 3361 3362 kvmppc_free_pimap(kvm); 3363 } 3364 3365 /* We don't need to emulate any privileged instructions or dcbz */ 3366 static int kvmppc_core_emulate_op_hv(struct kvm_run *run, struct kvm_vcpu *vcpu, 3367 unsigned int inst, int *advance) 3368 { 3369 return EMULATE_FAIL; 3370 } 3371 3372 static int kvmppc_core_emulate_mtspr_hv(struct kvm_vcpu *vcpu, int sprn, 3373 ulong spr_val) 3374 { 3375 return EMULATE_FAIL; 3376 } 3377 3378 static int kvmppc_core_emulate_mfspr_hv(struct kvm_vcpu *vcpu, int sprn, 3379 ulong *spr_val) 3380 { 3381 return EMULATE_FAIL; 3382 } 3383 3384 static int kvmppc_core_check_processor_compat_hv(void) 3385 { 3386 if (!cpu_has_feature(CPU_FTR_HVMODE) || 3387 !cpu_has_feature(CPU_FTR_ARCH_206)) 3388 return -EIO; 3389 /* 3390 * Disable KVM for Power9 in radix mode. 3391 */ 3392 if (cpu_has_feature(CPU_FTR_ARCH_300) && radix_enabled()) 3393 return -EIO; 3394 3395 return 0; 3396 } 3397 3398 #ifdef CONFIG_KVM_XICS 3399 3400 void kvmppc_free_pimap(struct kvm *kvm) 3401 { 3402 kfree(kvm->arch.pimap); 3403 } 3404 3405 static struct kvmppc_passthru_irqmap *kvmppc_alloc_pimap(void) 3406 { 3407 return kzalloc(sizeof(struct kvmppc_passthru_irqmap), GFP_KERNEL); 3408 } 3409 3410 static int kvmppc_set_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi) 3411 { 3412 struct irq_desc *desc; 3413 struct kvmppc_irq_map *irq_map; 3414 struct kvmppc_passthru_irqmap *pimap; 3415 struct irq_chip *chip; 3416 int i; 3417 3418 if (!kvm_irq_bypass) 3419 return 1; 3420 3421 desc = irq_to_desc(host_irq); 3422 if (!desc) 3423 return -EIO; 3424 3425 mutex_lock(&kvm->lock); 3426 3427 pimap = kvm->arch.pimap; 3428 if (pimap == NULL) { 3429 /* First call, allocate structure to hold IRQ map */ 3430 pimap = kvmppc_alloc_pimap(); 3431 if (pimap == NULL) { 3432 mutex_unlock(&kvm->lock); 3433 return -ENOMEM; 3434 } 3435 kvm->arch.pimap = pimap; 3436 } 3437 3438 /* 3439 * For now, we only support interrupts for which the EOI operation 3440 * is an OPAL call followed by a write to XIRR, since that's 3441 * what our real-mode EOI code does. 3442 */ 3443 chip = irq_data_get_irq_chip(&desc->irq_data); 3444 if (!chip || !is_pnv_opal_msi(chip)) { 3445 pr_warn("kvmppc_set_passthru_irq_hv: Could not assign IRQ map for (%d,%d)\n", 3446 host_irq, guest_gsi); 3447 mutex_unlock(&kvm->lock); 3448 return -ENOENT; 3449 } 3450 3451 /* 3452 * See if we already have an entry for this guest IRQ number. 3453 * If it's mapped to a hardware IRQ number, that's an error, 3454 * otherwise re-use this entry. 3455 */ 3456 for (i = 0; i < pimap->n_mapped; i++) { 3457 if (guest_gsi == pimap->mapped[i].v_hwirq) { 3458 if (pimap->mapped[i].r_hwirq) { 3459 mutex_unlock(&kvm->lock); 3460 return -EINVAL; 3461 } 3462 break; 3463 } 3464 } 3465 3466 if (i == KVMPPC_PIRQ_MAPPED) { 3467 mutex_unlock(&kvm->lock); 3468 return -EAGAIN; /* table is full */ 3469 } 3470 3471 irq_map = &pimap->mapped[i]; 3472 3473 irq_map->v_hwirq = guest_gsi; 3474 irq_map->desc = desc; 3475 3476 /* 3477 * Order the above two stores before the next to serialize with 3478 * the KVM real mode handler. 3479 */ 3480 smp_wmb(); 3481 irq_map->r_hwirq = desc->irq_data.hwirq; 3482 3483 if (i == pimap->n_mapped) 3484 pimap->n_mapped++; 3485 3486 kvmppc_xics_set_mapped(kvm, guest_gsi, desc->irq_data.hwirq); 3487 3488 mutex_unlock(&kvm->lock); 3489 3490 return 0; 3491 } 3492 3493 static int kvmppc_clr_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi) 3494 { 3495 struct irq_desc *desc; 3496 struct kvmppc_passthru_irqmap *pimap; 3497 int i; 3498 3499 if (!kvm_irq_bypass) 3500 return 0; 3501 3502 desc = irq_to_desc(host_irq); 3503 if (!desc) 3504 return -EIO; 3505 3506 mutex_lock(&kvm->lock); 3507 3508 if (kvm->arch.pimap == NULL) { 3509 mutex_unlock(&kvm->lock); 3510 return 0; 3511 } 3512 pimap = kvm->arch.pimap; 3513 3514 for (i = 0; i < pimap->n_mapped; i++) { 3515 if (guest_gsi == pimap->mapped[i].v_hwirq) 3516 break; 3517 } 3518 3519 if (i == pimap->n_mapped) { 3520 mutex_unlock(&kvm->lock); 3521 return -ENODEV; 3522 } 3523 3524 kvmppc_xics_clr_mapped(kvm, guest_gsi, pimap->mapped[i].r_hwirq); 3525 3526 /* invalidate the entry */ 3527 pimap->mapped[i].r_hwirq = 0; 3528 3529 /* 3530 * We don't free this structure even when the count goes to 3531 * zero. The structure is freed when we destroy the VM. 3532 */ 3533 3534 mutex_unlock(&kvm->lock); 3535 return 0; 3536 } 3537 3538 static int kvmppc_irq_bypass_add_producer_hv(struct irq_bypass_consumer *cons, 3539 struct irq_bypass_producer *prod) 3540 { 3541 int ret = 0; 3542 struct kvm_kernel_irqfd *irqfd = 3543 container_of(cons, struct kvm_kernel_irqfd, consumer); 3544 3545 irqfd->producer = prod; 3546 3547 ret = kvmppc_set_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi); 3548 if (ret) 3549 pr_info("kvmppc_set_passthru_irq (irq %d, gsi %d) fails: %d\n", 3550 prod->irq, irqfd->gsi, ret); 3551 3552 return ret; 3553 } 3554 3555 static void kvmppc_irq_bypass_del_producer_hv(struct irq_bypass_consumer *cons, 3556 struct irq_bypass_producer *prod) 3557 { 3558 int ret; 3559 struct kvm_kernel_irqfd *irqfd = 3560 container_of(cons, struct kvm_kernel_irqfd, consumer); 3561 3562 irqfd->producer = NULL; 3563 3564 /* 3565 * When producer of consumer is unregistered, we change back to 3566 * default external interrupt handling mode - KVM real mode 3567 * will switch back to host. 3568 */ 3569 ret = kvmppc_clr_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi); 3570 if (ret) 3571 pr_warn("kvmppc_clr_passthru_irq (irq %d, gsi %d) fails: %d\n", 3572 prod->irq, irqfd->gsi, ret); 3573 } 3574 #endif 3575 3576 static long kvm_arch_vm_ioctl_hv(struct file *filp, 3577 unsigned int ioctl, unsigned long arg) 3578 { 3579 struct kvm *kvm __maybe_unused = filp->private_data; 3580 void __user *argp = (void __user *)arg; 3581 long r; 3582 3583 switch (ioctl) { 3584 3585 case KVM_PPC_ALLOCATE_HTAB: { 3586 u32 htab_order; 3587 3588 r = -EFAULT; 3589 if (get_user(htab_order, (u32 __user *)argp)) 3590 break; 3591 r = kvmppc_alloc_reset_hpt(kvm, &htab_order); 3592 if (r) 3593 break; 3594 r = -EFAULT; 3595 if (put_user(htab_order, (u32 __user *)argp)) 3596 break; 3597 r = 0; 3598 break; 3599 } 3600 3601 case KVM_PPC_GET_HTAB_FD: { 3602 struct kvm_get_htab_fd ghf; 3603 3604 r = -EFAULT; 3605 if (copy_from_user(&ghf, argp, sizeof(ghf))) 3606 break; 3607 r = kvm_vm_ioctl_get_htab_fd(kvm, &ghf); 3608 break; 3609 } 3610 3611 default: 3612 r = -ENOTTY; 3613 } 3614 3615 return r; 3616 } 3617 3618 /* 3619 * List of hcall numbers to enable by default. 3620 * For compatibility with old userspace, we enable by default 3621 * all hcalls that were implemented before the hcall-enabling 3622 * facility was added. Note this list should not include H_RTAS. 3623 */ 3624 static unsigned int default_hcall_list[] = { 3625 H_REMOVE, 3626 H_ENTER, 3627 H_READ, 3628 H_PROTECT, 3629 H_BULK_REMOVE, 3630 H_GET_TCE, 3631 H_PUT_TCE, 3632 H_SET_DABR, 3633 H_SET_XDABR, 3634 H_CEDE, 3635 H_PROD, 3636 H_CONFER, 3637 H_REGISTER_VPA, 3638 #ifdef CONFIG_KVM_XICS 3639 H_EOI, 3640 H_CPPR, 3641 H_IPI, 3642 H_IPOLL, 3643 H_XIRR, 3644 H_XIRR_X, 3645 #endif 3646 0 3647 }; 3648 3649 static void init_default_hcalls(void) 3650 { 3651 int i; 3652 unsigned int hcall; 3653 3654 for (i = 0; default_hcall_list[i]; ++i) { 3655 hcall = default_hcall_list[i]; 3656 WARN_ON(!kvmppc_hcall_impl_hv(hcall)); 3657 __set_bit(hcall / 4, default_enabled_hcalls); 3658 } 3659 } 3660 3661 static struct kvmppc_ops kvm_ops_hv = { 3662 .get_sregs = kvm_arch_vcpu_ioctl_get_sregs_hv, 3663 .set_sregs = kvm_arch_vcpu_ioctl_set_sregs_hv, 3664 .get_one_reg = kvmppc_get_one_reg_hv, 3665 .set_one_reg = kvmppc_set_one_reg_hv, 3666 .vcpu_load = kvmppc_core_vcpu_load_hv, 3667 .vcpu_put = kvmppc_core_vcpu_put_hv, 3668 .set_msr = kvmppc_set_msr_hv, 3669 .vcpu_run = kvmppc_vcpu_run_hv, 3670 .vcpu_create = kvmppc_core_vcpu_create_hv, 3671 .vcpu_free = kvmppc_core_vcpu_free_hv, 3672 .check_requests = kvmppc_core_check_requests_hv, 3673 .get_dirty_log = kvm_vm_ioctl_get_dirty_log_hv, 3674 .flush_memslot = kvmppc_core_flush_memslot_hv, 3675 .prepare_memory_region = kvmppc_core_prepare_memory_region_hv, 3676 .commit_memory_region = kvmppc_core_commit_memory_region_hv, 3677 .unmap_hva = kvm_unmap_hva_hv, 3678 .unmap_hva_range = kvm_unmap_hva_range_hv, 3679 .age_hva = kvm_age_hva_hv, 3680 .test_age_hva = kvm_test_age_hva_hv, 3681 .set_spte_hva = kvm_set_spte_hva_hv, 3682 .mmu_destroy = kvmppc_mmu_destroy_hv, 3683 .free_memslot = kvmppc_core_free_memslot_hv, 3684 .create_memslot = kvmppc_core_create_memslot_hv, 3685 .init_vm = kvmppc_core_init_vm_hv, 3686 .destroy_vm = kvmppc_core_destroy_vm_hv, 3687 .get_smmu_info = kvm_vm_ioctl_get_smmu_info_hv, 3688 .emulate_op = kvmppc_core_emulate_op_hv, 3689 .emulate_mtspr = kvmppc_core_emulate_mtspr_hv, 3690 .emulate_mfspr = kvmppc_core_emulate_mfspr_hv, 3691 .fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv, 3692 .arch_vm_ioctl = kvm_arch_vm_ioctl_hv, 3693 .hcall_implemented = kvmppc_hcall_impl_hv, 3694 #ifdef CONFIG_KVM_XICS 3695 .irq_bypass_add_producer = kvmppc_irq_bypass_add_producer_hv, 3696 .irq_bypass_del_producer = kvmppc_irq_bypass_del_producer_hv, 3697 #endif 3698 }; 3699 3700 static int kvm_init_subcore_bitmap(void) 3701 { 3702 int i, j; 3703 int nr_cores = cpu_nr_cores(); 3704 struct sibling_subcore_state *sibling_subcore_state; 3705 3706 for (i = 0; i < nr_cores; i++) { 3707 int first_cpu = i * threads_per_core; 3708 int node = cpu_to_node(first_cpu); 3709 3710 /* Ignore if it is already allocated. */ 3711 if (paca[first_cpu].sibling_subcore_state) 3712 continue; 3713 3714 sibling_subcore_state = 3715 kmalloc_node(sizeof(struct sibling_subcore_state), 3716 GFP_KERNEL, node); 3717 if (!sibling_subcore_state) 3718 return -ENOMEM; 3719 3720 memset(sibling_subcore_state, 0, 3721 sizeof(struct sibling_subcore_state)); 3722 3723 for (j = 0; j < threads_per_core; j++) { 3724 int cpu = first_cpu + j; 3725 3726 paca[cpu].sibling_subcore_state = sibling_subcore_state; 3727 } 3728 } 3729 return 0; 3730 } 3731 3732 static int kvmppc_book3s_init_hv(void) 3733 { 3734 int r; 3735 /* 3736 * FIXME!! Do we need to check on all cpus ? 3737 */ 3738 r = kvmppc_core_check_processor_compat_hv(); 3739 if (r < 0) 3740 return -ENODEV; 3741 3742 r = kvm_init_subcore_bitmap(); 3743 if (r) 3744 return r; 3745 3746 /* 3747 * We need a way of accessing the XICS interrupt controller, 3748 * either directly, via paca[cpu].kvm_hstate.xics_phys, or 3749 * indirectly, via OPAL. 3750 */ 3751 #ifdef CONFIG_SMP 3752 if (!get_paca()->kvm_hstate.xics_phys) { 3753 struct device_node *np; 3754 3755 np = of_find_compatible_node(NULL, NULL, "ibm,opal-intc"); 3756 if (!np) { 3757 pr_err("KVM-HV: Cannot determine method for accessing XICS\n"); 3758 return -ENODEV; 3759 } 3760 } 3761 #endif 3762 3763 kvm_ops_hv.owner = THIS_MODULE; 3764 kvmppc_hv_ops = &kvm_ops_hv; 3765 3766 init_default_hcalls(); 3767 3768 init_vcore_lists(); 3769 3770 r = kvmppc_mmu_hv_init(); 3771 return r; 3772 } 3773 3774 static void kvmppc_book3s_exit_hv(void) 3775 { 3776 kvmppc_free_host_rm_ops(); 3777 kvmppc_hv_ops = NULL; 3778 } 3779 3780 module_init(kvmppc_book3s_init_hv); 3781 module_exit(kvmppc_book3s_exit_hv); 3782 MODULE_LICENSE("GPL"); 3783 MODULE_ALIAS_MISCDEV(KVM_MINOR); 3784 MODULE_ALIAS("devname:kvm"); 3785 3786