1 /* 2 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved. 3 * 4 * Authors: 5 * Alexander Graf <agraf@suse.de> 6 * Kevin Wolf <mail@kevin-wolf.de> 7 * Paul Mackerras <paulus@samba.org> 8 * 9 * Description: 10 * Functions relating to running KVM on Book 3S processors where 11 * we don't have access to hypervisor mode, and we run the guest 12 * in problem state (user mode). 13 * 14 * This file is derived from arch/powerpc/kvm/44x.c, 15 * by Hollis Blanchard <hollisb@us.ibm.com>. 16 * 17 * This program is free software; you can redistribute it and/or modify 18 * it under the terms of the GNU General Public License, version 2, as 19 * published by the Free Software Foundation. 20 */ 21 22 #include <linux/kvm_host.h> 23 #include <linux/export.h> 24 #include <linux/err.h> 25 #include <linux/slab.h> 26 27 #include <asm/reg.h> 28 #include <asm/cputable.h> 29 #include <asm/cacheflush.h> 30 #include <asm/tlbflush.h> 31 #include <linux/uaccess.h> 32 #include <asm/io.h> 33 #include <asm/kvm_ppc.h> 34 #include <asm/kvm_book3s.h> 35 #include <asm/mmu_context.h> 36 #include <asm/switch_to.h> 37 #include <asm/firmware.h> 38 #include <asm/setup.h> 39 #include <linux/gfp.h> 40 #include <linux/sched.h> 41 #include <linux/vmalloc.h> 42 #include <linux/highmem.h> 43 #include <linux/module.h> 44 #include <linux/miscdevice.h> 45 46 #include "book3s.h" 47 48 #define CREATE_TRACE_POINTS 49 #include "trace_pr.h" 50 51 /* #define EXIT_DEBUG */ 52 /* #define DEBUG_EXT */ 53 54 static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr, 55 ulong msr); 56 static void kvmppc_giveup_fac(struct kvm_vcpu *vcpu, ulong fac); 57 58 /* Some compatibility defines */ 59 #ifdef CONFIG_PPC_BOOK3S_32 60 #define MSR_USER32 MSR_USER 61 #define MSR_USER64 MSR_USER 62 #define HW_PAGE_SIZE PAGE_SIZE 63 #define HPTE_R_M _PAGE_COHERENT 64 #endif 65 66 static bool kvmppc_is_split_real(struct kvm_vcpu *vcpu) 67 { 68 ulong msr = kvmppc_get_msr(vcpu); 69 return (msr & (MSR_IR|MSR_DR)) == MSR_DR; 70 } 71 72 static void kvmppc_fixup_split_real(struct kvm_vcpu *vcpu) 73 { 74 ulong msr = kvmppc_get_msr(vcpu); 75 ulong pc = kvmppc_get_pc(vcpu); 76 77 /* We are in DR only split real mode */ 78 if ((msr & (MSR_IR|MSR_DR)) != MSR_DR) 79 return; 80 81 /* We have not fixed up the guest already */ 82 if (vcpu->arch.hflags & BOOK3S_HFLAG_SPLIT_HACK) 83 return; 84 85 /* The code is in fixupable address space */ 86 if (pc & SPLIT_HACK_MASK) 87 return; 88 89 vcpu->arch.hflags |= BOOK3S_HFLAG_SPLIT_HACK; 90 kvmppc_set_pc(vcpu, pc | SPLIT_HACK_OFFS); 91 } 92 93 void kvmppc_unfixup_split_real(struct kvm_vcpu *vcpu); 94 95 static void kvmppc_core_vcpu_load_pr(struct kvm_vcpu *vcpu, int cpu) 96 { 97 #ifdef CONFIG_PPC_BOOK3S_64 98 struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); 99 memcpy(svcpu->slb, to_book3s(vcpu)->slb_shadow, sizeof(svcpu->slb)); 100 svcpu->slb_max = to_book3s(vcpu)->slb_shadow_max; 101 svcpu->in_use = 0; 102 svcpu_put(svcpu); 103 #endif 104 105 /* Disable AIL if supported */ 106 if (cpu_has_feature(CPU_FTR_HVMODE) && 107 cpu_has_feature(CPU_FTR_ARCH_207S)) 108 mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) & ~LPCR_AIL); 109 110 vcpu->cpu = smp_processor_id(); 111 #ifdef CONFIG_PPC_BOOK3S_32 112 current->thread.kvm_shadow_vcpu = vcpu->arch.shadow_vcpu; 113 #endif 114 115 if (kvmppc_is_split_real(vcpu)) 116 kvmppc_fixup_split_real(vcpu); 117 } 118 119 static void kvmppc_core_vcpu_put_pr(struct kvm_vcpu *vcpu) 120 { 121 #ifdef CONFIG_PPC_BOOK3S_64 122 struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); 123 if (svcpu->in_use) { 124 kvmppc_copy_from_svcpu(vcpu); 125 } 126 memcpy(to_book3s(vcpu)->slb_shadow, svcpu->slb, sizeof(svcpu->slb)); 127 to_book3s(vcpu)->slb_shadow_max = svcpu->slb_max; 128 svcpu_put(svcpu); 129 #endif 130 131 if (kvmppc_is_split_real(vcpu)) 132 kvmppc_unfixup_split_real(vcpu); 133 134 kvmppc_giveup_ext(vcpu, MSR_FP | MSR_VEC | MSR_VSX); 135 kvmppc_giveup_fac(vcpu, FSCR_TAR_LG); 136 137 /* Enable AIL if supported */ 138 if (cpu_has_feature(CPU_FTR_HVMODE) && 139 cpu_has_feature(CPU_FTR_ARCH_207S)) 140 mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) | LPCR_AIL_3); 141 142 vcpu->cpu = -1; 143 } 144 145 /* Copy data needed by real-mode code from vcpu to shadow vcpu */ 146 void kvmppc_copy_to_svcpu(struct kvm_vcpu *vcpu) 147 { 148 struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); 149 150 svcpu->gpr[0] = vcpu->arch.gpr[0]; 151 svcpu->gpr[1] = vcpu->arch.gpr[1]; 152 svcpu->gpr[2] = vcpu->arch.gpr[2]; 153 svcpu->gpr[3] = vcpu->arch.gpr[3]; 154 svcpu->gpr[4] = vcpu->arch.gpr[4]; 155 svcpu->gpr[5] = vcpu->arch.gpr[5]; 156 svcpu->gpr[6] = vcpu->arch.gpr[6]; 157 svcpu->gpr[7] = vcpu->arch.gpr[7]; 158 svcpu->gpr[8] = vcpu->arch.gpr[8]; 159 svcpu->gpr[9] = vcpu->arch.gpr[9]; 160 svcpu->gpr[10] = vcpu->arch.gpr[10]; 161 svcpu->gpr[11] = vcpu->arch.gpr[11]; 162 svcpu->gpr[12] = vcpu->arch.gpr[12]; 163 svcpu->gpr[13] = vcpu->arch.gpr[13]; 164 svcpu->cr = vcpu->arch.cr; 165 svcpu->xer = vcpu->arch.xer; 166 svcpu->ctr = vcpu->arch.ctr; 167 svcpu->lr = vcpu->arch.lr; 168 svcpu->pc = vcpu->arch.pc; 169 #ifdef CONFIG_PPC_BOOK3S_64 170 svcpu->shadow_fscr = vcpu->arch.shadow_fscr; 171 #endif 172 /* 173 * Now also save the current time base value. We use this 174 * to find the guest purr and spurr value. 175 */ 176 vcpu->arch.entry_tb = get_tb(); 177 vcpu->arch.entry_vtb = get_vtb(); 178 if (cpu_has_feature(CPU_FTR_ARCH_207S)) 179 vcpu->arch.entry_ic = mfspr(SPRN_IC); 180 svcpu->in_use = true; 181 182 svcpu_put(svcpu); 183 } 184 185 /* Copy data touched by real-mode code from shadow vcpu back to vcpu */ 186 void kvmppc_copy_from_svcpu(struct kvm_vcpu *vcpu) 187 { 188 struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); 189 190 /* 191 * Maybe we were already preempted and synced the svcpu from 192 * our preempt notifiers. Don't bother touching this svcpu then. 193 */ 194 if (!svcpu->in_use) 195 goto out; 196 197 vcpu->arch.gpr[0] = svcpu->gpr[0]; 198 vcpu->arch.gpr[1] = svcpu->gpr[1]; 199 vcpu->arch.gpr[2] = svcpu->gpr[2]; 200 vcpu->arch.gpr[3] = svcpu->gpr[3]; 201 vcpu->arch.gpr[4] = svcpu->gpr[4]; 202 vcpu->arch.gpr[5] = svcpu->gpr[5]; 203 vcpu->arch.gpr[6] = svcpu->gpr[6]; 204 vcpu->arch.gpr[7] = svcpu->gpr[7]; 205 vcpu->arch.gpr[8] = svcpu->gpr[8]; 206 vcpu->arch.gpr[9] = svcpu->gpr[9]; 207 vcpu->arch.gpr[10] = svcpu->gpr[10]; 208 vcpu->arch.gpr[11] = svcpu->gpr[11]; 209 vcpu->arch.gpr[12] = svcpu->gpr[12]; 210 vcpu->arch.gpr[13] = svcpu->gpr[13]; 211 vcpu->arch.cr = svcpu->cr; 212 vcpu->arch.xer = svcpu->xer; 213 vcpu->arch.ctr = svcpu->ctr; 214 vcpu->arch.lr = svcpu->lr; 215 vcpu->arch.pc = svcpu->pc; 216 vcpu->arch.shadow_srr1 = svcpu->shadow_srr1; 217 vcpu->arch.fault_dar = svcpu->fault_dar; 218 vcpu->arch.fault_dsisr = svcpu->fault_dsisr; 219 vcpu->arch.last_inst = svcpu->last_inst; 220 #ifdef CONFIG_PPC_BOOK3S_64 221 vcpu->arch.shadow_fscr = svcpu->shadow_fscr; 222 #endif 223 /* 224 * Update purr and spurr using time base on exit. 225 */ 226 vcpu->arch.purr += get_tb() - vcpu->arch.entry_tb; 227 vcpu->arch.spurr += get_tb() - vcpu->arch.entry_tb; 228 to_book3s(vcpu)->vtb += get_vtb() - vcpu->arch.entry_vtb; 229 if (cpu_has_feature(CPU_FTR_ARCH_207S)) 230 vcpu->arch.ic += mfspr(SPRN_IC) - vcpu->arch.entry_ic; 231 svcpu->in_use = false; 232 233 out: 234 svcpu_put(svcpu); 235 } 236 237 static int kvmppc_core_check_requests_pr(struct kvm_vcpu *vcpu) 238 { 239 int r = 1; /* Indicate we want to get back into the guest */ 240 241 /* We misuse TLB_FLUSH to indicate that we want to clear 242 all shadow cache entries */ 243 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) 244 kvmppc_mmu_pte_flush(vcpu, 0, 0); 245 246 return r; 247 } 248 249 /************* MMU Notifiers *************/ 250 static void do_kvm_unmap_hva(struct kvm *kvm, unsigned long start, 251 unsigned long end) 252 { 253 long i; 254 struct kvm_vcpu *vcpu; 255 struct kvm_memslots *slots; 256 struct kvm_memory_slot *memslot; 257 258 slots = kvm_memslots(kvm); 259 kvm_for_each_memslot(memslot, slots) { 260 unsigned long hva_start, hva_end; 261 gfn_t gfn, gfn_end; 262 263 hva_start = max(start, memslot->userspace_addr); 264 hva_end = min(end, memslot->userspace_addr + 265 (memslot->npages << PAGE_SHIFT)); 266 if (hva_start >= hva_end) 267 continue; 268 /* 269 * {gfn(page) | page intersects with [hva_start, hva_end)} = 270 * {gfn, gfn+1, ..., gfn_end-1}. 271 */ 272 gfn = hva_to_gfn_memslot(hva_start, memslot); 273 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot); 274 kvm_for_each_vcpu(i, vcpu, kvm) 275 kvmppc_mmu_pte_pflush(vcpu, gfn << PAGE_SHIFT, 276 gfn_end << PAGE_SHIFT); 277 } 278 } 279 280 static int kvm_unmap_hva_range_pr(struct kvm *kvm, unsigned long start, 281 unsigned long end) 282 { 283 do_kvm_unmap_hva(kvm, start, end); 284 285 return 0; 286 } 287 288 static int kvm_age_hva_pr(struct kvm *kvm, unsigned long start, 289 unsigned long end) 290 { 291 /* XXX could be more clever ;) */ 292 return 0; 293 } 294 295 static int kvm_test_age_hva_pr(struct kvm *kvm, unsigned long hva) 296 { 297 /* XXX could be more clever ;) */ 298 return 0; 299 } 300 301 static void kvm_set_spte_hva_pr(struct kvm *kvm, unsigned long hva, pte_t pte) 302 { 303 /* The page will get remapped properly on its next fault */ 304 do_kvm_unmap_hva(kvm, hva, hva + PAGE_SIZE); 305 } 306 307 /*****************************************/ 308 309 static void kvmppc_recalc_shadow_msr(struct kvm_vcpu *vcpu) 310 { 311 ulong guest_msr = kvmppc_get_msr(vcpu); 312 ulong smsr = guest_msr; 313 314 /* Guest MSR values */ 315 smsr &= MSR_FE0 | MSR_FE1 | MSR_SF | MSR_SE | MSR_BE | MSR_LE; 316 /* Process MSR values */ 317 smsr |= MSR_ME | MSR_RI | MSR_IR | MSR_DR | MSR_PR | MSR_EE; 318 /* External providers the guest reserved */ 319 smsr |= (guest_msr & vcpu->arch.guest_owned_ext); 320 /* 64-bit Process MSR values */ 321 #ifdef CONFIG_PPC_BOOK3S_64 322 smsr |= MSR_ISF | MSR_HV; 323 #endif 324 vcpu->arch.shadow_msr = smsr; 325 } 326 327 static void kvmppc_set_msr_pr(struct kvm_vcpu *vcpu, u64 msr) 328 { 329 ulong old_msr = kvmppc_get_msr(vcpu); 330 331 #ifdef EXIT_DEBUG 332 printk(KERN_INFO "KVM: Set MSR to 0x%llx\n", msr); 333 #endif 334 335 msr &= to_book3s(vcpu)->msr_mask; 336 kvmppc_set_msr_fast(vcpu, msr); 337 kvmppc_recalc_shadow_msr(vcpu); 338 339 if (msr & MSR_POW) { 340 if (!vcpu->arch.pending_exceptions) { 341 kvm_vcpu_block(vcpu); 342 kvm_clear_request(KVM_REQ_UNHALT, vcpu); 343 vcpu->stat.halt_wakeup++; 344 345 /* Unset POW bit after we woke up */ 346 msr &= ~MSR_POW; 347 kvmppc_set_msr_fast(vcpu, msr); 348 } 349 } 350 351 if (kvmppc_is_split_real(vcpu)) 352 kvmppc_fixup_split_real(vcpu); 353 else 354 kvmppc_unfixup_split_real(vcpu); 355 356 if ((kvmppc_get_msr(vcpu) & (MSR_PR|MSR_IR|MSR_DR)) != 357 (old_msr & (MSR_PR|MSR_IR|MSR_DR))) { 358 kvmppc_mmu_flush_segments(vcpu); 359 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)); 360 361 /* Preload magic page segment when in kernel mode */ 362 if (!(msr & MSR_PR) && vcpu->arch.magic_page_pa) { 363 struct kvm_vcpu_arch *a = &vcpu->arch; 364 365 if (msr & MSR_DR) 366 kvmppc_mmu_map_segment(vcpu, a->magic_page_ea); 367 else 368 kvmppc_mmu_map_segment(vcpu, a->magic_page_pa); 369 } 370 } 371 372 /* 373 * When switching from 32 to 64-bit, we may have a stale 32-bit 374 * magic page around, we need to flush it. Typically 32-bit magic 375 * page will be instanciated when calling into RTAS. Note: We 376 * assume that such transition only happens while in kernel mode, 377 * ie, we never transition from user 32-bit to kernel 64-bit with 378 * a 32-bit magic page around. 379 */ 380 if (vcpu->arch.magic_page_pa && 381 !(old_msr & MSR_PR) && !(old_msr & MSR_SF) && (msr & MSR_SF)) { 382 /* going from RTAS to normal kernel code */ 383 kvmppc_mmu_pte_flush(vcpu, (uint32_t)vcpu->arch.magic_page_pa, 384 ~0xFFFUL); 385 } 386 387 /* Preload FPU if it's enabled */ 388 if (kvmppc_get_msr(vcpu) & MSR_FP) 389 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP); 390 } 391 392 void kvmppc_set_pvr_pr(struct kvm_vcpu *vcpu, u32 pvr) 393 { 394 u32 host_pvr; 395 396 vcpu->arch.hflags &= ~BOOK3S_HFLAG_SLB; 397 vcpu->arch.pvr = pvr; 398 #ifdef CONFIG_PPC_BOOK3S_64 399 if ((pvr >= 0x330000) && (pvr < 0x70330000)) { 400 kvmppc_mmu_book3s_64_init(vcpu); 401 if (!to_book3s(vcpu)->hior_explicit) 402 to_book3s(vcpu)->hior = 0xfff00000; 403 to_book3s(vcpu)->msr_mask = 0xffffffffffffffffULL; 404 vcpu->arch.cpu_type = KVM_CPU_3S_64; 405 } else 406 #endif 407 { 408 kvmppc_mmu_book3s_32_init(vcpu); 409 if (!to_book3s(vcpu)->hior_explicit) 410 to_book3s(vcpu)->hior = 0; 411 to_book3s(vcpu)->msr_mask = 0xffffffffULL; 412 vcpu->arch.cpu_type = KVM_CPU_3S_32; 413 } 414 415 kvmppc_sanity_check(vcpu); 416 417 /* If we are in hypervisor level on 970, we can tell the CPU to 418 * treat DCBZ as 32 bytes store */ 419 vcpu->arch.hflags &= ~BOOK3S_HFLAG_DCBZ32; 420 if (vcpu->arch.mmu.is_dcbz32(vcpu) && (mfmsr() & MSR_HV) && 421 !strcmp(cur_cpu_spec->platform, "ppc970")) 422 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32; 423 424 /* Cell performs badly if MSR_FEx are set. So let's hope nobody 425 really needs them in a VM on Cell and force disable them. */ 426 if (!strcmp(cur_cpu_spec->platform, "ppc-cell-be")) 427 to_book3s(vcpu)->msr_mask &= ~(MSR_FE0 | MSR_FE1); 428 429 /* 430 * If they're asking for POWER6 or later, set the flag 431 * indicating that we can do multiple large page sizes 432 * and 1TB segments. 433 * Also set the flag that indicates that tlbie has the large 434 * page bit in the RB operand instead of the instruction. 435 */ 436 switch (PVR_VER(pvr)) { 437 case PVR_POWER6: 438 case PVR_POWER7: 439 case PVR_POWER7p: 440 case PVR_POWER8: 441 case PVR_POWER8E: 442 case PVR_POWER8NVL: 443 vcpu->arch.hflags |= BOOK3S_HFLAG_MULTI_PGSIZE | 444 BOOK3S_HFLAG_NEW_TLBIE; 445 break; 446 } 447 448 #ifdef CONFIG_PPC_BOOK3S_32 449 /* 32 bit Book3S always has 32 byte dcbz */ 450 vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32; 451 #endif 452 453 /* On some CPUs we can execute paired single operations natively */ 454 asm ( "mfpvr %0" : "=r"(host_pvr)); 455 switch (host_pvr) { 456 case 0x00080200: /* lonestar 2.0 */ 457 case 0x00088202: /* lonestar 2.2 */ 458 case 0x70000100: /* gekko 1.0 */ 459 case 0x00080100: /* gekko 2.0 */ 460 case 0x00083203: /* gekko 2.3a */ 461 case 0x00083213: /* gekko 2.3b */ 462 case 0x00083204: /* gekko 2.4 */ 463 case 0x00083214: /* gekko 2.4e (8SE) - retail HW2 */ 464 case 0x00087200: /* broadway */ 465 vcpu->arch.hflags |= BOOK3S_HFLAG_NATIVE_PS; 466 /* Enable HID2.PSE - in case we need it later */ 467 mtspr(SPRN_HID2_GEKKO, mfspr(SPRN_HID2_GEKKO) | (1 << 29)); 468 } 469 } 470 471 /* Book3s_32 CPUs always have 32 bytes cache line size, which Linux assumes. To 472 * make Book3s_32 Linux work on Book3s_64, we have to make sure we trap dcbz to 473 * emulate 32 bytes dcbz length. 474 * 475 * The Book3s_64 inventors also realized this case and implemented a special bit 476 * in the HID5 register, which is a hypervisor ressource. Thus we can't use it. 477 * 478 * My approach here is to patch the dcbz instruction on executing pages. 479 */ 480 static void kvmppc_patch_dcbz(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte) 481 { 482 struct page *hpage; 483 u64 hpage_offset; 484 u32 *page; 485 int i; 486 487 hpage = gfn_to_page(vcpu->kvm, pte->raddr >> PAGE_SHIFT); 488 if (is_error_page(hpage)) 489 return; 490 491 hpage_offset = pte->raddr & ~PAGE_MASK; 492 hpage_offset &= ~0xFFFULL; 493 hpage_offset /= 4; 494 495 get_page(hpage); 496 page = kmap_atomic(hpage); 497 498 /* patch dcbz into reserved instruction, so we trap */ 499 for (i=hpage_offset; i < hpage_offset + (HW_PAGE_SIZE / 4); i++) 500 if ((be32_to_cpu(page[i]) & 0xff0007ff) == INS_DCBZ) 501 page[i] &= cpu_to_be32(0xfffffff7); 502 503 kunmap_atomic(page); 504 put_page(hpage); 505 } 506 507 static bool kvmppc_visible_gpa(struct kvm_vcpu *vcpu, gpa_t gpa) 508 { 509 ulong mp_pa = vcpu->arch.magic_page_pa; 510 511 if (!(kvmppc_get_msr(vcpu) & MSR_SF)) 512 mp_pa = (uint32_t)mp_pa; 513 514 gpa &= ~0xFFFULL; 515 if (unlikely(mp_pa) && unlikely((mp_pa & KVM_PAM) == (gpa & KVM_PAM))) { 516 return true; 517 } 518 519 return kvm_is_visible_gfn(vcpu->kvm, gpa >> PAGE_SHIFT); 520 } 521 522 int kvmppc_handle_pagefault(struct kvm_run *run, struct kvm_vcpu *vcpu, 523 ulong eaddr, int vec) 524 { 525 bool data = (vec == BOOK3S_INTERRUPT_DATA_STORAGE); 526 bool iswrite = false; 527 int r = RESUME_GUEST; 528 int relocated; 529 int page_found = 0; 530 struct kvmppc_pte pte = { 0 }; 531 bool dr = (kvmppc_get_msr(vcpu) & MSR_DR) ? true : false; 532 bool ir = (kvmppc_get_msr(vcpu) & MSR_IR) ? true : false; 533 u64 vsid; 534 535 relocated = data ? dr : ir; 536 if (data && (vcpu->arch.fault_dsisr & DSISR_ISSTORE)) 537 iswrite = true; 538 539 /* Resolve real address if translation turned on */ 540 if (relocated) { 541 page_found = vcpu->arch.mmu.xlate(vcpu, eaddr, &pte, data, iswrite); 542 } else { 543 pte.may_execute = true; 544 pte.may_read = true; 545 pte.may_write = true; 546 pte.raddr = eaddr & KVM_PAM; 547 pte.eaddr = eaddr; 548 pte.vpage = eaddr >> 12; 549 pte.page_size = MMU_PAGE_64K; 550 pte.wimg = HPTE_R_M; 551 } 552 553 switch (kvmppc_get_msr(vcpu) & (MSR_DR|MSR_IR)) { 554 case 0: 555 pte.vpage |= ((u64)VSID_REAL << (SID_SHIFT - 12)); 556 break; 557 case MSR_DR: 558 if (!data && 559 (vcpu->arch.hflags & BOOK3S_HFLAG_SPLIT_HACK) && 560 ((pte.raddr & SPLIT_HACK_MASK) == SPLIT_HACK_OFFS)) 561 pte.raddr &= ~SPLIT_HACK_MASK; 562 /* fall through */ 563 case MSR_IR: 564 vcpu->arch.mmu.esid_to_vsid(vcpu, eaddr >> SID_SHIFT, &vsid); 565 566 if ((kvmppc_get_msr(vcpu) & (MSR_DR|MSR_IR)) == MSR_DR) 567 pte.vpage |= ((u64)VSID_REAL_DR << (SID_SHIFT - 12)); 568 else 569 pte.vpage |= ((u64)VSID_REAL_IR << (SID_SHIFT - 12)); 570 pte.vpage |= vsid; 571 572 if (vsid == -1) 573 page_found = -EINVAL; 574 break; 575 } 576 577 if (vcpu->arch.mmu.is_dcbz32(vcpu) && 578 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) { 579 /* 580 * If we do the dcbz hack, we have to NX on every execution, 581 * so we can patch the executing code. This renders our guest 582 * NX-less. 583 */ 584 pte.may_execute = !data; 585 } 586 587 if (page_found == -ENOENT) { 588 /* Page not found in guest PTE entries */ 589 u64 ssrr1 = vcpu->arch.shadow_srr1; 590 u64 msr = kvmppc_get_msr(vcpu); 591 kvmppc_set_dar(vcpu, kvmppc_get_fault_dar(vcpu)); 592 kvmppc_set_dsisr(vcpu, vcpu->arch.fault_dsisr); 593 kvmppc_set_msr_fast(vcpu, msr | (ssrr1 & 0xf8000000ULL)); 594 kvmppc_book3s_queue_irqprio(vcpu, vec); 595 } else if (page_found == -EPERM) { 596 /* Storage protection */ 597 u32 dsisr = vcpu->arch.fault_dsisr; 598 u64 ssrr1 = vcpu->arch.shadow_srr1; 599 u64 msr = kvmppc_get_msr(vcpu); 600 kvmppc_set_dar(vcpu, kvmppc_get_fault_dar(vcpu)); 601 dsisr = (dsisr & ~DSISR_NOHPTE) | DSISR_PROTFAULT; 602 kvmppc_set_dsisr(vcpu, dsisr); 603 kvmppc_set_msr_fast(vcpu, msr | (ssrr1 & 0xf8000000ULL)); 604 kvmppc_book3s_queue_irqprio(vcpu, vec); 605 } else if (page_found == -EINVAL) { 606 /* Page not found in guest SLB */ 607 kvmppc_set_dar(vcpu, kvmppc_get_fault_dar(vcpu)); 608 kvmppc_book3s_queue_irqprio(vcpu, vec + 0x80); 609 } else if (kvmppc_visible_gpa(vcpu, pte.raddr)) { 610 if (data && !(vcpu->arch.fault_dsisr & DSISR_NOHPTE)) { 611 /* 612 * There is already a host HPTE there, presumably 613 * a read-only one for a page the guest thinks 614 * is writable, so get rid of it first. 615 */ 616 kvmppc_mmu_unmap_page(vcpu, &pte); 617 } 618 /* The guest's PTE is not mapped yet. Map on the host */ 619 if (kvmppc_mmu_map_page(vcpu, &pte, iswrite) == -EIO) { 620 /* Exit KVM if mapping failed */ 621 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 622 return RESUME_HOST; 623 } 624 if (data) 625 vcpu->stat.sp_storage++; 626 else if (vcpu->arch.mmu.is_dcbz32(vcpu) && 627 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) 628 kvmppc_patch_dcbz(vcpu, &pte); 629 } else { 630 /* MMIO */ 631 vcpu->stat.mmio_exits++; 632 vcpu->arch.paddr_accessed = pte.raddr; 633 vcpu->arch.vaddr_accessed = pte.eaddr; 634 r = kvmppc_emulate_mmio(run, vcpu); 635 if ( r == RESUME_HOST_NV ) 636 r = RESUME_HOST; 637 } 638 639 return r; 640 } 641 642 /* Give up external provider (FPU, Altivec, VSX) */ 643 void kvmppc_giveup_ext(struct kvm_vcpu *vcpu, ulong msr) 644 { 645 struct thread_struct *t = ¤t->thread; 646 647 /* 648 * VSX instructions can access FP and vector registers, so if 649 * we are giving up VSX, make sure we give up FP and VMX as well. 650 */ 651 if (msr & MSR_VSX) 652 msr |= MSR_FP | MSR_VEC; 653 654 msr &= vcpu->arch.guest_owned_ext; 655 if (!msr) 656 return; 657 658 #ifdef DEBUG_EXT 659 printk(KERN_INFO "Giving up ext 0x%lx\n", msr); 660 #endif 661 662 if (msr & MSR_FP) { 663 /* 664 * Note that on CPUs with VSX, giveup_fpu stores 665 * both the traditional FP registers and the added VSX 666 * registers into thread.fp_state.fpr[]. 667 */ 668 if (t->regs->msr & MSR_FP) 669 giveup_fpu(current); 670 t->fp_save_area = NULL; 671 } 672 673 #ifdef CONFIG_ALTIVEC 674 if (msr & MSR_VEC) { 675 if (current->thread.regs->msr & MSR_VEC) 676 giveup_altivec(current); 677 t->vr_save_area = NULL; 678 } 679 #endif 680 681 vcpu->arch.guest_owned_ext &= ~(msr | MSR_VSX); 682 kvmppc_recalc_shadow_msr(vcpu); 683 } 684 685 /* Give up facility (TAR / EBB / DSCR) */ 686 static void kvmppc_giveup_fac(struct kvm_vcpu *vcpu, ulong fac) 687 { 688 #ifdef CONFIG_PPC_BOOK3S_64 689 if (!(vcpu->arch.shadow_fscr & (1ULL << fac))) { 690 /* Facility not available to the guest, ignore giveup request*/ 691 return; 692 } 693 694 switch (fac) { 695 case FSCR_TAR_LG: 696 vcpu->arch.tar = mfspr(SPRN_TAR); 697 mtspr(SPRN_TAR, current->thread.tar); 698 vcpu->arch.shadow_fscr &= ~FSCR_TAR; 699 break; 700 } 701 #endif 702 } 703 704 /* Handle external providers (FPU, Altivec, VSX) */ 705 static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr, 706 ulong msr) 707 { 708 struct thread_struct *t = ¤t->thread; 709 710 /* When we have paired singles, we emulate in software */ 711 if (vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE) 712 return RESUME_GUEST; 713 714 if (!(kvmppc_get_msr(vcpu) & msr)) { 715 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 716 return RESUME_GUEST; 717 } 718 719 if (msr == MSR_VSX) { 720 /* No VSX? Give an illegal instruction interrupt */ 721 #ifdef CONFIG_VSX 722 if (!cpu_has_feature(CPU_FTR_VSX)) 723 #endif 724 { 725 kvmppc_core_queue_program(vcpu, SRR1_PROGILL); 726 return RESUME_GUEST; 727 } 728 729 /* 730 * We have to load up all the FP and VMX registers before 731 * we can let the guest use VSX instructions. 732 */ 733 msr = MSR_FP | MSR_VEC | MSR_VSX; 734 } 735 736 /* See if we already own all the ext(s) needed */ 737 msr &= ~vcpu->arch.guest_owned_ext; 738 if (!msr) 739 return RESUME_GUEST; 740 741 #ifdef DEBUG_EXT 742 printk(KERN_INFO "Loading up ext 0x%lx\n", msr); 743 #endif 744 745 if (msr & MSR_FP) { 746 preempt_disable(); 747 enable_kernel_fp(); 748 load_fp_state(&vcpu->arch.fp); 749 disable_kernel_fp(); 750 t->fp_save_area = &vcpu->arch.fp; 751 preempt_enable(); 752 } 753 754 if (msr & MSR_VEC) { 755 #ifdef CONFIG_ALTIVEC 756 preempt_disable(); 757 enable_kernel_altivec(); 758 load_vr_state(&vcpu->arch.vr); 759 disable_kernel_altivec(); 760 t->vr_save_area = &vcpu->arch.vr; 761 preempt_enable(); 762 #endif 763 } 764 765 t->regs->msr |= msr; 766 vcpu->arch.guest_owned_ext |= msr; 767 kvmppc_recalc_shadow_msr(vcpu); 768 769 return RESUME_GUEST; 770 } 771 772 /* 773 * Kernel code using FP or VMX could have flushed guest state to 774 * the thread_struct; if so, get it back now. 775 */ 776 static void kvmppc_handle_lost_ext(struct kvm_vcpu *vcpu) 777 { 778 unsigned long lost_ext; 779 780 lost_ext = vcpu->arch.guest_owned_ext & ~current->thread.regs->msr; 781 if (!lost_ext) 782 return; 783 784 if (lost_ext & MSR_FP) { 785 preempt_disable(); 786 enable_kernel_fp(); 787 load_fp_state(&vcpu->arch.fp); 788 disable_kernel_fp(); 789 preempt_enable(); 790 } 791 #ifdef CONFIG_ALTIVEC 792 if (lost_ext & MSR_VEC) { 793 preempt_disable(); 794 enable_kernel_altivec(); 795 load_vr_state(&vcpu->arch.vr); 796 disable_kernel_altivec(); 797 preempt_enable(); 798 } 799 #endif 800 current->thread.regs->msr |= lost_ext; 801 } 802 803 #ifdef CONFIG_PPC_BOOK3S_64 804 805 static void kvmppc_trigger_fac_interrupt(struct kvm_vcpu *vcpu, ulong fac) 806 { 807 /* Inject the Interrupt Cause field and trigger a guest interrupt */ 808 vcpu->arch.fscr &= ~(0xffULL << 56); 809 vcpu->arch.fscr |= (fac << 56); 810 kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_FAC_UNAVAIL); 811 } 812 813 static void kvmppc_emulate_fac(struct kvm_vcpu *vcpu, ulong fac) 814 { 815 enum emulation_result er = EMULATE_FAIL; 816 817 if (!(kvmppc_get_msr(vcpu) & MSR_PR)) 818 er = kvmppc_emulate_instruction(vcpu->run, vcpu); 819 820 if ((er != EMULATE_DONE) && (er != EMULATE_AGAIN)) { 821 /* Couldn't emulate, trigger interrupt in guest */ 822 kvmppc_trigger_fac_interrupt(vcpu, fac); 823 } 824 } 825 826 /* Enable facilities (TAR, EBB, DSCR) for the guest */ 827 static int kvmppc_handle_fac(struct kvm_vcpu *vcpu, ulong fac) 828 { 829 bool guest_fac_enabled; 830 BUG_ON(!cpu_has_feature(CPU_FTR_ARCH_207S)); 831 832 /* 833 * Not every facility is enabled by FSCR bits, check whether the 834 * guest has this facility enabled at all. 835 */ 836 switch (fac) { 837 case FSCR_TAR_LG: 838 case FSCR_EBB_LG: 839 guest_fac_enabled = (vcpu->arch.fscr & (1ULL << fac)); 840 break; 841 case FSCR_TM_LG: 842 guest_fac_enabled = kvmppc_get_msr(vcpu) & MSR_TM; 843 break; 844 default: 845 guest_fac_enabled = false; 846 break; 847 } 848 849 if (!guest_fac_enabled) { 850 /* Facility not enabled by the guest */ 851 kvmppc_trigger_fac_interrupt(vcpu, fac); 852 return RESUME_GUEST; 853 } 854 855 switch (fac) { 856 case FSCR_TAR_LG: 857 /* TAR switching isn't lazy in Linux yet */ 858 current->thread.tar = mfspr(SPRN_TAR); 859 mtspr(SPRN_TAR, vcpu->arch.tar); 860 vcpu->arch.shadow_fscr |= FSCR_TAR; 861 break; 862 default: 863 kvmppc_emulate_fac(vcpu, fac); 864 break; 865 } 866 867 return RESUME_GUEST; 868 } 869 870 void kvmppc_set_fscr(struct kvm_vcpu *vcpu, u64 fscr) 871 { 872 if ((vcpu->arch.fscr & FSCR_TAR) && !(fscr & FSCR_TAR)) { 873 /* TAR got dropped, drop it in shadow too */ 874 kvmppc_giveup_fac(vcpu, FSCR_TAR_LG); 875 } 876 vcpu->arch.fscr = fscr; 877 } 878 #endif 879 880 static void kvmppc_setup_debug(struct kvm_vcpu *vcpu) 881 { 882 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) { 883 u64 msr = kvmppc_get_msr(vcpu); 884 885 kvmppc_set_msr(vcpu, msr | MSR_SE); 886 } 887 } 888 889 static void kvmppc_clear_debug(struct kvm_vcpu *vcpu) 890 { 891 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) { 892 u64 msr = kvmppc_get_msr(vcpu); 893 894 kvmppc_set_msr(vcpu, msr & ~MSR_SE); 895 } 896 } 897 898 static int kvmppc_exit_pr_progint(struct kvm_run *run, struct kvm_vcpu *vcpu, 899 unsigned int exit_nr) 900 { 901 enum emulation_result er; 902 ulong flags; 903 u32 last_inst; 904 int emul, r; 905 906 /* 907 * shadow_srr1 only contains valid flags if we came here via a program 908 * exception. The other exceptions (emulation assist, FP unavailable, 909 * etc.) do not provide flags in SRR1, so use an illegal-instruction 910 * exception when injecting a program interrupt into the guest. 911 */ 912 if (exit_nr == BOOK3S_INTERRUPT_PROGRAM) 913 flags = vcpu->arch.shadow_srr1 & 0x1f0000ull; 914 else 915 flags = SRR1_PROGILL; 916 917 emul = kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst); 918 if (emul != EMULATE_DONE) 919 return RESUME_GUEST; 920 921 if (kvmppc_get_msr(vcpu) & MSR_PR) { 922 #ifdef EXIT_DEBUG 923 pr_info("Userspace triggered 0x700 exception at\n 0x%lx (0x%x)\n", 924 kvmppc_get_pc(vcpu), last_inst); 925 #endif 926 if ((last_inst & 0xff0007ff) != (INS_DCBZ & 0xfffffff7)) { 927 kvmppc_core_queue_program(vcpu, flags); 928 return RESUME_GUEST; 929 } 930 } 931 932 vcpu->stat.emulated_inst_exits++; 933 er = kvmppc_emulate_instruction(run, vcpu); 934 switch (er) { 935 case EMULATE_DONE: 936 r = RESUME_GUEST_NV; 937 break; 938 case EMULATE_AGAIN: 939 r = RESUME_GUEST; 940 break; 941 case EMULATE_FAIL: 942 pr_crit("%s: emulation at %lx failed (%08x)\n", 943 __func__, kvmppc_get_pc(vcpu), last_inst); 944 kvmppc_core_queue_program(vcpu, flags); 945 r = RESUME_GUEST; 946 break; 947 case EMULATE_DO_MMIO: 948 run->exit_reason = KVM_EXIT_MMIO; 949 r = RESUME_HOST_NV; 950 break; 951 case EMULATE_EXIT_USER: 952 r = RESUME_HOST_NV; 953 break; 954 default: 955 BUG(); 956 } 957 958 return r; 959 } 960 961 int kvmppc_handle_exit_pr(struct kvm_run *run, struct kvm_vcpu *vcpu, 962 unsigned int exit_nr) 963 { 964 int r = RESUME_HOST; 965 int s; 966 967 vcpu->stat.sum_exits++; 968 969 run->exit_reason = KVM_EXIT_UNKNOWN; 970 run->ready_for_interrupt_injection = 1; 971 972 /* We get here with MSR.EE=1 */ 973 974 trace_kvm_exit(exit_nr, vcpu); 975 guest_exit(); 976 977 switch (exit_nr) { 978 case BOOK3S_INTERRUPT_INST_STORAGE: 979 { 980 ulong shadow_srr1 = vcpu->arch.shadow_srr1; 981 vcpu->stat.pf_instruc++; 982 983 if (kvmppc_is_split_real(vcpu)) 984 kvmppc_fixup_split_real(vcpu); 985 986 #ifdef CONFIG_PPC_BOOK3S_32 987 /* We set segments as unused segments when invalidating them. So 988 * treat the respective fault as segment fault. */ 989 { 990 struct kvmppc_book3s_shadow_vcpu *svcpu; 991 u32 sr; 992 993 svcpu = svcpu_get(vcpu); 994 sr = svcpu->sr[kvmppc_get_pc(vcpu) >> SID_SHIFT]; 995 svcpu_put(svcpu); 996 if (sr == SR_INVALID) { 997 kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)); 998 r = RESUME_GUEST; 999 break; 1000 } 1001 } 1002 #endif 1003 1004 /* only care about PTEG not found errors, but leave NX alone */ 1005 if (shadow_srr1 & 0x40000000) { 1006 int idx = srcu_read_lock(&vcpu->kvm->srcu); 1007 r = kvmppc_handle_pagefault(run, vcpu, kvmppc_get_pc(vcpu), exit_nr); 1008 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1009 vcpu->stat.sp_instruc++; 1010 } else if (vcpu->arch.mmu.is_dcbz32(vcpu) && 1011 (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) { 1012 /* 1013 * XXX If we do the dcbz hack we use the NX bit to flush&patch the page, 1014 * so we can't use the NX bit inside the guest. Let's cross our fingers, 1015 * that no guest that needs the dcbz hack does NX. 1016 */ 1017 kvmppc_mmu_pte_flush(vcpu, kvmppc_get_pc(vcpu), ~0xFFFUL); 1018 r = RESUME_GUEST; 1019 } else { 1020 u64 msr = kvmppc_get_msr(vcpu); 1021 msr |= shadow_srr1 & 0x58000000; 1022 kvmppc_set_msr_fast(vcpu, msr); 1023 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 1024 r = RESUME_GUEST; 1025 } 1026 break; 1027 } 1028 case BOOK3S_INTERRUPT_DATA_STORAGE: 1029 { 1030 ulong dar = kvmppc_get_fault_dar(vcpu); 1031 u32 fault_dsisr = vcpu->arch.fault_dsisr; 1032 vcpu->stat.pf_storage++; 1033 1034 #ifdef CONFIG_PPC_BOOK3S_32 1035 /* We set segments as unused segments when invalidating them. So 1036 * treat the respective fault as segment fault. */ 1037 { 1038 struct kvmppc_book3s_shadow_vcpu *svcpu; 1039 u32 sr; 1040 1041 svcpu = svcpu_get(vcpu); 1042 sr = svcpu->sr[dar >> SID_SHIFT]; 1043 svcpu_put(svcpu); 1044 if (sr == SR_INVALID) { 1045 kvmppc_mmu_map_segment(vcpu, dar); 1046 r = RESUME_GUEST; 1047 break; 1048 } 1049 } 1050 #endif 1051 1052 /* 1053 * We need to handle missing shadow PTEs, and 1054 * protection faults due to us mapping a page read-only 1055 * when the guest thinks it is writable. 1056 */ 1057 if (fault_dsisr & (DSISR_NOHPTE | DSISR_PROTFAULT)) { 1058 int idx = srcu_read_lock(&vcpu->kvm->srcu); 1059 r = kvmppc_handle_pagefault(run, vcpu, dar, exit_nr); 1060 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1061 } else { 1062 kvmppc_set_dar(vcpu, dar); 1063 kvmppc_set_dsisr(vcpu, fault_dsisr); 1064 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 1065 r = RESUME_GUEST; 1066 } 1067 break; 1068 } 1069 case BOOK3S_INTERRUPT_DATA_SEGMENT: 1070 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_fault_dar(vcpu)) < 0) { 1071 kvmppc_set_dar(vcpu, kvmppc_get_fault_dar(vcpu)); 1072 kvmppc_book3s_queue_irqprio(vcpu, 1073 BOOK3S_INTERRUPT_DATA_SEGMENT); 1074 } 1075 r = RESUME_GUEST; 1076 break; 1077 case BOOK3S_INTERRUPT_INST_SEGMENT: 1078 if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)) < 0) { 1079 kvmppc_book3s_queue_irqprio(vcpu, 1080 BOOK3S_INTERRUPT_INST_SEGMENT); 1081 } 1082 r = RESUME_GUEST; 1083 break; 1084 /* We're good on these - the host merely wanted to get our attention */ 1085 case BOOK3S_INTERRUPT_DECREMENTER: 1086 case BOOK3S_INTERRUPT_HV_DECREMENTER: 1087 case BOOK3S_INTERRUPT_DOORBELL: 1088 case BOOK3S_INTERRUPT_H_DOORBELL: 1089 vcpu->stat.dec_exits++; 1090 r = RESUME_GUEST; 1091 break; 1092 case BOOK3S_INTERRUPT_EXTERNAL: 1093 case BOOK3S_INTERRUPT_EXTERNAL_LEVEL: 1094 case BOOK3S_INTERRUPT_EXTERNAL_HV: 1095 vcpu->stat.ext_intr_exits++; 1096 r = RESUME_GUEST; 1097 break; 1098 case BOOK3S_INTERRUPT_PERFMON: 1099 r = RESUME_GUEST; 1100 break; 1101 case BOOK3S_INTERRUPT_PROGRAM: 1102 case BOOK3S_INTERRUPT_H_EMUL_ASSIST: 1103 r = kvmppc_exit_pr_progint(run, vcpu, exit_nr); 1104 break; 1105 case BOOK3S_INTERRUPT_SYSCALL: 1106 { 1107 u32 last_sc; 1108 int emul; 1109 1110 /* Get last sc for papr */ 1111 if (vcpu->arch.papr_enabled) { 1112 /* The sc instuction points SRR0 to the next inst */ 1113 emul = kvmppc_get_last_inst(vcpu, INST_SC, &last_sc); 1114 if (emul != EMULATE_DONE) { 1115 kvmppc_set_pc(vcpu, kvmppc_get_pc(vcpu) - 4); 1116 r = RESUME_GUEST; 1117 break; 1118 } 1119 } 1120 1121 if (vcpu->arch.papr_enabled && 1122 (last_sc == 0x44000022) && 1123 !(kvmppc_get_msr(vcpu) & MSR_PR)) { 1124 /* SC 1 papr hypercalls */ 1125 ulong cmd = kvmppc_get_gpr(vcpu, 3); 1126 int i; 1127 1128 #ifdef CONFIG_PPC_BOOK3S_64 1129 if (kvmppc_h_pr(vcpu, cmd) == EMULATE_DONE) { 1130 r = RESUME_GUEST; 1131 break; 1132 } 1133 #endif 1134 1135 run->papr_hcall.nr = cmd; 1136 for (i = 0; i < 9; ++i) { 1137 ulong gpr = kvmppc_get_gpr(vcpu, 4 + i); 1138 run->papr_hcall.args[i] = gpr; 1139 } 1140 run->exit_reason = KVM_EXIT_PAPR_HCALL; 1141 vcpu->arch.hcall_needed = 1; 1142 r = RESUME_HOST; 1143 } else if (vcpu->arch.osi_enabled && 1144 (((u32)kvmppc_get_gpr(vcpu, 3)) == OSI_SC_MAGIC_R3) && 1145 (((u32)kvmppc_get_gpr(vcpu, 4)) == OSI_SC_MAGIC_R4)) { 1146 /* MOL hypercalls */ 1147 u64 *gprs = run->osi.gprs; 1148 int i; 1149 1150 run->exit_reason = KVM_EXIT_OSI; 1151 for (i = 0; i < 32; i++) 1152 gprs[i] = kvmppc_get_gpr(vcpu, i); 1153 vcpu->arch.osi_needed = 1; 1154 r = RESUME_HOST_NV; 1155 } else if (!(kvmppc_get_msr(vcpu) & MSR_PR) && 1156 (((u32)kvmppc_get_gpr(vcpu, 0)) == KVM_SC_MAGIC_R0)) { 1157 /* KVM PV hypercalls */ 1158 kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu)); 1159 r = RESUME_GUEST; 1160 } else { 1161 /* Guest syscalls */ 1162 vcpu->stat.syscall_exits++; 1163 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 1164 r = RESUME_GUEST; 1165 } 1166 break; 1167 } 1168 case BOOK3S_INTERRUPT_FP_UNAVAIL: 1169 case BOOK3S_INTERRUPT_ALTIVEC: 1170 case BOOK3S_INTERRUPT_VSX: 1171 { 1172 int ext_msr = 0; 1173 int emul; 1174 u32 last_inst; 1175 1176 if (vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE) { 1177 /* Do paired single instruction emulation */ 1178 emul = kvmppc_get_last_inst(vcpu, INST_GENERIC, 1179 &last_inst); 1180 if (emul == EMULATE_DONE) 1181 r = kvmppc_exit_pr_progint(run, vcpu, exit_nr); 1182 else 1183 r = RESUME_GUEST; 1184 1185 break; 1186 } 1187 1188 /* Enable external provider */ 1189 switch (exit_nr) { 1190 case BOOK3S_INTERRUPT_FP_UNAVAIL: 1191 ext_msr = MSR_FP; 1192 break; 1193 1194 case BOOK3S_INTERRUPT_ALTIVEC: 1195 ext_msr = MSR_VEC; 1196 break; 1197 1198 case BOOK3S_INTERRUPT_VSX: 1199 ext_msr = MSR_VSX; 1200 break; 1201 } 1202 1203 r = kvmppc_handle_ext(vcpu, exit_nr, ext_msr); 1204 break; 1205 } 1206 case BOOK3S_INTERRUPT_ALIGNMENT: 1207 { 1208 u32 last_inst; 1209 int emul = kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst); 1210 1211 if (emul == EMULATE_DONE) { 1212 u32 dsisr; 1213 u64 dar; 1214 1215 dsisr = kvmppc_alignment_dsisr(vcpu, last_inst); 1216 dar = kvmppc_alignment_dar(vcpu, last_inst); 1217 1218 kvmppc_set_dsisr(vcpu, dsisr); 1219 kvmppc_set_dar(vcpu, dar); 1220 1221 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 1222 } 1223 r = RESUME_GUEST; 1224 break; 1225 } 1226 #ifdef CONFIG_PPC_BOOK3S_64 1227 case BOOK3S_INTERRUPT_FAC_UNAVAIL: 1228 kvmppc_handle_fac(vcpu, vcpu->arch.shadow_fscr >> 56); 1229 r = RESUME_GUEST; 1230 break; 1231 #endif 1232 case BOOK3S_INTERRUPT_MACHINE_CHECK: 1233 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 1234 r = RESUME_GUEST; 1235 break; 1236 case BOOK3S_INTERRUPT_TRACE: 1237 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) { 1238 run->exit_reason = KVM_EXIT_DEBUG; 1239 r = RESUME_HOST; 1240 } else { 1241 kvmppc_book3s_queue_irqprio(vcpu, exit_nr); 1242 r = RESUME_GUEST; 1243 } 1244 break; 1245 default: 1246 { 1247 ulong shadow_srr1 = vcpu->arch.shadow_srr1; 1248 /* Ugh - bork here! What did we get? */ 1249 printk(KERN_EMERG "exit_nr=0x%x | pc=0x%lx | msr=0x%lx\n", 1250 exit_nr, kvmppc_get_pc(vcpu), shadow_srr1); 1251 r = RESUME_HOST; 1252 BUG(); 1253 break; 1254 } 1255 } 1256 1257 if (!(r & RESUME_HOST)) { 1258 /* To avoid clobbering exit_reason, only check for signals if 1259 * we aren't already exiting to userspace for some other 1260 * reason. */ 1261 1262 /* 1263 * Interrupts could be timers for the guest which we have to 1264 * inject again, so let's postpone them until we're in the guest 1265 * and if we really did time things so badly, then we just exit 1266 * again due to a host external interrupt. 1267 */ 1268 s = kvmppc_prepare_to_enter(vcpu); 1269 if (s <= 0) 1270 r = s; 1271 else { 1272 /* interrupts now hard-disabled */ 1273 kvmppc_fix_ee_before_entry(); 1274 } 1275 1276 kvmppc_handle_lost_ext(vcpu); 1277 } 1278 1279 trace_kvm_book3s_reenter(r, vcpu); 1280 1281 return r; 1282 } 1283 1284 static int kvm_arch_vcpu_ioctl_get_sregs_pr(struct kvm_vcpu *vcpu, 1285 struct kvm_sregs *sregs) 1286 { 1287 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu); 1288 int i; 1289 1290 sregs->pvr = vcpu->arch.pvr; 1291 1292 sregs->u.s.sdr1 = to_book3s(vcpu)->sdr1; 1293 if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) { 1294 for (i = 0; i < 64; i++) { 1295 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige | i; 1296 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv; 1297 } 1298 } else { 1299 for (i = 0; i < 16; i++) 1300 sregs->u.s.ppc32.sr[i] = kvmppc_get_sr(vcpu, i); 1301 1302 for (i = 0; i < 8; i++) { 1303 sregs->u.s.ppc32.ibat[i] = vcpu3s->ibat[i].raw; 1304 sregs->u.s.ppc32.dbat[i] = vcpu3s->dbat[i].raw; 1305 } 1306 } 1307 1308 return 0; 1309 } 1310 1311 static int kvm_arch_vcpu_ioctl_set_sregs_pr(struct kvm_vcpu *vcpu, 1312 struct kvm_sregs *sregs) 1313 { 1314 struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu); 1315 int i; 1316 1317 kvmppc_set_pvr_pr(vcpu, sregs->pvr); 1318 1319 vcpu3s->sdr1 = sregs->u.s.sdr1; 1320 #ifdef CONFIG_PPC_BOOK3S_64 1321 if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) { 1322 /* Flush all SLB entries */ 1323 vcpu->arch.mmu.slbmte(vcpu, 0, 0); 1324 vcpu->arch.mmu.slbia(vcpu); 1325 1326 for (i = 0; i < 64; i++) { 1327 u64 rb = sregs->u.s.ppc64.slb[i].slbe; 1328 u64 rs = sregs->u.s.ppc64.slb[i].slbv; 1329 1330 if (rb & SLB_ESID_V) 1331 vcpu->arch.mmu.slbmte(vcpu, rs, rb); 1332 } 1333 } else 1334 #endif 1335 { 1336 for (i = 0; i < 16; i++) { 1337 vcpu->arch.mmu.mtsrin(vcpu, i, sregs->u.s.ppc32.sr[i]); 1338 } 1339 for (i = 0; i < 8; i++) { 1340 kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), false, 1341 (u32)sregs->u.s.ppc32.ibat[i]); 1342 kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), true, 1343 (u32)(sregs->u.s.ppc32.ibat[i] >> 32)); 1344 kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), false, 1345 (u32)sregs->u.s.ppc32.dbat[i]); 1346 kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), true, 1347 (u32)(sregs->u.s.ppc32.dbat[i] >> 32)); 1348 } 1349 } 1350 1351 /* Flush the MMU after messing with the segments */ 1352 kvmppc_mmu_pte_flush(vcpu, 0, 0); 1353 1354 return 0; 1355 } 1356 1357 static int kvmppc_get_one_reg_pr(struct kvm_vcpu *vcpu, u64 id, 1358 union kvmppc_one_reg *val) 1359 { 1360 int r = 0; 1361 1362 switch (id) { 1363 case KVM_REG_PPC_DEBUG_INST: 1364 *val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT); 1365 break; 1366 case KVM_REG_PPC_HIOR: 1367 *val = get_reg_val(id, to_book3s(vcpu)->hior); 1368 break; 1369 case KVM_REG_PPC_VTB: 1370 *val = get_reg_val(id, to_book3s(vcpu)->vtb); 1371 break; 1372 case KVM_REG_PPC_LPCR: 1373 case KVM_REG_PPC_LPCR_64: 1374 /* 1375 * We are only interested in the LPCR_ILE bit 1376 */ 1377 if (vcpu->arch.intr_msr & MSR_LE) 1378 *val = get_reg_val(id, LPCR_ILE); 1379 else 1380 *val = get_reg_val(id, 0); 1381 break; 1382 default: 1383 r = -EINVAL; 1384 break; 1385 } 1386 1387 return r; 1388 } 1389 1390 static void kvmppc_set_lpcr_pr(struct kvm_vcpu *vcpu, u64 new_lpcr) 1391 { 1392 if (new_lpcr & LPCR_ILE) 1393 vcpu->arch.intr_msr |= MSR_LE; 1394 else 1395 vcpu->arch.intr_msr &= ~MSR_LE; 1396 } 1397 1398 static int kvmppc_set_one_reg_pr(struct kvm_vcpu *vcpu, u64 id, 1399 union kvmppc_one_reg *val) 1400 { 1401 int r = 0; 1402 1403 switch (id) { 1404 case KVM_REG_PPC_HIOR: 1405 to_book3s(vcpu)->hior = set_reg_val(id, *val); 1406 to_book3s(vcpu)->hior_explicit = true; 1407 break; 1408 case KVM_REG_PPC_VTB: 1409 to_book3s(vcpu)->vtb = set_reg_val(id, *val); 1410 break; 1411 case KVM_REG_PPC_LPCR: 1412 case KVM_REG_PPC_LPCR_64: 1413 kvmppc_set_lpcr_pr(vcpu, set_reg_val(id, *val)); 1414 break; 1415 default: 1416 r = -EINVAL; 1417 break; 1418 } 1419 1420 return r; 1421 } 1422 1423 static struct kvm_vcpu *kvmppc_core_vcpu_create_pr(struct kvm *kvm, 1424 unsigned int id) 1425 { 1426 struct kvmppc_vcpu_book3s *vcpu_book3s; 1427 struct kvm_vcpu *vcpu; 1428 int err = -ENOMEM; 1429 unsigned long p; 1430 1431 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); 1432 if (!vcpu) 1433 goto out; 1434 1435 vcpu_book3s = vzalloc(sizeof(struct kvmppc_vcpu_book3s)); 1436 if (!vcpu_book3s) 1437 goto free_vcpu; 1438 vcpu->arch.book3s = vcpu_book3s; 1439 1440 #ifdef CONFIG_KVM_BOOK3S_32_HANDLER 1441 vcpu->arch.shadow_vcpu = 1442 kzalloc(sizeof(*vcpu->arch.shadow_vcpu), GFP_KERNEL); 1443 if (!vcpu->arch.shadow_vcpu) 1444 goto free_vcpu3s; 1445 #endif 1446 1447 err = kvm_vcpu_init(vcpu, kvm, id); 1448 if (err) 1449 goto free_shadow_vcpu; 1450 1451 err = -ENOMEM; 1452 p = __get_free_page(GFP_KERNEL|__GFP_ZERO); 1453 if (!p) 1454 goto uninit_vcpu; 1455 vcpu->arch.shared = (void *)p; 1456 #ifdef CONFIG_PPC_BOOK3S_64 1457 /* Always start the shared struct in native endian mode */ 1458 #ifdef __BIG_ENDIAN__ 1459 vcpu->arch.shared_big_endian = true; 1460 #else 1461 vcpu->arch.shared_big_endian = false; 1462 #endif 1463 1464 /* 1465 * Default to the same as the host if we're on sufficiently 1466 * recent machine that we have 1TB segments; 1467 * otherwise default to PPC970FX. 1468 */ 1469 vcpu->arch.pvr = 0x3C0301; 1470 if (mmu_has_feature(MMU_FTR_1T_SEGMENT)) 1471 vcpu->arch.pvr = mfspr(SPRN_PVR); 1472 vcpu->arch.intr_msr = MSR_SF; 1473 #else 1474 /* default to book3s_32 (750) */ 1475 vcpu->arch.pvr = 0x84202; 1476 #endif 1477 kvmppc_set_pvr_pr(vcpu, vcpu->arch.pvr); 1478 vcpu->arch.slb_nr = 64; 1479 1480 vcpu->arch.shadow_msr = MSR_USER64 & ~MSR_LE; 1481 1482 err = kvmppc_mmu_init(vcpu); 1483 if (err < 0) 1484 goto uninit_vcpu; 1485 1486 return vcpu; 1487 1488 uninit_vcpu: 1489 kvm_vcpu_uninit(vcpu); 1490 free_shadow_vcpu: 1491 #ifdef CONFIG_KVM_BOOK3S_32_HANDLER 1492 kfree(vcpu->arch.shadow_vcpu); 1493 free_vcpu3s: 1494 #endif 1495 vfree(vcpu_book3s); 1496 free_vcpu: 1497 kmem_cache_free(kvm_vcpu_cache, vcpu); 1498 out: 1499 return ERR_PTR(err); 1500 } 1501 1502 static void kvmppc_core_vcpu_free_pr(struct kvm_vcpu *vcpu) 1503 { 1504 struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu); 1505 1506 free_page((unsigned long)vcpu->arch.shared & PAGE_MASK); 1507 kvm_vcpu_uninit(vcpu); 1508 #ifdef CONFIG_KVM_BOOK3S_32_HANDLER 1509 kfree(vcpu->arch.shadow_vcpu); 1510 #endif 1511 vfree(vcpu_book3s); 1512 kmem_cache_free(kvm_vcpu_cache, vcpu); 1513 } 1514 1515 static int kvmppc_vcpu_run_pr(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) 1516 { 1517 int ret; 1518 #ifdef CONFIG_ALTIVEC 1519 unsigned long uninitialized_var(vrsave); 1520 #endif 1521 1522 /* Check if we can run the vcpu at all */ 1523 if (!vcpu->arch.sane) { 1524 kvm_run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 1525 ret = -EINVAL; 1526 goto out; 1527 } 1528 1529 kvmppc_setup_debug(vcpu); 1530 1531 /* 1532 * Interrupts could be timers for the guest which we have to inject 1533 * again, so let's postpone them until we're in the guest and if we 1534 * really did time things so badly, then we just exit again due to 1535 * a host external interrupt. 1536 */ 1537 ret = kvmppc_prepare_to_enter(vcpu); 1538 if (ret <= 0) 1539 goto out; 1540 /* interrupts now hard-disabled */ 1541 1542 /* Save FPU, Altivec and VSX state */ 1543 giveup_all(current); 1544 1545 /* Preload FPU if it's enabled */ 1546 if (kvmppc_get_msr(vcpu) & MSR_FP) 1547 kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP); 1548 1549 kvmppc_fix_ee_before_entry(); 1550 1551 ret = __kvmppc_vcpu_run(kvm_run, vcpu); 1552 1553 kvmppc_clear_debug(vcpu); 1554 1555 /* No need for guest_exit. It's done in handle_exit. 1556 We also get here with interrupts enabled. */ 1557 1558 /* Make sure we save the guest FPU/Altivec/VSX state */ 1559 kvmppc_giveup_ext(vcpu, MSR_FP | MSR_VEC | MSR_VSX); 1560 1561 /* Make sure we save the guest TAR/EBB/DSCR state */ 1562 kvmppc_giveup_fac(vcpu, FSCR_TAR_LG); 1563 1564 out: 1565 vcpu->mode = OUTSIDE_GUEST_MODE; 1566 return ret; 1567 } 1568 1569 /* 1570 * Get (and clear) the dirty memory log for a memory slot. 1571 */ 1572 static int kvm_vm_ioctl_get_dirty_log_pr(struct kvm *kvm, 1573 struct kvm_dirty_log *log) 1574 { 1575 struct kvm_memslots *slots; 1576 struct kvm_memory_slot *memslot; 1577 struct kvm_vcpu *vcpu; 1578 ulong ga, ga_end; 1579 int is_dirty = 0; 1580 int r; 1581 unsigned long n; 1582 1583 mutex_lock(&kvm->slots_lock); 1584 1585 r = kvm_get_dirty_log(kvm, log, &is_dirty); 1586 if (r) 1587 goto out; 1588 1589 /* If nothing is dirty, don't bother messing with page tables. */ 1590 if (is_dirty) { 1591 slots = kvm_memslots(kvm); 1592 memslot = id_to_memslot(slots, log->slot); 1593 1594 ga = memslot->base_gfn << PAGE_SHIFT; 1595 ga_end = ga + (memslot->npages << PAGE_SHIFT); 1596 1597 kvm_for_each_vcpu(n, vcpu, kvm) 1598 kvmppc_mmu_pte_pflush(vcpu, ga, ga_end); 1599 1600 n = kvm_dirty_bitmap_bytes(memslot); 1601 memset(memslot->dirty_bitmap, 0, n); 1602 } 1603 1604 r = 0; 1605 out: 1606 mutex_unlock(&kvm->slots_lock); 1607 return r; 1608 } 1609 1610 static void kvmppc_core_flush_memslot_pr(struct kvm *kvm, 1611 struct kvm_memory_slot *memslot) 1612 { 1613 return; 1614 } 1615 1616 static int kvmppc_core_prepare_memory_region_pr(struct kvm *kvm, 1617 struct kvm_memory_slot *memslot, 1618 const struct kvm_userspace_memory_region *mem) 1619 { 1620 return 0; 1621 } 1622 1623 static void kvmppc_core_commit_memory_region_pr(struct kvm *kvm, 1624 const struct kvm_userspace_memory_region *mem, 1625 const struct kvm_memory_slot *old, 1626 const struct kvm_memory_slot *new) 1627 { 1628 return; 1629 } 1630 1631 static void kvmppc_core_free_memslot_pr(struct kvm_memory_slot *free, 1632 struct kvm_memory_slot *dont) 1633 { 1634 return; 1635 } 1636 1637 static int kvmppc_core_create_memslot_pr(struct kvm_memory_slot *slot, 1638 unsigned long npages) 1639 { 1640 return 0; 1641 } 1642 1643 1644 #ifdef CONFIG_PPC64 1645 static int kvm_vm_ioctl_get_smmu_info_pr(struct kvm *kvm, 1646 struct kvm_ppc_smmu_info *info) 1647 { 1648 long int i; 1649 struct kvm_vcpu *vcpu; 1650 1651 info->flags = 0; 1652 1653 /* SLB is always 64 entries */ 1654 info->slb_size = 64; 1655 1656 /* Standard 4k base page size segment */ 1657 info->sps[0].page_shift = 12; 1658 info->sps[0].slb_enc = 0; 1659 info->sps[0].enc[0].page_shift = 12; 1660 info->sps[0].enc[0].pte_enc = 0; 1661 1662 /* 1663 * 64k large page size. 1664 * We only want to put this in if the CPUs we're emulating 1665 * support it, but unfortunately we don't have a vcpu easily 1666 * to hand here to test. Just pick the first vcpu, and if 1667 * that doesn't exist yet, report the minimum capability, 1668 * i.e., no 64k pages. 1669 * 1T segment support goes along with 64k pages. 1670 */ 1671 i = 1; 1672 vcpu = kvm_get_vcpu(kvm, 0); 1673 if (vcpu && (vcpu->arch.hflags & BOOK3S_HFLAG_MULTI_PGSIZE)) { 1674 info->flags = KVM_PPC_1T_SEGMENTS; 1675 info->sps[i].page_shift = 16; 1676 info->sps[i].slb_enc = SLB_VSID_L | SLB_VSID_LP_01; 1677 info->sps[i].enc[0].page_shift = 16; 1678 info->sps[i].enc[0].pte_enc = 1; 1679 ++i; 1680 } 1681 1682 /* Standard 16M large page size segment */ 1683 info->sps[i].page_shift = 24; 1684 info->sps[i].slb_enc = SLB_VSID_L; 1685 info->sps[i].enc[0].page_shift = 24; 1686 info->sps[i].enc[0].pte_enc = 0; 1687 1688 return 0; 1689 } 1690 #else 1691 static int kvm_vm_ioctl_get_smmu_info_pr(struct kvm *kvm, 1692 struct kvm_ppc_smmu_info *info) 1693 { 1694 /* We should not get called */ 1695 BUG(); 1696 } 1697 #endif /* CONFIG_PPC64 */ 1698 1699 static unsigned int kvm_global_user_count = 0; 1700 static DEFINE_SPINLOCK(kvm_global_user_count_lock); 1701 1702 static int kvmppc_core_init_vm_pr(struct kvm *kvm) 1703 { 1704 mutex_init(&kvm->arch.hpt_mutex); 1705 1706 #ifdef CONFIG_PPC_BOOK3S_64 1707 /* Start out with the default set of hcalls enabled */ 1708 kvmppc_pr_init_default_hcalls(kvm); 1709 #endif 1710 1711 if (firmware_has_feature(FW_FEATURE_SET_MODE)) { 1712 spin_lock(&kvm_global_user_count_lock); 1713 if (++kvm_global_user_count == 1) 1714 pseries_disable_reloc_on_exc(); 1715 spin_unlock(&kvm_global_user_count_lock); 1716 } 1717 return 0; 1718 } 1719 1720 static void kvmppc_core_destroy_vm_pr(struct kvm *kvm) 1721 { 1722 #ifdef CONFIG_PPC64 1723 WARN_ON(!list_empty(&kvm->arch.spapr_tce_tables)); 1724 #endif 1725 1726 if (firmware_has_feature(FW_FEATURE_SET_MODE)) { 1727 spin_lock(&kvm_global_user_count_lock); 1728 BUG_ON(kvm_global_user_count == 0); 1729 if (--kvm_global_user_count == 0) 1730 pseries_enable_reloc_on_exc(); 1731 spin_unlock(&kvm_global_user_count_lock); 1732 } 1733 } 1734 1735 static int kvmppc_core_check_processor_compat_pr(void) 1736 { 1737 /* 1738 * Disable KVM for Power9 untill the required bits merged. 1739 */ 1740 if (cpu_has_feature(CPU_FTR_ARCH_300)) 1741 return -EIO; 1742 return 0; 1743 } 1744 1745 static long kvm_arch_vm_ioctl_pr(struct file *filp, 1746 unsigned int ioctl, unsigned long arg) 1747 { 1748 return -ENOTTY; 1749 } 1750 1751 static struct kvmppc_ops kvm_ops_pr = { 1752 .get_sregs = kvm_arch_vcpu_ioctl_get_sregs_pr, 1753 .set_sregs = kvm_arch_vcpu_ioctl_set_sregs_pr, 1754 .get_one_reg = kvmppc_get_one_reg_pr, 1755 .set_one_reg = kvmppc_set_one_reg_pr, 1756 .vcpu_load = kvmppc_core_vcpu_load_pr, 1757 .vcpu_put = kvmppc_core_vcpu_put_pr, 1758 .set_msr = kvmppc_set_msr_pr, 1759 .vcpu_run = kvmppc_vcpu_run_pr, 1760 .vcpu_create = kvmppc_core_vcpu_create_pr, 1761 .vcpu_free = kvmppc_core_vcpu_free_pr, 1762 .check_requests = kvmppc_core_check_requests_pr, 1763 .get_dirty_log = kvm_vm_ioctl_get_dirty_log_pr, 1764 .flush_memslot = kvmppc_core_flush_memslot_pr, 1765 .prepare_memory_region = kvmppc_core_prepare_memory_region_pr, 1766 .commit_memory_region = kvmppc_core_commit_memory_region_pr, 1767 .unmap_hva_range = kvm_unmap_hva_range_pr, 1768 .age_hva = kvm_age_hva_pr, 1769 .test_age_hva = kvm_test_age_hva_pr, 1770 .set_spte_hva = kvm_set_spte_hva_pr, 1771 .mmu_destroy = kvmppc_mmu_destroy_pr, 1772 .free_memslot = kvmppc_core_free_memslot_pr, 1773 .create_memslot = kvmppc_core_create_memslot_pr, 1774 .init_vm = kvmppc_core_init_vm_pr, 1775 .destroy_vm = kvmppc_core_destroy_vm_pr, 1776 .get_smmu_info = kvm_vm_ioctl_get_smmu_info_pr, 1777 .emulate_op = kvmppc_core_emulate_op_pr, 1778 .emulate_mtspr = kvmppc_core_emulate_mtspr_pr, 1779 .emulate_mfspr = kvmppc_core_emulate_mfspr_pr, 1780 .fast_vcpu_kick = kvm_vcpu_kick, 1781 .arch_vm_ioctl = kvm_arch_vm_ioctl_pr, 1782 #ifdef CONFIG_PPC_BOOK3S_64 1783 .hcall_implemented = kvmppc_hcall_impl_pr, 1784 #endif 1785 }; 1786 1787 1788 int kvmppc_book3s_init_pr(void) 1789 { 1790 int r; 1791 1792 r = kvmppc_core_check_processor_compat_pr(); 1793 if (r < 0) 1794 return r; 1795 1796 kvm_ops_pr.owner = THIS_MODULE; 1797 kvmppc_pr_ops = &kvm_ops_pr; 1798 1799 r = kvmppc_mmu_hpte_sysinit(); 1800 return r; 1801 } 1802 1803 void kvmppc_book3s_exit_pr(void) 1804 { 1805 kvmppc_pr_ops = NULL; 1806 kvmppc_mmu_hpte_sysexit(); 1807 } 1808 1809 /* 1810 * We only support separate modules for book3s 64 1811 */ 1812 #ifdef CONFIG_PPC_BOOK3S_64 1813 1814 module_init(kvmppc_book3s_init_pr); 1815 module_exit(kvmppc_book3s_exit_pr); 1816 1817 MODULE_LICENSE("GPL"); 1818 MODULE_ALIAS_MISCDEV(KVM_MINOR); 1819 MODULE_ALIAS("devname:kvm"); 1820 #endif 1821