Lines Matching +full:fw +full:- +full:cfg +full:- +full:mmio
1 // SPDX-License-Identifier: GPL-2.0-only
52 return !!(v->arch.pending_exceptions) || kvm_request_pending(v); in kvm_arch_vcpu_runnable()
96 vcpu->run->exit_reason = KVM_EXIT_INTR; in kvmppc_prepare_to_enter()
97 r = -EINTR; in kvmppc_prepare_to_enter()
101 vcpu->mode = IN_GUEST_MODE; in kvmppc_prepare_to_enter()
104 * Reading vcpu->requests must happen after setting vcpu->mode, in kvmppc_prepare_to_enter()
144 struct kvm_vcpu_arch_shared *shared = vcpu->arch.shared; in kvmppc_swab_shared()
147 shared->sprg0 = swab64(shared->sprg0); in kvmppc_swab_shared()
148 shared->sprg1 = swab64(shared->sprg1); in kvmppc_swab_shared()
149 shared->sprg2 = swab64(shared->sprg2); in kvmppc_swab_shared()
150 shared->sprg3 = swab64(shared->sprg3); in kvmppc_swab_shared()
151 shared->srr0 = swab64(shared->srr0); in kvmppc_swab_shared()
152 shared->srr1 = swab64(shared->srr1); in kvmppc_swab_shared()
153 shared->dar = swab64(shared->dar); in kvmppc_swab_shared()
154 shared->msr = swab64(shared->msr); in kvmppc_swab_shared()
155 shared->dsisr = swab32(shared->dsisr); in kvmppc_swab_shared()
156 shared->int_pending = swab32(shared->int_pending); in kvmppc_swab_shared()
157 for (i = 0; i < ARRAY_SIZE(shared->sr); i++) in kvmppc_swab_shared()
158 shared->sr[i] = swab32(shared->sr[i]); in kvmppc_swab_shared()
186 if (vcpu->arch.intr_msr & MSR_LE) in kvmppc_kvm_pv()
188 if (shared_big_endian != vcpu->arch.shared_big_endian) in kvmppc_kvm_pv()
190 vcpu->arch.shared_big_endian = shared_big_endian; in kvmppc_kvm_pv()
199 vcpu->arch.disable_kernel_nx = true; in kvmppc_kvm_pv()
203 vcpu->arch.magic_page_pa = param1 & ~0xfffULL; in kvmppc_kvm_pv()
204 vcpu->arch.magic_page_ea = param2 & ~0xfffULL; in kvmppc_kvm_pv()
211 if ((vcpu->arch.magic_page_pa & 0xf000) != in kvmppc_kvm_pv()
212 ((ulong)vcpu->arch.shared & 0xf000)) { in kvmppc_kvm_pv()
213 void *old_shared = vcpu->arch.shared; in kvmppc_kvm_pv()
214 ulong shared = (ulong)vcpu->arch.shared; in kvmppc_kvm_pv()
218 shared |= vcpu->arch.magic_page_pa & 0xf000; in kvmppc_kvm_pv()
221 vcpu->arch.shared = new_shared; in kvmppc_kvm_pv()
258 if (!vcpu->arch.pvr) in kvmppc_sanity_check()
262 if ((vcpu->arch.cpu_type != KVM_CPU_3S_64) && vcpu->arch.papr_enabled) in kvmppc_sanity_check()
266 if (!vcpu->arch.papr_enabled && is_kvmppc_hv_enabled(vcpu->kvm)) in kvmppc_sanity_check()
277 vcpu->arch.sane = r; in kvmppc_sanity_check()
278 return r ? 0 : -EINVAL; in kvmppc_sanity_check()
290 /* Future optimization: only reload non-volatiles if they were in kvmppc_emulate_mmio()
298 vcpu->run->exit_reason = KVM_EXIT_MMIO; in kvmppc_emulate_mmio()
301 /* Future optimization: only reload non-volatiles if they were in kvmppc_emulate_mmio()
321 if (vcpu->mmio_is_write) in kvmppc_emulate_mmio()
326 vcpu->arch.vaddr_accessed, dsisr); in kvmppc_emulate_mmio()
351 ulong mp_pa = vcpu->arch.magic_page_pa & KVM_PAM & PAGE_MASK; in kvmppc_st()
353 int r = -EINVAL; in kvmppc_st()
355 vcpu->stat.st++; in kvmppc_st()
357 if (vcpu->kvm->arch.kvm_ops && vcpu->kvm->arch.kvm_ops->store_to_eaddr) in kvmppc_st()
358 r = vcpu->kvm->arch.kvm_ops->store_to_eaddr(vcpu, eaddr, ptr, in kvmppc_st()
361 if ((!r) || (r == -EAGAIN)) in kvmppc_st()
372 return -EPERM; in kvmppc_st()
378 void *magic = vcpu->arch.shared; in kvmppc_st()
384 if (kvm_write_guest(vcpu->kvm, pte.raddr, ptr, size)) in kvmppc_st()
394 ulong mp_pa = vcpu->arch.magic_page_pa & KVM_PAM & PAGE_MASK; in kvmppc_ld()
396 int rc = -EINVAL; in kvmppc_ld()
398 vcpu->stat.ld++; in kvmppc_ld()
400 if (vcpu->kvm->arch.kvm_ops && vcpu->kvm->arch.kvm_ops->load_from_eaddr) in kvmppc_ld()
401 rc = vcpu->kvm->arch.kvm_ops->load_from_eaddr(vcpu, eaddr, ptr, in kvmppc_ld()
404 if ((!rc) || (rc == -EAGAIN)) in kvmppc_ld()
415 return -EPERM; in kvmppc_ld()
418 return -ENOEXEC; in kvmppc_ld()
424 void *magic = vcpu->arch.shared; in kvmppc_ld()
431 rc = kvm_read_guest(vcpu->kvm, pte.raddr, ptr, size); in kvmppc_ld()
466 if (!try_module_get(kvm_ops->owner)) in kvm_arch_init_vm()
467 return -ENOENT; in kvm_arch_init_vm()
469 kvm->arch.kvm_ops = kvm_ops; in kvm_arch_init_vm()
472 module_put(kvm_ops->owner); in kvm_arch_init_vm()
475 return -EINVAL; in kvm_arch_init_vm()
492 mutex_lock(&kvm->lock); in kvm_arch_destroy_vm()
496 mutex_unlock(&kvm->lock); in kvm_arch_destroy_vm()
499 module_put(kvm->arch.kvm_ops->owner); in kvm_arch_destroy_vm()
510 * Hooray - we know which VM type we're running on. Depend on in kvm_vm_ioctl_check_extension()
594 if (kvm->arch.emul_smt_mode > 1) in kvm_vm_ioctl_check_extension()
595 r = kvm->arch.emul_smt_mode; in kvm_vm_ioctl_check_extension()
597 r = kvm->arch.smt_mode; in kvm_vm_ioctl_check_extension()
609 r = ((threads_per_subcore << 1) - 1); in kvm_vm_ioctl_check_extension()
625 r = !!(hv_enabled && kvmppc_hv_ops->hash_v3_possible && in kvm_vm_ioctl_check_extension()
626 kvmppc_hv_ops->hash_v3_possible()); in kvm_vm_ioctl_check_extension()
629 r = !!(hv_enabled && kvmppc_hv_ops->enable_nested && in kvm_vm_ioctl_check_extension()
630 !kvmppc_hv_ops->enable_nested(NULL)); in kvm_vm_ioctl_check_extension()
645 * return the number of present CPUs for -HV (since a host in kvm_vm_ioctl_check_extension()
678 r = !!(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_HTM) || in kvm_vm_ioctl_check_extension()
684 r = hv_enabled && kvmppc_hv_ops->enable_svm && in kvm_vm_ioctl_check_extension()
685 !kvmppc_hv_ops->enable_svm(NULL); in kvm_vm_ioctl_check_extension()
688 r = !!(hv_enabled && kvmppc_hv_ops->enable_dawr1 && in kvm_vm_ioctl_check_extension()
689 !kvmppc_hv_ops->enable_dawr1(NULL)); in kvm_vm_ioctl_check_extension()
723 return -EINVAL; in kvm_arch_dev_ioctl()
772 hrtimer_init(&vcpu->arch.dec_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS); in kvm_arch_vcpu_create()
773 vcpu->arch.dec_timer.function = kvmppc_decrementer_wakeup; in kvm_arch_vcpu_create()
776 mutex_init(&vcpu->arch.exit_timing_lock); in kvm_arch_vcpu_create()
786 rcuwait_init(&vcpu->arch.wait); in kvm_arch_vcpu_create()
787 vcpu->arch.waitp = &vcpu->arch.wait; in kvm_arch_vcpu_create()
802 hrtimer_cancel(&vcpu->arch.dec_timer); in kvm_arch_vcpu_destroy()
804 switch (vcpu->arch.irq_type) { in kvm_arch_vcpu_destroy()
806 kvmppc_mpic_disconnect_vcpu(vcpu->arch.mpic, vcpu); in kvm_arch_vcpu_destroy()
836 * On non-booke this is associated with Altivec and in kvm_arch_vcpu_load()
839 mtspr(SPRN_VRSAVE, vcpu->arch.vrsave); in kvm_arch_vcpu_load()
848 vcpu->arch.vrsave = mfspr(SPRN_VRSAVE); in kvm_arch_vcpu_put()
860 return ((kvmppc_hv_ops && kvmppc_hv_ops->irq_bypass_add_producer) || in kvm_arch_has_irq_bypass()
861 (kvmppc_pr_ops && kvmppc_pr_ops->irq_bypass_add_producer)); in kvm_arch_has_irq_bypass()
869 struct kvm *kvm = irqfd->kvm; in kvm_arch_irq_bypass_add_producer()
871 if (kvm->arch.kvm_ops->irq_bypass_add_producer) in kvm_arch_irq_bypass_add_producer()
872 return kvm->arch.kvm_ops->irq_bypass_add_producer(cons, prod); in kvm_arch_irq_bypass_add_producer()
882 struct kvm *kvm = irqfd->kvm; in kvm_arch_irq_bypass_del_producer()
884 if (kvm->arch.kvm_ops->irq_bypass_del_producer) in kvm_arch_irq_bypass_del_producer()
885 kvm->arch.kvm_ops->irq_bypass_del_producer(cons, prod); in kvm_arch_irq_bypass_del_producer()
894 return -1; in kvmppc_get_vsr_dword_offset()
899 offset = 1 - index; in kvmppc_get_vsr_dword_offset()
910 return -1; in kvmppc_get_vsr_word_offset()
915 offset = 3 - index; in kvmppc_get_vsr_word_offset()
924 int offset = kvmppc_get_vsr_dword_offset(vcpu->arch.mmio_vsx_offset); in kvmppc_set_vsr_dword()
925 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; in kvmppc_set_vsr_dword()
927 if (offset == -1) in kvmppc_set_vsr_dword()
931 kvmppc_get_vsx_vr(vcpu, index - 32, &val.vval); in kvmppc_set_vsr_dword()
933 kvmppc_set_vsx_vr(vcpu, index - 32, &val.vval); in kvmppc_set_vsr_dword()
943 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; in kvmppc_set_vsr_dword_dump()
946 kvmppc_get_vsx_vr(vcpu, index - 32, &val.vval); in kvmppc_set_vsr_dword_dump()
949 kvmppc_set_vsx_vr(vcpu, index - 32, &val.vval); in kvmppc_set_vsr_dword_dump()
960 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; in kvmppc_set_vsr_word_dump()
967 kvmppc_set_vsx_vr(vcpu, index - 32, &val.vval); in kvmppc_set_vsr_word_dump()
980 int offset = kvmppc_get_vsr_word_offset(vcpu->arch.mmio_vsx_offset); in kvmppc_set_vsr_word()
981 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; in kvmppc_set_vsr_word()
984 if (offset == -1) in kvmppc_set_vsr_word()
988 kvmppc_get_vsx_vr(vcpu, index - 32, &val.vval); in kvmppc_set_vsr_word()
990 kvmppc_set_vsx_vr(vcpu, index - 32, &val.vval); in kvmppc_set_vsr_word()
1009 return -1; in kvmppc_get_vmx_offset_generic()
1012 offset = elts - index - 1; in kvmppc_get_vmx_offset_generic()
1049 vcpu->arch.mmio_vmx_offset); in kvmppc_set_vmx_dword()
1050 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; in kvmppc_set_vmx_dword()
1052 if (offset == -1) in kvmppc_set_vmx_dword()
1065 vcpu->arch.mmio_vmx_offset); in kvmppc_set_vmx_word()
1066 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; in kvmppc_set_vmx_word()
1068 if (offset == -1) in kvmppc_set_vmx_word()
1081 vcpu->arch.mmio_vmx_offset); in kvmppc_set_vmx_hword()
1082 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; in kvmppc_set_vmx_hword()
1084 if (offset == -1) in kvmppc_set_vmx_hword()
1097 vcpu->arch.mmio_vmx_offset); in kvmppc_set_vmx_byte()
1098 int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK; in kvmppc_set_vmx_byte()
1100 if (offset == -1) in kvmppc_set_vmx_byte()
1141 struct kvm_run *run = vcpu->run; in kvmppc_complete_mmio_load()
1144 if (run->mmio.len > sizeof(gpr)) in kvmppc_complete_mmio_load()
1147 if (!vcpu->arch.mmio_host_swabbed) { in kvmppc_complete_mmio_load()
1148 switch (run->mmio.len) { in kvmppc_complete_mmio_load()
1149 case 8: gpr = *(u64 *)run->mmio.data; break; in kvmppc_complete_mmio_load()
1150 case 4: gpr = *(u32 *)run->mmio.data; break; in kvmppc_complete_mmio_load()
1151 case 2: gpr = *(u16 *)run->mmio.data; break; in kvmppc_complete_mmio_load()
1152 case 1: gpr = *(u8 *)run->mmio.data; break; in kvmppc_complete_mmio_load()
1155 switch (run->mmio.len) { in kvmppc_complete_mmio_load()
1156 case 8: gpr = swab64(*(u64 *)run->mmio.data); break; in kvmppc_complete_mmio_load()
1157 case 4: gpr = swab32(*(u32 *)run->mmio.data); break; in kvmppc_complete_mmio_load()
1158 case 2: gpr = swab16(*(u16 *)run->mmio.data); break; in kvmppc_complete_mmio_load()
1159 case 1: gpr = *(u8 *)run->mmio.data; break; in kvmppc_complete_mmio_load()
1164 if ((vcpu->arch.mmio_sp64_extend) && (run->mmio.len == 4)) in kvmppc_complete_mmio_load()
1167 if (vcpu->arch.mmio_sign_extend) { in kvmppc_complete_mmio_load()
1168 switch (run->mmio.len) { in kvmppc_complete_mmio_load()
1183 switch (vcpu->arch.io_gpr & KVM_MMIO_REG_EXT_MASK) { in kvmppc_complete_mmio_load()
1185 kvmppc_set_gpr(vcpu, vcpu->arch.io_gpr, gpr); in kvmppc_complete_mmio_load()
1188 if (vcpu->kvm->arch.kvm_ops->giveup_ext) in kvmppc_complete_mmio_load()
1189 vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_FP); in kvmppc_complete_mmio_load()
1191 kvmppc_set_fpr(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK, gpr); in kvmppc_complete_mmio_load()
1195 vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_MMIO_REG_MASK] = gpr; in kvmppc_complete_mmio_load()
1198 kvmppc_set_fpr(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK, gpr); in kvmppc_complete_mmio_load()
1199 vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_MMIO_REG_MASK] = gpr; in kvmppc_complete_mmio_load()
1204 if (vcpu->kvm->arch.kvm_ops->giveup_ext) in kvmppc_complete_mmio_load()
1205 vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_VSX); in kvmppc_complete_mmio_load()
1207 if (vcpu->arch.mmio_copy_type == KVMPPC_VSX_COPY_DWORD) in kvmppc_complete_mmio_load()
1209 else if (vcpu->arch.mmio_copy_type == KVMPPC_VSX_COPY_WORD) in kvmppc_complete_mmio_load()
1211 else if (vcpu->arch.mmio_copy_type == in kvmppc_complete_mmio_load()
1214 else if (vcpu->arch.mmio_copy_type == in kvmppc_complete_mmio_load()
1221 if (vcpu->kvm->arch.kvm_ops->giveup_ext) in kvmppc_complete_mmio_load()
1222 vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_VEC); in kvmppc_complete_mmio_load()
1224 if (vcpu->arch.mmio_copy_type == KVMPPC_VMX_COPY_DWORD) in kvmppc_complete_mmio_load()
1226 else if (vcpu->arch.mmio_copy_type == KVMPPC_VMX_COPY_WORD) in kvmppc_complete_mmio_load()
1228 else if (vcpu->arch.mmio_copy_type == in kvmppc_complete_mmio_load()
1231 else if (vcpu->arch.mmio_copy_type == in kvmppc_complete_mmio_load()
1240 kvm_vcpu_write_guest(vcpu, vcpu->arch.nested_io_gpr, &gpr, in kvmppc_complete_mmio_load()
1253 struct kvm_run *run = vcpu->run; in __kvmppc_handle_load()
1264 if (bytes > sizeof(run->mmio.data)) in __kvmppc_handle_load()
1267 run->mmio.phys_addr = vcpu->arch.paddr_accessed; in __kvmppc_handle_load()
1268 run->mmio.len = bytes; in __kvmppc_handle_load()
1269 run->mmio.is_write = 0; in __kvmppc_handle_load()
1271 vcpu->arch.io_gpr = rt; in __kvmppc_handle_load()
1272 vcpu->arch.mmio_host_swabbed = host_swabbed; in __kvmppc_handle_load()
1273 vcpu->mmio_needed = 1; in __kvmppc_handle_load()
1274 vcpu->mmio_is_write = 0; in __kvmppc_handle_load()
1275 vcpu->arch.mmio_sign_extend = sign_extend; in __kvmppc_handle_load()
1277 idx = srcu_read_lock(&vcpu->kvm->srcu); in __kvmppc_handle_load()
1279 ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, in __kvmppc_handle_load()
1280 bytes, &run->mmio.data); in __kvmppc_handle_load()
1282 srcu_read_unlock(&vcpu->kvm->srcu, idx); in __kvmppc_handle_load()
1286 vcpu->mmio_needed = 0; in __kvmppc_handle_load()
1317 if (vcpu->arch.mmio_vsx_copy_nums > 4) in kvmppc_handle_vsx_load()
1320 while (vcpu->arch.mmio_vsx_copy_nums) { in kvmppc_handle_vsx_load()
1327 vcpu->arch.paddr_accessed += vcpu->run->mmio.len; in kvmppc_handle_vsx_load()
1329 vcpu->arch.mmio_vsx_copy_nums--; in kvmppc_handle_vsx_load()
1330 vcpu->arch.mmio_vsx_offset++; in kvmppc_handle_vsx_load()
1339 struct kvm_run *run = vcpu->run; in kvmppc_handle_store()
1340 void *data = run->mmio.data; in kvmppc_handle_store()
1351 if (bytes > sizeof(run->mmio.data)) in kvmppc_handle_store()
1354 run->mmio.phys_addr = vcpu->arch.paddr_accessed; in kvmppc_handle_store()
1355 run->mmio.len = bytes; in kvmppc_handle_store()
1356 run->mmio.is_write = 1; in kvmppc_handle_store()
1357 vcpu->mmio_needed = 1; in kvmppc_handle_store()
1358 vcpu->mmio_is_write = 1; in kvmppc_handle_store()
1360 if ((vcpu->arch.mmio_sp64_extend) && (bytes == 4)) in kvmppc_handle_store()
1380 idx = srcu_read_lock(&vcpu->kvm->srcu); in kvmppc_handle_store()
1382 ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, in kvmppc_handle_store()
1383 bytes, &run->mmio.data); in kvmppc_handle_store()
1385 srcu_read_unlock(&vcpu->kvm->srcu, idx); in kvmppc_handle_store()
1388 vcpu->mmio_needed = 0; in kvmppc_handle_store()
1402 int copy_type = vcpu->arch.mmio_copy_type; in kvmppc_get_vsr_data()
1408 kvmppc_get_vsr_dword_offset(vcpu->arch.mmio_vsx_offset); in kvmppc_get_vsr_data()
1410 if (vsx_offset == -1) { in kvmppc_get_vsr_data()
1411 result = -1; in kvmppc_get_vsr_data()
1418 kvmppc_get_vsx_vr(vcpu, rs - 32, ®.vval); in kvmppc_get_vsr_data()
1425 kvmppc_get_vsr_word_offset(vcpu->arch.mmio_vsx_offset); in kvmppc_get_vsr_data()
1427 if (vsx_offset == -1) { in kvmppc_get_vsr_data()
1428 result = -1; in kvmppc_get_vsr_data()
1438 kvmppc_get_vsx_vr(vcpu, rs - 32, ®.vval); in kvmppc_get_vsr_data()
1444 result = -1; in kvmppc_get_vsr_data()
1457 vcpu->arch.io_gpr = rs; in kvmppc_handle_vsx_store()
1460 if (vcpu->arch.mmio_vsx_copy_nums > 4) in kvmppc_handle_vsx_store()
1463 while (vcpu->arch.mmio_vsx_copy_nums) { in kvmppc_handle_vsx_store()
1464 if (kvmppc_get_vsr_data(vcpu, rs, &val) == -1) in kvmppc_handle_vsx_store()
1473 vcpu->arch.paddr_accessed += vcpu->run->mmio.len; in kvmppc_handle_vsx_store()
1475 vcpu->arch.mmio_vsx_copy_nums--; in kvmppc_handle_vsx_store()
1476 vcpu->arch.mmio_vsx_offset++; in kvmppc_handle_vsx_store()
1484 struct kvm_run *run = vcpu->run; in kvmppc_emulate_mmio_vsx_loadstore()
1488 vcpu->arch.paddr_accessed += run->mmio.len; in kvmppc_emulate_mmio_vsx_loadstore()
1490 if (!vcpu->mmio_is_write) { in kvmppc_emulate_mmio_vsx_loadstore()
1491 emulated = kvmppc_handle_vsx_load(vcpu, vcpu->arch.io_gpr, in kvmppc_emulate_mmio_vsx_loadstore()
1492 run->mmio.len, 1, vcpu->arch.mmio_sign_extend); in kvmppc_emulate_mmio_vsx_loadstore()
1495 vcpu->arch.io_gpr, run->mmio.len, 1); in kvmppc_emulate_mmio_vsx_loadstore()
1500 run->exit_reason = KVM_EXIT_MMIO; in kvmppc_emulate_mmio_vsx_loadstore()
1504 pr_info("KVM: MMIO emulation failed (VSX repeat)\n"); in kvmppc_emulate_mmio_vsx_loadstore()
1505 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; in kvmppc_emulate_mmio_vsx_loadstore()
1506 run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; in kvmppc_emulate_mmio_vsx_loadstore()
1523 if (vcpu->arch.mmio_vmx_copy_nums > 2) in kvmppc_handle_vmx_load()
1526 while (vcpu->arch.mmio_vmx_copy_nums) { in kvmppc_handle_vmx_load()
1533 vcpu->arch.paddr_accessed += vcpu->run->mmio.len; in kvmppc_handle_vmx_load()
1534 vcpu->arch.mmio_vmx_copy_nums--; in kvmppc_handle_vmx_load()
1535 vcpu->arch.mmio_vmx_offset++; in kvmppc_handle_vmx_load()
1548 kvmppc_get_vmx_dword_offset(vcpu, vcpu->arch.mmio_vmx_offset); in kvmppc_get_vmx_dword()
1550 if (vmx_offset == -1) in kvmppc_get_vmx_dword()
1551 return -1; in kvmppc_get_vmx_dword()
1566 kvmppc_get_vmx_word_offset(vcpu, vcpu->arch.mmio_vmx_offset); in kvmppc_get_vmx_word()
1568 if (vmx_offset == -1) in kvmppc_get_vmx_word()
1569 return -1; in kvmppc_get_vmx_word()
1584 kvmppc_get_vmx_hword_offset(vcpu, vcpu->arch.mmio_vmx_offset); in kvmppc_get_vmx_hword()
1586 if (vmx_offset == -1) in kvmppc_get_vmx_hword()
1587 return -1; in kvmppc_get_vmx_hword()
1602 kvmppc_get_vmx_byte_offset(vcpu, vcpu->arch.mmio_vmx_offset); in kvmppc_get_vmx_byte()
1604 if (vmx_offset == -1) in kvmppc_get_vmx_byte()
1605 return -1; in kvmppc_get_vmx_byte()
1620 if (vcpu->arch.mmio_vmx_copy_nums > 2) in kvmppc_handle_vmx_store()
1623 vcpu->arch.io_gpr = rs; in kvmppc_handle_vmx_store()
1625 while (vcpu->arch.mmio_vmx_copy_nums) { in kvmppc_handle_vmx_store()
1626 switch (vcpu->arch.mmio_copy_type) { in kvmppc_handle_vmx_store()
1628 if (kvmppc_get_vmx_dword(vcpu, index, &val) == -1) in kvmppc_handle_vmx_store()
1633 if (kvmppc_get_vmx_word(vcpu, index, &val) == -1) in kvmppc_handle_vmx_store()
1637 if (kvmppc_get_vmx_hword(vcpu, index, &val) == -1) in kvmppc_handle_vmx_store()
1641 if (kvmppc_get_vmx_byte(vcpu, index, &val) == -1) in kvmppc_handle_vmx_store()
1653 vcpu->arch.paddr_accessed += vcpu->run->mmio.len; in kvmppc_handle_vmx_store()
1654 vcpu->arch.mmio_vmx_copy_nums--; in kvmppc_handle_vmx_store()
1655 vcpu->arch.mmio_vmx_offset++; in kvmppc_handle_vmx_store()
1663 struct kvm_run *run = vcpu->run; in kvmppc_emulate_mmio_vmx_loadstore()
1667 vcpu->arch.paddr_accessed += run->mmio.len; in kvmppc_emulate_mmio_vmx_loadstore()
1669 if (!vcpu->mmio_is_write) { in kvmppc_emulate_mmio_vmx_loadstore()
1671 vcpu->arch.io_gpr, run->mmio.len, 1); in kvmppc_emulate_mmio_vmx_loadstore()
1674 vcpu->arch.io_gpr, run->mmio.len, 1); in kvmppc_emulate_mmio_vmx_loadstore()
1679 run->exit_reason = KVM_EXIT_MMIO; in kvmppc_emulate_mmio_vmx_loadstore()
1683 pr_info("KVM: MMIO emulation failed (VMX repeat)\n"); in kvmppc_emulate_mmio_vmx_loadstore()
1684 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; in kvmppc_emulate_mmio_vmx_loadstore()
1685 run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; in kvmppc_emulate_mmio_vmx_loadstore()
1702 size = one_reg_size(reg->id); in kvm_vcpu_ioctl_get_one_reg()
1704 return -EINVAL; in kvm_vcpu_ioctl_get_one_reg()
1706 r = kvmppc_get_one_reg(vcpu, reg->id, &val); in kvm_vcpu_ioctl_get_one_reg()
1707 if (r == -EINVAL) { in kvm_vcpu_ioctl_get_one_reg()
1709 switch (reg->id) { in kvm_vcpu_ioctl_get_one_reg()
1713 r = -ENXIO; in kvm_vcpu_ioctl_get_one_reg()
1716 kvmppc_get_vsx_vr(vcpu, reg->id - KVM_REG_PPC_VR0, &val.vval); in kvm_vcpu_ioctl_get_one_reg()
1720 r = -ENXIO; in kvm_vcpu_ioctl_get_one_reg()
1723 val = get_reg_val(reg->id, kvmppc_get_vscr(vcpu)); in kvm_vcpu_ioctl_get_one_reg()
1726 val = get_reg_val(reg->id, kvmppc_get_vrsave(vcpu)); in kvm_vcpu_ioctl_get_one_reg()
1730 r = -EINVAL; in kvm_vcpu_ioctl_get_one_reg()
1738 if (copy_to_user((char __user *)(unsigned long)reg->addr, &val, size)) in kvm_vcpu_ioctl_get_one_reg()
1739 r = -EFAULT; in kvm_vcpu_ioctl_get_one_reg()
1750 size = one_reg_size(reg->id); in kvm_vcpu_ioctl_set_one_reg()
1752 return -EINVAL; in kvm_vcpu_ioctl_set_one_reg()
1754 if (copy_from_user(&val, (char __user *)(unsigned long)reg->addr, size)) in kvm_vcpu_ioctl_set_one_reg()
1755 return -EFAULT; in kvm_vcpu_ioctl_set_one_reg()
1757 r = kvmppc_set_one_reg(vcpu, reg->id, &val); in kvm_vcpu_ioctl_set_one_reg()
1758 if (r == -EINVAL) { in kvm_vcpu_ioctl_set_one_reg()
1760 switch (reg->id) { in kvm_vcpu_ioctl_set_one_reg()
1764 r = -ENXIO; in kvm_vcpu_ioctl_set_one_reg()
1767 kvmppc_set_vsx_vr(vcpu, reg->id - KVM_REG_PPC_VR0, &val.vval); in kvm_vcpu_ioctl_set_one_reg()
1771 r = -ENXIO; in kvm_vcpu_ioctl_set_one_reg()
1774 kvmppc_set_vscr(vcpu, set_reg_val(reg->id, val)); in kvm_vcpu_ioctl_set_one_reg()
1778 r = -ENXIO; in kvm_vcpu_ioctl_set_one_reg()
1781 kvmppc_set_vrsave(vcpu, set_reg_val(reg->id, val)); in kvm_vcpu_ioctl_set_one_reg()
1785 r = -EINVAL; in kvm_vcpu_ioctl_set_one_reg()
1795 struct kvm_run *run = vcpu->run; in kvm_arch_vcpu_ioctl_run()
1800 if (vcpu->mmio_needed) { in kvm_arch_vcpu_ioctl_run()
1801 vcpu->mmio_needed = 0; in kvm_arch_vcpu_ioctl_run()
1802 if (!vcpu->mmio_is_write) in kvm_arch_vcpu_ioctl_run()
1805 if (vcpu->arch.mmio_vsx_copy_nums > 0) { in kvm_arch_vcpu_ioctl_run()
1806 vcpu->arch.mmio_vsx_copy_nums--; in kvm_arch_vcpu_ioctl_run()
1807 vcpu->arch.mmio_vsx_offset++; in kvm_arch_vcpu_ioctl_run()
1810 if (vcpu->arch.mmio_vsx_copy_nums > 0) { in kvm_arch_vcpu_ioctl_run()
1813 vcpu->mmio_needed = 1; in kvm_arch_vcpu_ioctl_run()
1819 if (vcpu->arch.mmio_vmx_copy_nums > 0) { in kvm_arch_vcpu_ioctl_run()
1820 vcpu->arch.mmio_vmx_copy_nums--; in kvm_arch_vcpu_ioctl_run()
1821 vcpu->arch.mmio_vmx_offset++; in kvm_arch_vcpu_ioctl_run()
1824 if (vcpu->arch.mmio_vmx_copy_nums > 0) { in kvm_arch_vcpu_ioctl_run()
1827 vcpu->mmio_needed = 1; in kvm_arch_vcpu_ioctl_run()
1832 } else if (vcpu->arch.osi_needed) { in kvm_arch_vcpu_ioctl_run()
1833 u64 *gprs = run->osi.gprs; in kvm_arch_vcpu_ioctl_run()
1838 vcpu->arch.osi_needed = 0; in kvm_arch_vcpu_ioctl_run()
1839 } else if (vcpu->arch.hcall_needed) { in kvm_arch_vcpu_ioctl_run()
1842 kvmppc_set_gpr(vcpu, 3, run->papr_hcall.ret); in kvm_arch_vcpu_ioctl_run()
1844 kvmppc_set_gpr(vcpu, 4 + i, run->papr_hcall.args[i]); in kvm_arch_vcpu_ioctl_run()
1845 vcpu->arch.hcall_needed = 0; in kvm_arch_vcpu_ioctl_run()
1847 } else if (vcpu->arch.epr_needed) { in kvm_arch_vcpu_ioctl_run()
1848 kvmppc_set_epr(vcpu, run->epr.epr); in kvm_arch_vcpu_ioctl_run()
1849 vcpu->arch.epr_needed = 0; in kvm_arch_vcpu_ioctl_run()
1855 if (!vcpu->wants_to_run) in kvm_arch_vcpu_ioctl_run()
1856 r = -EINTR; in kvm_arch_vcpu_ioctl_run()
1879 if (irq->irq == KVM_INTERRUPT_UNSET) { in kvm_vcpu_ioctl_interrupt()
1896 if (cap->flags) in kvm_vcpu_ioctl_enable_cap()
1897 return -EINVAL; in kvm_vcpu_ioctl_enable_cap()
1899 switch (cap->cap) { in kvm_vcpu_ioctl_enable_cap()
1902 vcpu->arch.osi_enabled = true; in kvm_vcpu_ioctl_enable_cap()
1906 vcpu->arch.papr_enabled = true; in kvm_vcpu_ioctl_enable_cap()
1910 if (cap->args[0]) in kvm_vcpu_ioctl_enable_cap()
1911 vcpu->arch.epr_flags |= KVMPPC_EPR_USER; in kvm_vcpu_ioctl_enable_cap()
1913 vcpu->arch.epr_flags &= ~KVMPPC_EPR_USER; in kvm_vcpu_ioctl_enable_cap()
1918 vcpu->arch.watchdog_enabled = true; in kvm_vcpu_ioctl_enable_cap()
1923 struct kvm_config_tlb cfg; in kvm_vcpu_ioctl_enable_cap() local
1924 void __user *user_ptr = (void __user *)(uintptr_t)cap->args[0]; in kvm_vcpu_ioctl_enable_cap()
1926 r = -EFAULT; in kvm_vcpu_ioctl_enable_cap()
1927 if (copy_from_user(&cfg, user_ptr, sizeof(cfg))) in kvm_vcpu_ioctl_enable_cap()
1930 r = kvm_vcpu_ioctl_config_tlb(vcpu, &cfg); in kvm_vcpu_ioctl_enable_cap()
1939 r = -EBADF; in kvm_vcpu_ioctl_enable_cap()
1940 f = fdget(cap->args[0]); in kvm_vcpu_ioctl_enable_cap()
1944 r = -EPERM; in kvm_vcpu_ioctl_enable_cap()
1947 r = kvmppc_mpic_connect_vcpu(dev, vcpu, cap->args[1]); in kvm_vcpu_ioctl_enable_cap()
1958 r = -EBADF; in kvm_vcpu_ioctl_enable_cap()
1959 f = fdget(cap->args[0]); in kvm_vcpu_ioctl_enable_cap()
1963 r = -EPERM; in kvm_vcpu_ioctl_enable_cap()
1967 r = kvmppc_xive_connect_vcpu(dev, vcpu, cap->args[1]); in kvm_vcpu_ioctl_enable_cap()
1969 r = kvmppc_xics_connect_vcpu(dev, vcpu, cap->args[1]); in kvm_vcpu_ioctl_enable_cap()
1981 r = -EBADF; in kvm_vcpu_ioctl_enable_cap()
1982 f = fdget(cap->args[0]); in kvm_vcpu_ioctl_enable_cap()
1986 r = -ENXIO; in kvm_vcpu_ioctl_enable_cap()
1992 r = -EPERM; in kvm_vcpu_ioctl_enable_cap()
1996 cap->args[1]); in kvm_vcpu_ioctl_enable_cap()
2004 r = -EINVAL; in kvm_vcpu_ioctl_enable_cap()
2005 if (!is_kvmppc_hv_enabled(vcpu->kvm)) in kvm_vcpu_ioctl_enable_cap()
2008 vcpu->kvm->arch.fwnmi_enabled = true; in kvm_vcpu_ioctl_enable_cap()
2012 r = -EINVAL; in kvm_vcpu_ioctl_enable_cap()
2025 if (kvm->arch.mpic) in kvm_arch_intc_initialized()
2029 if (kvm->arch.xics || kvm->arch.xive) in kvm_arch_intc_initialized()
2038 return -EINVAL; in kvm_arch_vcpu_ioctl_get_mpstate()
2044 return -EINVAL; in kvm_arch_vcpu_ioctl_set_mpstate()
2050 struct kvm_vcpu *vcpu = filp->private_data; in kvm_arch_vcpu_async_ioctl()
2056 return -EFAULT; in kvm_arch_vcpu_async_ioctl()
2059 return -ENOIOCTLCMD; in kvm_arch_vcpu_async_ioctl()
2065 struct kvm_vcpu *vcpu = filp->private_data; in kvm_arch_vcpu_ioctl()
2073 r = -EFAULT; in kvm_arch_vcpu_ioctl()
2086 r = -EFAULT; in kvm_arch_vcpu_ioctl()
2099 r = -EFAULT; in kvm_arch_vcpu_ioctl()
2109 r = -EINVAL; in kvm_arch_vcpu_ioctl()
2126 pvinfo->hcall[0] = cpu_to_be32(inst_sc1); in kvm_vm_ioctl_get_pvinfo()
2127 pvinfo->hcall[1] = cpu_to_be32(inst_nop); in kvm_vm_ioctl_get_pvinfo()
2128 pvinfo->hcall[2] = cpu_to_be32(inst_nop); in kvm_vm_ioctl_get_pvinfo()
2129 pvinfo->hcall[3] = cpu_to_be32(inst_nop); in kvm_vm_ioctl_get_pvinfo()
2145 pvinfo->hcall[0] = cpu_to_be32(inst_lis | ((KVM_SC_MAGIC_R0 >> 16) & inst_imm_mask)); in kvm_vm_ioctl_get_pvinfo()
2146 pvinfo->hcall[1] = cpu_to_be32(inst_ori | (KVM_SC_MAGIC_R0 & inst_imm_mask)); in kvm_vm_ioctl_get_pvinfo()
2147 pvinfo->hcall[2] = cpu_to_be32(inst_sc); in kvm_vm_ioctl_get_pvinfo()
2148 pvinfo->hcall[3] = cpu_to_be32(inst_nop); in kvm_vm_ioctl_get_pvinfo()
2151 pvinfo->flags = KVM_PPC_PVINFO_FLAGS_EV_IDLE; in kvm_vm_ioctl_get_pvinfo()
2161 ret = ret || (kvm->arch.mpic != NULL); in kvm_arch_irqchip_in_kernel()
2164 ret = ret || (kvm->arch.xics != NULL); in kvm_arch_irqchip_in_kernel()
2165 ret = ret || (kvm->arch.xive != NULL); in kvm_arch_irqchip_in_kernel()
2175 return -ENXIO; in kvm_vm_ioctl_irq_line()
2177 irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, in kvm_vm_ioctl_irq_line()
2178 irq_event->irq, irq_event->level, in kvm_vm_ioctl_irq_line()
2189 if (cap->flags) in kvm_vm_ioctl_enable_cap()
2190 return -EINVAL; in kvm_vm_ioctl_enable_cap()
2192 switch (cap->cap) { in kvm_vm_ioctl_enable_cap()
2195 unsigned long hcall = cap->args[0]; in kvm_vm_ioctl_enable_cap()
2197 r = -EINVAL; in kvm_vm_ioctl_enable_cap()
2199 cap->args[1] > 1) in kvm_vm_ioctl_enable_cap()
2203 if (cap->args[1]) in kvm_vm_ioctl_enable_cap()
2204 set_bit(hcall / 4, kvm->arch.enabled_hcalls); in kvm_vm_ioctl_enable_cap()
2206 clear_bit(hcall / 4, kvm->arch.enabled_hcalls); in kvm_vm_ioctl_enable_cap()
2211 unsigned long mode = cap->args[0]; in kvm_vm_ioctl_enable_cap()
2212 unsigned long flags = cap->args[1]; in kvm_vm_ioctl_enable_cap()
2214 r = -EINVAL; in kvm_vm_ioctl_enable_cap()
2215 if (kvm->arch.kvm_ops->set_smt_mode) in kvm_vm_ioctl_enable_cap()
2216 r = kvm->arch.kvm_ops->set_smt_mode(kvm, mode, flags); in kvm_vm_ioctl_enable_cap()
2221 r = -EINVAL; in kvm_vm_ioctl_enable_cap()
2223 !kvm->arch.kvm_ops->enable_nested) in kvm_vm_ioctl_enable_cap()
2225 r = kvm->arch.kvm_ops->enable_nested(kvm); in kvm_vm_ioctl_enable_cap()
2230 r = -EINVAL; in kvm_vm_ioctl_enable_cap()
2231 if (!is_kvmppc_hv_enabled(kvm) || !kvm->arch.kvm_ops->enable_svm) in kvm_vm_ioctl_enable_cap()
2233 r = kvm->arch.kvm_ops->enable_svm(kvm); in kvm_vm_ioctl_enable_cap()
2236 r = -EINVAL; in kvm_vm_ioctl_enable_cap()
2237 if (!is_kvmppc_hv_enabled(kvm) || !kvm->arch.kvm_ops->enable_dawr1) in kvm_vm_ioctl_enable_cap()
2239 r = kvm->arch.kvm_ops->enable_dawr1(kvm); in kvm_vm_ioctl_enable_cap()
2243 r = -EINVAL; in kvm_vm_ioctl_enable_cap()
2266 return -ENOTTY; in pseries_get_cpu_char()
2270 cp->character = c.character; in pseries_get_cpu_char()
2271 cp->behaviour = c.behaviour; in pseries_get_cpu_char()
2272 cp->character_mask = KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31 | in pseries_get_cpu_char()
2281 cp->behaviour_mask = KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY | in pseries_get_cpu_char()
2291 return -ENOTTY; in pseries_get_cpu_char()
2316 if (r != -ENOTTY) in kvmppc_get_cpu_char()
2321 fw_features = of_get_child_by_name(np, "fw-features"); in kvmppc_get_cpu_char()
2326 "inst-spec-barrier-ori31,31,0")) in kvmppc_get_cpu_char()
2327 cp->character |= KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31; in kvmppc_get_cpu_char()
2329 "fw-bcctrl-serialized")) in kvmppc_get_cpu_char()
2330 cp->character |= KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED; in kvmppc_get_cpu_char()
2332 "inst-l1d-flush-ori30,30,0")) in kvmppc_get_cpu_char()
2333 cp->character |= KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30; in kvmppc_get_cpu_char()
2335 "inst-l1d-flush-trig2")) in kvmppc_get_cpu_char()
2336 cp->character |= KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2; in kvmppc_get_cpu_char()
2338 "fw-l1d-thread-split")) in kvmppc_get_cpu_char()
2339 cp->character |= KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV; in kvmppc_get_cpu_char()
2341 "fw-count-cache-disabled")) in kvmppc_get_cpu_char()
2342 cp->character |= KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS; in kvmppc_get_cpu_char()
2344 "fw-count-cache-flush-bcctr2,0,0")) in kvmppc_get_cpu_char()
2345 cp->character |= KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST; in kvmppc_get_cpu_char()
2346 cp->character_mask = KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31 | in kvmppc_get_cpu_char()
2355 "speculation-policy-favor-security")) in kvmppc_get_cpu_char()
2356 cp->behaviour |= KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY; in kvmppc_get_cpu_char()
2358 "needs-l1d-flush-msr-pr-0-to-1")) in kvmppc_get_cpu_char()
2359 cp->behaviour |= KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR; in kvmppc_get_cpu_char()
2361 "needs-spec-barrier-for-bound-checks")) in kvmppc_get_cpu_char()
2362 cp->behaviour |= KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR; in kvmppc_get_cpu_char()
2364 "needs-count-cache-flush-on-context-switch")) in kvmppc_get_cpu_char()
2365 cp->behaviour |= KVM_PPC_CPU_BEHAV_FLUSH_COUNT_CACHE; in kvmppc_get_cpu_char()
2366 cp->behaviour_mask = KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY | in kvmppc_get_cpu_char()
2380 struct kvm *kvm __maybe_unused = filp->private_data; in kvm_arch_vm_ioctl()
2390 r = -EFAULT; in kvm_arch_vm_ioctl()
2400 r = -EFAULT; in kvm_arch_vm_ioctl()
2404 r = -EINVAL; in kvm_arch_vm_ioctl()
2414 r = -EFAULT; in kvm_arch_vm_ioctl()
2431 struct kvm *kvm = filp->private_data; in kvm_arch_vm_ioctl()
2434 r = kvm->arch.kvm_ops->get_smmu_info(kvm, &info); in kvm_arch_vm_ioctl()
2436 r = -EFAULT; in kvm_arch_vm_ioctl()
2440 struct kvm *kvm = filp->private_data; in kvm_arch_vm_ioctl()
2446 struct kvm *kvm = filp->private_data; in kvm_arch_vm_ioctl()
2447 struct kvm_ppc_mmuv3_cfg cfg; in kvm_arch_vm_ioctl() local
2449 r = -EINVAL; in kvm_arch_vm_ioctl()
2450 if (!kvm->arch.kvm_ops->configure_mmu) in kvm_arch_vm_ioctl()
2452 r = -EFAULT; in kvm_arch_vm_ioctl()
2453 if (copy_from_user(&cfg, argp, sizeof(cfg))) in kvm_arch_vm_ioctl()
2455 r = kvm->arch.kvm_ops->configure_mmu(kvm, &cfg); in kvm_arch_vm_ioctl()
2459 struct kvm *kvm = filp->private_data; in kvm_arch_vm_ioctl()
2462 r = -EINVAL; in kvm_arch_vm_ioctl()
2463 if (!kvm->arch.kvm_ops->get_rmmu_info) in kvm_arch_vm_ioctl()
2465 r = kvm->arch.kvm_ops->get_rmmu_info(kvm, &info); in kvm_arch_vm_ioctl()
2467 r = -EFAULT; in kvm_arch_vm_ioctl()
2475 r = -EFAULT; in kvm_arch_vm_ioctl()
2479 struct kvm *kvm = filp->private_data; in kvm_arch_vm_ioctl()
2482 if (!kvm->arch.kvm_ops->svm_off) in kvm_arch_vm_ioctl()
2485 r = kvm->arch.kvm_ops->svm_off(kvm); in kvm_arch_vm_ioctl()
2489 struct kvm *kvm = filp->private_data; in kvm_arch_vm_ioctl()
2490 r = kvm->arch.kvm_ops->arch_vm_ioctl(filp, ioctl, arg); in kvm_arch_vm_ioctl()
2494 r = -ENOTTY; in kvm_arch_vm_ioctl()
2509 lpid = ida_alloc_range(&lpid_inuse, 1, nr_lpids - 1, GFP_KERNEL); in kvmppc_alloc_lpid()
2511 if (lpid == -ENOMEM) in kvmppc_alloc_lpid()
2515 return -ENOMEM; in kvmppc_alloc_lpid()
2539 if (vcpu->kvm->arch.kvm_ops->create_vcpu_debugfs) in kvm_arch_create_vcpu_debugfs()
2540 vcpu->kvm->arch.kvm_ops->create_vcpu_debugfs(vcpu, debugfs_dentry); in kvm_arch_create_vcpu_debugfs()
2545 if (kvm->arch.kvm_ops->create_vm_debugfs) in kvm_arch_create_vm_debugfs()
2546 kvm->arch.kvm_ops->create_vm_debugfs(kvm); in kvm_arch_create_vm_debugfs()