1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012,2013 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 * 6 * Derived from arch/arm/kvm/guest.c: 7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 8 * Author: Christoffer Dall <c.dall@virtualopensystems.com> 9 */ 10 11 #include <linux/bits.h> 12 #include <linux/errno.h> 13 #include <linux/err.h> 14 #include <linux/nospec.h> 15 #include <linux/kvm_host.h> 16 #include <linux/module.h> 17 #include <linux/stddef.h> 18 #include <linux/string.h> 19 #include <linux/vmalloc.h> 20 #include <linux/fs.h> 21 #include <kvm/arm_hypercalls.h> 22 #include <asm/cputype.h> 23 #include <linux/uaccess.h> 24 #include <asm/fpsimd.h> 25 #include <asm/kvm.h> 26 #include <asm/kvm_emulate.h> 27 #include <asm/kvm_nested.h> 28 #include <asm/sigcontext.h> 29 30 #include "trace.h" 31 32 const struct kvm_stats_desc kvm_vm_stats_desc[] = { 33 KVM_GENERIC_VM_STATS() 34 }; 35 36 const struct kvm_stats_header kvm_vm_stats_header = { 37 .name_size = KVM_STATS_NAME_SIZE, 38 .num_desc = ARRAY_SIZE(kvm_vm_stats_desc), 39 .id_offset = sizeof(struct kvm_stats_header), 40 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 41 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 42 sizeof(kvm_vm_stats_desc), 43 }; 44 45 const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { 46 KVM_GENERIC_VCPU_STATS(), 47 STATS_DESC_COUNTER(VCPU, hvc_exit_stat), 48 STATS_DESC_COUNTER(VCPU, wfe_exit_stat), 49 STATS_DESC_COUNTER(VCPU, wfi_exit_stat), 50 STATS_DESC_COUNTER(VCPU, mmio_exit_user), 51 STATS_DESC_COUNTER(VCPU, mmio_exit_kernel), 52 STATS_DESC_COUNTER(VCPU, signal_exits), 53 STATS_DESC_COUNTER(VCPU, exits) 54 }; 55 56 const struct kvm_stats_header kvm_vcpu_stats_header = { 57 .name_size = KVM_STATS_NAME_SIZE, 58 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), 59 .id_offset = sizeof(struct kvm_stats_header), 60 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 61 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 62 sizeof(kvm_vcpu_stats_desc), 63 }; 64 65 #if ARM64_S390_COMMON == 1 66 static bool core_reg_offset_is_vreg(u64 off) 67 { 68 return off >= KVM_REG_ARM_CORE_REG(fp_regs.vregs) && 69 off < KVM_REG_ARM_CORE_REG(fp_regs.fpsr); 70 } 71 72 static u64 core_reg_offset_from_id(u64 id) 73 { 74 return id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE); 75 } 76 77 static int core_reg_size_from_offset(const struct kvm_vcpu *vcpu, u64 off) 78 { 79 int size; 80 81 switch (off) { 82 case KVM_REG_ARM_CORE_REG(regs.regs[0]) ... 83 KVM_REG_ARM_CORE_REG(regs.regs[30]): 84 case KVM_REG_ARM_CORE_REG(regs.sp): 85 case KVM_REG_ARM_CORE_REG(regs.pc): 86 case KVM_REG_ARM_CORE_REG(regs.pstate): 87 case KVM_REG_ARM_CORE_REG(sp_el1): 88 case KVM_REG_ARM_CORE_REG(elr_el1): 89 case KVM_REG_ARM_CORE_REG(spsr[0]) ... 90 KVM_REG_ARM_CORE_REG(spsr[KVM_NR_SPSR - 1]): 91 size = sizeof(__u64); 92 break; 93 94 case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ... 95 KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]): 96 size = sizeof(__uint128_t); 97 break; 98 99 case KVM_REG_ARM_CORE_REG(fp_regs.fpsr): 100 case KVM_REG_ARM_CORE_REG(fp_regs.fpcr): 101 size = sizeof(__u32); 102 break; 103 104 default: 105 return -EINVAL; 106 } 107 108 if (!IS_ALIGNED(off, size / sizeof(__u32))) 109 return -EINVAL; 110 111 /* 112 * The KVM_REG_ARM64_SVE regs must be used instead of 113 * KVM_REG_ARM_CORE for accessing the FPSIMD V-registers on 114 * SVE-enabled vcpus: 115 */ 116 if (vcpu_has_sve(vcpu) && core_reg_offset_is_vreg(off)) 117 return -EINVAL; 118 119 return size; 120 } 121 122 static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 123 { 124 u64 off = core_reg_offset_from_id(reg->id); 125 int size = core_reg_size_from_offset(vcpu, off); 126 127 if (size < 0) 128 return NULL; 129 130 if (KVM_REG_SIZE(reg->id) != size) 131 return NULL; 132 133 switch (off) { 134 case KVM_REG_ARM_CORE_REG(regs.regs[0]) ... 135 KVM_REG_ARM_CORE_REG(regs.regs[30]): 136 off -= KVM_REG_ARM_CORE_REG(regs.regs[0]); 137 off /= 2; 138 return &vcpu_gp_regs(vcpu)[off]; 139 140 case KVM_REG_ARM_CORE_REG(regs.sp): 141 return vcpu_sp_el0(vcpu); 142 143 case KVM_REG_ARM_CORE_REG(regs.pc): 144 return vcpu_pc(vcpu); 145 146 case KVM_REG_ARM_CORE_REG(regs.pstate): 147 return vcpu_cpsr(vcpu); 148 149 case KVM_REG_ARM_CORE_REG(sp_el1): 150 return vcpu_sp_el1(vcpu); 151 152 case KVM_REG_ARM_CORE_REG(elr_el1): 153 return __ctxt_sys_reg(&vcpu->arch.ctxt, ELR_EL1); 154 155 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_EL1]): 156 return __ctxt_sys_reg(&vcpu->arch.ctxt, SPSR_EL1); 157 158 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_ABT]): 159 return &vcpu->arch.ctxt.spsr_abt; 160 161 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_UND]): 162 return &vcpu->arch.ctxt.spsr_und; 163 164 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_IRQ]): 165 return &vcpu->arch.ctxt.spsr_irq; 166 167 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_FIQ]): 168 return &vcpu->arch.ctxt.spsr_fiq; 169 170 case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ... 171 KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]): 172 off -= KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]); 173 off /= 4; 174 return vcpu_vreg(vcpu, off); 175 176 case KVM_REG_ARM_CORE_REG(fp_regs.fpsr): 177 return vcpu_fpsr(vcpu); 178 179 case KVM_REG_ARM_CORE_REG(fp_regs.fpcr): 180 return vcpu_fpcr(vcpu); 181 182 default: 183 return NULL; 184 } 185 } 186 187 static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 188 { 189 /* 190 * Because the kvm_regs structure is a mix of 32, 64 and 191 * 128bit fields, we index it as if it was a 32bit 192 * array. Hence below, nr_regs is the number of entries, and 193 * off the index in the "array". 194 */ 195 __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr; 196 int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32); 197 void *addr; 198 u32 off; 199 200 /* Our ID is an index into the kvm_regs struct. */ 201 off = core_reg_offset_from_id(reg->id); 202 if (off >= nr_regs || 203 (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs) 204 return -ENOENT; 205 206 addr = core_reg_addr(vcpu, reg); 207 if (!addr) 208 return -EINVAL; 209 210 if (copy_to_user(uaddr, addr, KVM_REG_SIZE(reg->id))) 211 return -EFAULT; 212 213 return 0; 214 } 215 216 static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 217 { 218 __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr; 219 int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32); 220 __uint128_t tmp; 221 void *valp = &tmp, *addr; 222 u64 off; 223 int err = 0; 224 225 /* Our ID is an index into the kvm_regs struct. */ 226 off = core_reg_offset_from_id(reg->id); 227 if (off >= nr_regs || 228 (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs) 229 return -ENOENT; 230 231 addr = core_reg_addr(vcpu, reg); 232 if (!addr) 233 return -EINVAL; 234 235 if (KVM_REG_SIZE(reg->id) > sizeof(tmp)) 236 return -EINVAL; 237 238 if (copy_from_user(valp, uaddr, KVM_REG_SIZE(reg->id))) { 239 err = -EFAULT; 240 goto out; 241 } 242 243 if (off == KVM_REG_ARM_CORE_REG(regs.pstate)) { 244 u64 mode = (*(u64 *)valp) & PSR_AA32_MODE_MASK; 245 switch (mode) { 246 case PSR_AA32_MODE_USR: 247 if (!kvm_supports_32bit_el0()) 248 return -EINVAL; 249 break; 250 case PSR_AA32_MODE_FIQ: 251 case PSR_AA32_MODE_IRQ: 252 case PSR_AA32_MODE_SVC: 253 case PSR_AA32_MODE_ABT: 254 case PSR_AA32_MODE_UND: 255 case PSR_AA32_MODE_SYS: 256 if (!vcpu_el1_is_32bit(vcpu)) 257 return -EINVAL; 258 break; 259 case PSR_MODE_EL2h: 260 case PSR_MODE_EL2t: 261 if (!vcpu_has_nv(vcpu)) 262 return -EINVAL; 263 fallthrough; 264 case PSR_MODE_EL0t: 265 case PSR_MODE_EL1t: 266 case PSR_MODE_EL1h: 267 if (vcpu_el1_is_32bit(vcpu)) 268 return -EINVAL; 269 break; 270 default: 271 err = -EINVAL; 272 goto out; 273 } 274 } 275 276 memcpy(addr, valp, KVM_REG_SIZE(reg->id)); 277 278 if (*vcpu_cpsr(vcpu) & PSR_MODE32_BIT) { 279 int i, nr_reg; 280 281 switch (*vcpu_cpsr(vcpu) & PSR_AA32_MODE_MASK) { 282 /* 283 * Either we are dealing with user mode, and only the 284 * first 15 registers (+ PC) must be narrowed to 32bit. 285 * AArch32 r0-r14 conveniently map to AArch64 x0-x14. 286 */ 287 case PSR_AA32_MODE_USR: 288 case PSR_AA32_MODE_SYS: 289 nr_reg = 15; 290 break; 291 292 /* 293 * Otherwise, this is a privileged mode, and *all* the 294 * registers must be narrowed to 32bit. 295 */ 296 default: 297 nr_reg = 31; 298 break; 299 } 300 301 for (i = 0; i < nr_reg; i++) 302 vcpu_set_reg(vcpu, i, (u32)vcpu_get_reg(vcpu, i)); 303 304 *vcpu_pc(vcpu) = (u32)*vcpu_pc(vcpu); 305 } 306 out: 307 return err; 308 } 309 310 #endif /* ARM64_S390_COMMON == 1 */ 311 312 #define vq_word(vq) (((vq) - SVE_VQ_MIN) / 64) 313 #define vq_mask(vq) ((u64)1 << ((vq) - SVE_VQ_MIN) % 64) 314 #define vq_present(vqs, vq) (!!((vqs)[vq_word(vq)] & vq_mask(vq))) 315 316 static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 317 { 318 unsigned int max_vq, vq; 319 u64 vqs[KVM_ARM64_SVE_VLS_WORDS]; 320 321 if (!vcpu_has_sve(vcpu)) 322 return -ENOENT; 323 324 if (WARN_ON(!sve_vl_valid(vcpu->arch.sve_max_vl))) 325 return -EINVAL; 326 327 memset(vqs, 0, sizeof(vqs)); 328 329 max_vq = vcpu_sve_max_vq(vcpu); 330 for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq) 331 if (sve_vq_available(vq)) 332 vqs[vq_word(vq)] |= vq_mask(vq); 333 334 if (copy_to_user((void __user *)reg->addr, vqs, sizeof(vqs))) 335 return -EFAULT; 336 337 return 0; 338 } 339 340 static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 341 { 342 unsigned int max_vq, vq; 343 u64 vqs[KVM_ARM64_SVE_VLS_WORDS]; 344 345 if (!vcpu_has_sve(vcpu)) 346 return -ENOENT; 347 348 if (kvm_arm_vcpu_sve_finalized(vcpu)) 349 return -EPERM; /* too late! */ 350 351 if (WARN_ON(vcpu->arch.sve_state)) 352 return -EINVAL; 353 354 if (copy_from_user(vqs, (const void __user *)reg->addr, sizeof(vqs))) 355 return -EFAULT; 356 357 max_vq = 0; 358 for (vq = SVE_VQ_MIN; vq <= SVE_VQ_MAX; ++vq) 359 if (vq_present(vqs, vq)) 360 max_vq = vq; 361 362 if (max_vq > sve_vq_from_vl(kvm_sve_max_vl)) 363 return -EINVAL; 364 365 /* 366 * Vector lengths supported by the host can't currently be 367 * hidden from the guest individually: instead we can only set a 368 * maximum via ZCR_EL2.LEN. So, make sure the available vector 369 * lengths match the set requested exactly up to the requested 370 * maximum: 371 */ 372 for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq) 373 if (vq_present(vqs, vq) != sve_vq_available(vq)) 374 return -EINVAL; 375 376 /* Can't run with no vector lengths at all: */ 377 if (max_vq < SVE_VQ_MIN) 378 return -EINVAL; 379 380 /* vcpu->arch.sve_state will be alloc'd by kvm_vcpu_finalize_sve() */ 381 vcpu->arch.sve_max_vl = sve_vl_from_vq(max_vq); 382 383 return 0; 384 } 385 386 #define SVE_REG_SLICE_SHIFT 0 387 #define SVE_REG_SLICE_BITS 5 388 #define SVE_REG_ID_SHIFT (SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS) 389 #define SVE_REG_ID_BITS 5 390 391 #define SVE_REG_SLICE_MASK \ 392 GENMASK(SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS - 1, \ 393 SVE_REG_SLICE_SHIFT) 394 #define SVE_REG_ID_MASK \ 395 GENMASK(SVE_REG_ID_SHIFT + SVE_REG_ID_BITS - 1, SVE_REG_ID_SHIFT) 396 397 #define SVE_NUM_SLICES (1 << SVE_REG_SLICE_BITS) 398 399 #define KVM_SVE_ZREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_ZREG(0, 0)) 400 #define KVM_SVE_PREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_PREG(0, 0)) 401 402 /* 403 * Number of register slices required to cover each whole SVE register. 404 * NOTE: Only the first slice every exists, for now. 405 * If you are tempted to modify this, you must also rework sve_reg_to_region() 406 * to match: 407 */ 408 #define vcpu_sve_slices(vcpu) 1 409 410 /* Bounds of a single SVE register slice within vcpu->arch.sve_state */ 411 struct sve_state_reg_region { 412 unsigned int koffset; /* offset into sve_state in kernel memory */ 413 unsigned int klen; /* length in kernel memory */ 414 unsigned int upad; /* extra trailing padding in user memory */ 415 }; 416 417 /* 418 * Validate SVE register ID and get sanitised bounds for user/kernel SVE 419 * register copy 420 */ 421 static int sve_reg_to_region(struct sve_state_reg_region *region, 422 struct kvm_vcpu *vcpu, 423 const struct kvm_one_reg *reg) 424 { 425 /* reg ID ranges for Z- registers */ 426 const u64 zreg_id_min = KVM_REG_ARM64_SVE_ZREG(0, 0); 427 const u64 zreg_id_max = KVM_REG_ARM64_SVE_ZREG(SVE_NUM_ZREGS - 1, 428 SVE_NUM_SLICES - 1); 429 430 /* reg ID ranges for P- registers and FFR (which are contiguous) */ 431 const u64 preg_id_min = KVM_REG_ARM64_SVE_PREG(0, 0); 432 const u64 preg_id_max = KVM_REG_ARM64_SVE_FFR(SVE_NUM_SLICES - 1); 433 434 unsigned int vq; 435 unsigned int reg_num; 436 437 unsigned int reqoffset, reqlen; /* User-requested offset and length */ 438 unsigned int maxlen; /* Maximum permitted length */ 439 440 size_t sve_state_size; 441 442 const u64 last_preg_id = KVM_REG_ARM64_SVE_PREG(SVE_NUM_PREGS - 1, 443 SVE_NUM_SLICES - 1); 444 445 /* Verify that the P-regs and FFR really do have contiguous IDs: */ 446 BUILD_BUG_ON(KVM_REG_ARM64_SVE_FFR(0) != last_preg_id + 1); 447 448 /* Verify that we match the UAPI header: */ 449 BUILD_BUG_ON(SVE_NUM_SLICES != KVM_ARM64_SVE_MAX_SLICES); 450 451 reg_num = (reg->id & SVE_REG_ID_MASK) >> SVE_REG_ID_SHIFT; 452 453 if (reg->id >= zreg_id_min && reg->id <= zreg_id_max) { 454 if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0) 455 return -ENOENT; 456 457 vq = vcpu_sve_max_vq(vcpu); 458 459 reqoffset = SVE_SIG_ZREG_OFFSET(vq, reg_num) - 460 SVE_SIG_REGS_OFFSET; 461 reqlen = KVM_SVE_ZREG_SIZE; 462 maxlen = SVE_SIG_ZREG_SIZE(vq); 463 } else if (reg->id >= preg_id_min && reg->id <= preg_id_max) { 464 if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0) 465 return -ENOENT; 466 467 vq = vcpu_sve_max_vq(vcpu); 468 469 reqoffset = SVE_SIG_PREG_OFFSET(vq, reg_num) - 470 SVE_SIG_REGS_OFFSET; 471 reqlen = KVM_SVE_PREG_SIZE; 472 maxlen = SVE_SIG_PREG_SIZE(vq); 473 } else { 474 return -EINVAL; 475 } 476 477 sve_state_size = vcpu_sve_state_size(vcpu); 478 if (WARN_ON(!sve_state_size)) 479 return -EINVAL; 480 481 region->koffset = array_index_nospec(reqoffset, sve_state_size); 482 region->klen = min(maxlen, reqlen); 483 region->upad = reqlen - region->klen; 484 485 return 0; 486 } 487 488 static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 489 { 490 int ret; 491 struct sve_state_reg_region region; 492 char __user *uptr = (char __user *)reg->addr; 493 494 /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */ 495 if (reg->id == KVM_REG_ARM64_SVE_VLS) 496 return get_sve_vls(vcpu, reg); 497 498 /* Try to interpret reg ID as an architectural SVE register... */ 499 ret = sve_reg_to_region(®ion, vcpu, reg); 500 if (ret) 501 return ret; 502 503 if (!kvm_arm_vcpu_sve_finalized(vcpu)) 504 return -EPERM; 505 506 if (copy_to_user(uptr, (void *)vcpu->arch.sve_state + region.koffset, 507 region.klen) || 508 clear_user(uptr + region.klen, region.upad)) 509 return -EFAULT; 510 511 return 0; 512 } 513 514 static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 515 { 516 int ret; 517 struct sve_state_reg_region region; 518 const char __user *uptr = (const char __user *)reg->addr; 519 520 /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */ 521 if (reg->id == KVM_REG_ARM64_SVE_VLS) 522 return set_sve_vls(vcpu, reg); 523 524 /* Try to interpret reg ID as an architectural SVE register... */ 525 ret = sve_reg_to_region(®ion, vcpu, reg); 526 if (ret) 527 return ret; 528 529 if (!kvm_arm_vcpu_sve_finalized(vcpu)) 530 return -EPERM; 531 532 if (copy_from_user((void *)vcpu->arch.sve_state + region.koffset, uptr, 533 region.klen)) 534 return -EFAULT; 535 536 return 0; 537 } 538 539 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 540 { 541 return -EINVAL; 542 } 543 544 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 545 { 546 return -EINVAL; 547 } 548 549 #if ARM64_S390_COMMON == 1 550 static int copy_core_reg_indices(const struct kvm_vcpu *vcpu, 551 u64 __user *uindices) 552 { 553 unsigned int i; 554 int n = 0; 555 556 for (i = 0; i < sizeof(struct kvm_regs) / sizeof(__u32); i++) { 557 u64 reg = KVM_REG_ARM64 | KVM_REG_ARM_CORE | i; 558 int size = core_reg_size_from_offset(vcpu, i); 559 560 if (size < 0) 561 continue; 562 563 switch (size) { 564 case sizeof(__u32): 565 reg |= KVM_REG_SIZE_U32; 566 break; 567 568 case sizeof(__u64): 569 reg |= KVM_REG_SIZE_U64; 570 break; 571 572 case sizeof(__uint128_t): 573 reg |= KVM_REG_SIZE_U128; 574 break; 575 576 default: 577 WARN_ON(1); 578 continue; 579 } 580 581 if (uindices) { 582 if (put_user(reg, uindices)) 583 return -EFAULT; 584 uindices++; 585 } 586 587 n++; 588 } 589 590 return n; 591 } 592 593 static unsigned long num_core_regs(const struct kvm_vcpu *vcpu) 594 { 595 return copy_core_reg_indices(vcpu, NULL); 596 } 597 598 #endif /* ARM64_S390_COMMON == 1 */ 599 600 static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu) 601 { 602 const unsigned int slices = vcpu_sve_slices(vcpu); 603 604 if (!vcpu_has_sve(vcpu)) 605 return 0; 606 607 /* Policed by KVM_GET_REG_LIST: */ 608 WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu)); 609 610 return slices * (SVE_NUM_PREGS + SVE_NUM_ZREGS + 1 /* FFR */) 611 + 1; /* KVM_REG_ARM64_SVE_VLS */ 612 } 613 614 static int copy_sve_reg_indices(const struct kvm_vcpu *vcpu, 615 u64 __user *uindices) 616 { 617 const unsigned int slices = vcpu_sve_slices(vcpu); 618 u64 reg; 619 unsigned int i, n; 620 int num_regs = 0; 621 622 if (!vcpu_has_sve(vcpu)) 623 return 0; 624 625 /* Policed by KVM_GET_REG_LIST: */ 626 WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu)); 627 628 /* 629 * Enumerate this first, so that userspace can save/restore in 630 * the order reported by KVM_GET_REG_LIST: 631 */ 632 reg = KVM_REG_ARM64_SVE_VLS; 633 if (put_user(reg, uindices++)) 634 return -EFAULT; 635 ++num_regs; 636 637 for (i = 0; i < slices; i++) { 638 for (n = 0; n < SVE_NUM_ZREGS; n++) { 639 reg = KVM_REG_ARM64_SVE_ZREG(n, i); 640 if (put_user(reg, uindices++)) 641 return -EFAULT; 642 num_regs++; 643 } 644 645 for (n = 0; n < SVE_NUM_PREGS; n++) { 646 reg = KVM_REG_ARM64_SVE_PREG(n, i); 647 if (put_user(reg, uindices++)) 648 return -EFAULT; 649 num_regs++; 650 } 651 652 reg = KVM_REG_ARM64_SVE_FFR(i); 653 if (put_user(reg, uindices++)) 654 return -EFAULT; 655 num_regs++; 656 } 657 658 return num_regs; 659 } 660 661 /** 662 * kvm_arm_num_regs - how many registers do we present via KVM_GET_ONE_REG 663 * @vcpu: the vCPU pointer 664 * 665 * This is for all registers. 666 */ 667 unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu) 668 { 669 unsigned long res = 0; 670 671 res += num_core_regs(vcpu); 672 res += num_sve_regs(vcpu); 673 res += kvm_arm_num_sys_reg_descs(vcpu); 674 res += kvm_arm_get_fw_num_regs(vcpu); 675 676 return res; 677 } 678 679 /** 680 * kvm_arm_copy_reg_indices - get indices of all registers. 681 * @vcpu: the vCPU pointer 682 * @uindices: register list to copy 683 * 684 * We do core registers right here, then we append system regs. 685 */ 686 int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) 687 { 688 int ret; 689 690 ret = copy_core_reg_indices(vcpu, uindices); 691 if (ret < 0) 692 return ret; 693 uindices += ret; 694 695 ret = copy_sve_reg_indices(vcpu, uindices); 696 if (ret < 0) 697 return ret; 698 uindices += ret; 699 700 ret = kvm_arm_copy_fw_reg_indices(vcpu, uindices); 701 if (ret < 0) 702 return ret; 703 uindices += kvm_arm_get_fw_num_regs(vcpu); 704 705 return kvm_arm_copy_sys_reg_indices(vcpu, uindices); 706 } 707 708 int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 709 { 710 /* We currently use nothing arch-specific in upper 32 bits */ 711 if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) 712 return -EINVAL; 713 714 switch (reg->id & KVM_REG_ARM_COPROC_MASK) { 715 case KVM_REG_ARM_CORE: return get_core_reg(vcpu, reg); 716 case KVM_REG_ARM_FW: 717 case KVM_REG_ARM_FW_FEAT_BMAP: 718 return kvm_arm_get_fw_reg(vcpu, reg); 719 case KVM_REG_ARM64_SVE: return get_sve_reg(vcpu, reg); 720 } 721 722 return kvm_arm_sys_reg_get_reg(vcpu, reg); 723 } 724 725 int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 726 { 727 /* We currently use nothing arch-specific in upper 32 bits */ 728 if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) 729 return -EINVAL; 730 731 switch (reg->id & KVM_REG_ARM_COPROC_MASK) { 732 case KVM_REG_ARM_CORE: return set_core_reg(vcpu, reg); 733 case KVM_REG_ARM_FW: 734 case KVM_REG_ARM_FW_FEAT_BMAP: 735 return kvm_arm_set_fw_reg(vcpu, reg); 736 case KVM_REG_ARM64_SVE: return set_sve_reg(vcpu, reg); 737 } 738 739 return kvm_arm_sys_reg_set_reg(vcpu, reg); 740 } 741 742 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 743 struct kvm_sregs *sregs) 744 { 745 return -EINVAL; 746 } 747 748 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 749 struct kvm_sregs *sregs) 750 { 751 return -EINVAL; 752 } 753 754 int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, 755 struct kvm_vcpu_events *events) 756 { 757 events->exception.serror_has_esr = cpus_have_final_cap(ARM64_HAS_RAS_EXTN); 758 events->exception.serror_pending = (vcpu->arch.hcr_el2 & HCR_VSE) || 759 vcpu_get_flag(vcpu, NESTED_SERROR_PENDING); 760 761 if (events->exception.serror_pending && events->exception.serror_has_esr) 762 events->exception.serror_esr = vcpu_get_vsesr(vcpu); 763 764 /* 765 * We never return a pending ext_dabt here because we deliver it to 766 * the virtual CPU directly when setting the event and it's no longer 767 * 'pending' at this point. 768 */ 769 770 return 0; 771 } 772 773 static void commit_pending_events(struct kvm_vcpu *vcpu) 774 { 775 if (!vcpu_get_flag(vcpu, PENDING_EXCEPTION)) 776 return; 777 778 /* 779 * Reset the MMIO emulation state to avoid stepping PC after emulating 780 * the exception entry. 781 */ 782 vcpu->mmio_needed = false; 783 kvm_call_hyp(__kvm_adjust_pc, vcpu); 784 } 785 786 int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, 787 struct kvm_vcpu_events *events) 788 { 789 bool serror_pending = events->exception.serror_pending; 790 bool has_esr = events->exception.serror_has_esr; 791 bool ext_dabt_pending = events->exception.ext_dabt_pending; 792 u64 esr = events->exception.serror_esr; 793 int ret = 0; 794 795 /* 796 * EL2 injects an external abort only to complete a forwarded abort. 797 * SError injection is forwarded. 798 */ 799 if (vcpu_is_protected(vcpu) && ext_dabt_pending) 800 return -EPERM; 801 802 /* 803 * Immediately commit the pending SEA to the vCPU's architectural 804 * state which is necessary since we do not return a pending SEA 805 * to userspace via KVM_GET_VCPU_EVENTS. 806 */ 807 if (ext_dabt_pending) { 808 ret = kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu)); 809 commit_pending_events(vcpu); 810 } 811 812 if (ret < 0) 813 return ret; 814 815 if (!serror_pending) 816 return 0; 817 818 if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && has_esr) 819 return -EINVAL; 820 821 if (has_esr && (esr & ~ESR_ELx_ISS_MASK)) 822 return -EINVAL; 823 824 if (has_esr) 825 ret = kvm_inject_serror_esr(vcpu, esr); 826 else 827 ret = kvm_inject_serror(vcpu); 828 829 /* 830 * We could've decided that the SError is due for immediate software 831 * injection; commit the exception in case userspace decides it wants 832 * to inject more exceptions for some strange reason. 833 */ 834 commit_pending_events(vcpu); 835 return (ret < 0) ? ret : 0; 836 } 837 838 u32 __attribute_const__ kvm_target_cpu(void) 839 { 840 unsigned long implementor = read_cpuid_implementor(); 841 unsigned long part_number = read_cpuid_part_number(); 842 843 switch (implementor) { 844 case ARM_CPU_IMP_ARM: 845 switch (part_number) { 846 case ARM_CPU_PART_AEM_V8: 847 return KVM_ARM_TARGET_AEM_V8; 848 case ARM_CPU_PART_FOUNDATION: 849 return KVM_ARM_TARGET_FOUNDATION_V8; 850 case ARM_CPU_PART_CORTEX_A53: 851 return KVM_ARM_TARGET_CORTEX_A53; 852 case ARM_CPU_PART_CORTEX_A57: 853 return KVM_ARM_TARGET_CORTEX_A57; 854 } 855 break; 856 case ARM_CPU_IMP_APM: 857 switch (part_number) { 858 case APM_CPU_PART_XGENE: 859 return KVM_ARM_TARGET_XGENE_POTENZA; 860 } 861 break; 862 } 863 864 /* Return a default generic target */ 865 return KVM_ARM_TARGET_GENERIC_V8; 866 } 867 868 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 869 { 870 return -EINVAL; 871 } 872 873 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 874 { 875 return -EINVAL; 876 } 877 878 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 879 struct kvm_translation *tr) 880 { 881 return -EINVAL; 882 } 883 884 /** 885 * kvm_arch_vcpu_ioctl_set_guest_debug - set up guest debugging 886 * @vcpu: the vCPU pointer 887 * @dbg: the ioctl data buffer 888 * 889 * This sets up and enables the VM for guest debugging. Userspace 890 * passes in a control flag to enable different debug types and 891 * potentially other architecture specific information in the rest of 892 * the structure. 893 */ 894 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 895 struct kvm_guest_debug *dbg) 896 { 897 trace_kvm_set_guest_debug(vcpu, dbg->control); 898 899 /* A protected guest's debug state is not exposed to the host. */ 900 if (vcpu_is_protected(vcpu)) 901 return -EPERM; 902 903 if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) 904 return -EINVAL; 905 906 if (!(dbg->control & KVM_GUESTDBG_ENABLE)) { 907 vcpu->guest_debug = 0; 908 vcpu_clear_flag(vcpu, HOST_SS_ACTIVE_PENDING); 909 return 0; 910 } 911 912 vcpu->guest_debug = dbg->control; 913 914 /* Hardware assisted Break and Watch points */ 915 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW) 916 vcpu->arch.external_debug_state = dbg->arch; 917 918 return 0; 919 } 920 921 int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu, 922 struct kvm_device_attr *attr) 923 { 924 int ret; 925 926 switch (attr->group) { 927 case KVM_ARM_VCPU_PMU_V3_CTRL: 928 mutex_lock(&vcpu->kvm->arch.config_lock); 929 ret = kvm_arm_pmu_v3_set_attr(vcpu, attr); 930 mutex_unlock(&vcpu->kvm->arch.config_lock); 931 break; 932 case KVM_ARM_VCPU_TIMER_CTRL: 933 ret = kvm_arm_timer_set_attr(vcpu, attr); 934 break; 935 case KVM_ARM_VCPU_PVTIME_CTRL: 936 ret = kvm_arm_pvtime_set_attr(vcpu, attr); 937 break; 938 default: 939 ret = -ENXIO; 940 break; 941 } 942 943 return ret; 944 } 945 946 int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu, 947 struct kvm_device_attr *attr) 948 { 949 int ret; 950 951 switch (attr->group) { 952 case KVM_ARM_VCPU_PMU_V3_CTRL: 953 ret = kvm_arm_pmu_v3_get_attr(vcpu, attr); 954 break; 955 case KVM_ARM_VCPU_TIMER_CTRL: 956 ret = kvm_arm_timer_get_attr(vcpu, attr); 957 break; 958 case KVM_ARM_VCPU_PVTIME_CTRL: 959 ret = kvm_arm_pvtime_get_attr(vcpu, attr); 960 break; 961 default: 962 ret = -ENXIO; 963 break; 964 } 965 966 return ret; 967 } 968 969 int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu, 970 struct kvm_device_attr *attr) 971 { 972 int ret; 973 974 switch (attr->group) { 975 case KVM_ARM_VCPU_PMU_V3_CTRL: 976 ret = kvm_arm_pmu_v3_has_attr(vcpu, attr); 977 break; 978 case KVM_ARM_VCPU_TIMER_CTRL: 979 ret = kvm_arm_timer_has_attr(vcpu, attr); 980 break; 981 case KVM_ARM_VCPU_PVTIME_CTRL: 982 ret = kvm_arm_pvtime_has_attr(vcpu, attr); 983 break; 984 default: 985 ret = -ENXIO; 986 break; 987 } 988 989 return ret; 990 } 991 992 int kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm, 993 struct kvm_arm_copy_mte_tags *copy_tags) 994 { 995 gpa_t guest_ipa = copy_tags->guest_ipa; 996 size_t length = copy_tags->length; 997 void __user *tags = copy_tags->addr; 998 gpa_t gfn; 999 bool write = !(copy_tags->flags & KVM_ARM_TAGS_FROM_GUEST); 1000 int ret = 0; 1001 1002 if (!kvm_has_mte(kvm)) 1003 return -EINVAL; 1004 1005 if (copy_tags->reserved[0] || copy_tags->reserved[1]) 1006 return -EINVAL; 1007 1008 if (copy_tags->flags & ~KVM_ARM_TAGS_FROM_GUEST) 1009 return -EINVAL; 1010 1011 if (length & ~PAGE_MASK || guest_ipa & ~PAGE_MASK) 1012 return -EINVAL; 1013 1014 /* Lengths above INT_MAX cannot be represented in the return value */ 1015 if (length > INT_MAX) 1016 return -EINVAL; 1017 1018 gfn = gpa_to_gfn(guest_ipa); 1019 1020 mutex_lock(&kvm->slots_lock); 1021 1022 if (write && atomic_read(&kvm->nr_memslots_dirty_logging)) { 1023 ret = -EBUSY; 1024 goto out; 1025 } 1026 1027 while (length > 0) { 1028 struct page *page = __gfn_to_page(kvm, gfn, write); 1029 void *maddr; 1030 unsigned long num_tags; 1031 struct folio *folio; 1032 1033 if (!page) { 1034 ret = -EFAULT; 1035 goto out; 1036 } 1037 1038 if (!pfn_to_online_page(page_to_pfn(page))) { 1039 /* Reject ZONE_DEVICE memory */ 1040 kvm_release_page_unused(page); 1041 ret = -EFAULT; 1042 goto out; 1043 } 1044 folio = page_folio(page); 1045 maddr = page_address(page); 1046 1047 if (!write) { 1048 if ((folio_test_hugetlb(folio) && 1049 folio_test_hugetlb_mte_tagged(folio)) || 1050 page_mte_tagged(page)) 1051 num_tags = mte_copy_tags_to_user(tags, maddr, 1052 MTE_GRANULES_PER_PAGE); 1053 else 1054 /* No tags in memory, so write zeros */ 1055 num_tags = MTE_GRANULES_PER_PAGE - 1056 clear_user(tags, MTE_GRANULES_PER_PAGE); 1057 kvm_release_page_clean(page); 1058 } else { 1059 /* 1060 * Only locking to serialise with a concurrent 1061 * __set_ptes() in the VMM but still overriding the 1062 * tags, hence ignoring the return value. 1063 */ 1064 if (folio_test_hugetlb(folio)) 1065 folio_try_hugetlb_mte_tagging(folio); 1066 else 1067 try_page_mte_tagging(page); 1068 num_tags = mte_copy_tags_from_user(maddr, tags, 1069 MTE_GRANULES_PER_PAGE); 1070 1071 /* uaccess failed, don't leave stale tags */ 1072 if (num_tags != MTE_GRANULES_PER_PAGE) 1073 mte_clear_page_tags(maddr); 1074 if (folio_test_hugetlb(folio)) 1075 folio_set_hugetlb_mte_tagged(folio); 1076 else 1077 set_page_mte_tagged(page); 1078 1079 kvm_release_page_dirty(page); 1080 } 1081 1082 if (num_tags != MTE_GRANULES_PER_PAGE) { 1083 ret = -EFAULT; 1084 goto out; 1085 } 1086 1087 gfn++; 1088 tags += num_tags; 1089 length -= PAGE_SIZE; 1090 } 1091 1092 out: 1093 mutex_unlock(&kvm->slots_lock); 1094 /* If some data has been copied report the number of bytes copied */ 1095 if (length != copy_tags->length) 1096 return copy_tags->length - length; 1097 return ret; 1098 } 1099