1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012,2013 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 * 6 * Derived from arch/arm/kvm/guest.c: 7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University 8 * Author: Christoffer Dall <c.dall@virtualopensystems.com> 9 */ 10 11 #include <linux/bits.h> 12 #include <linux/errno.h> 13 #include <linux/err.h> 14 #include <linux/nospec.h> 15 #include <linux/kvm_host.h> 16 #include <linux/module.h> 17 #include <linux/stddef.h> 18 #include <linux/string.h> 19 #include <linux/vmalloc.h> 20 #include <linux/fs.h> 21 #include <kvm/arm_hypercalls.h> 22 #include <asm/cputype.h> 23 #include <linux/uaccess.h> 24 #include <asm/fpsimd.h> 25 #include <asm/kvm.h> 26 #include <asm/kvm_emulate.h> 27 #include <asm/kvm_nested.h> 28 #include <asm/sigcontext.h> 29 30 #include "trace.h" 31 32 const struct _kvm_stats_desc kvm_vm_stats_desc[] = { 33 KVM_GENERIC_VM_STATS() 34 }; 35 36 const struct kvm_stats_header kvm_vm_stats_header = { 37 .name_size = KVM_STATS_NAME_SIZE, 38 .num_desc = ARRAY_SIZE(kvm_vm_stats_desc), 39 .id_offset = sizeof(struct kvm_stats_header), 40 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 41 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 42 sizeof(kvm_vm_stats_desc), 43 }; 44 45 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { 46 KVM_GENERIC_VCPU_STATS(), 47 STATS_DESC_COUNTER(VCPU, hvc_exit_stat), 48 STATS_DESC_COUNTER(VCPU, wfe_exit_stat), 49 STATS_DESC_COUNTER(VCPU, wfi_exit_stat), 50 STATS_DESC_COUNTER(VCPU, mmio_exit_user), 51 STATS_DESC_COUNTER(VCPU, mmio_exit_kernel), 52 STATS_DESC_COUNTER(VCPU, signal_exits), 53 STATS_DESC_COUNTER(VCPU, exits) 54 }; 55 56 const struct kvm_stats_header kvm_vcpu_stats_header = { 57 .name_size = KVM_STATS_NAME_SIZE, 58 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), 59 .id_offset = sizeof(struct kvm_stats_header), 60 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 61 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 62 sizeof(kvm_vcpu_stats_desc), 63 }; 64 65 static bool core_reg_offset_is_vreg(u64 off) 66 { 67 return off >= KVM_REG_ARM_CORE_REG(fp_regs.vregs) && 68 off < KVM_REG_ARM_CORE_REG(fp_regs.fpsr); 69 } 70 71 static u64 core_reg_offset_from_id(u64 id) 72 { 73 return id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE); 74 } 75 76 static int core_reg_size_from_offset(const struct kvm_vcpu *vcpu, u64 off) 77 { 78 int size; 79 80 switch (off) { 81 case KVM_REG_ARM_CORE_REG(regs.regs[0]) ... 82 KVM_REG_ARM_CORE_REG(regs.regs[30]): 83 case KVM_REG_ARM_CORE_REG(regs.sp): 84 case KVM_REG_ARM_CORE_REG(regs.pc): 85 case KVM_REG_ARM_CORE_REG(regs.pstate): 86 case KVM_REG_ARM_CORE_REG(sp_el1): 87 case KVM_REG_ARM_CORE_REG(elr_el1): 88 case KVM_REG_ARM_CORE_REG(spsr[0]) ... 89 KVM_REG_ARM_CORE_REG(spsr[KVM_NR_SPSR - 1]): 90 size = sizeof(__u64); 91 break; 92 93 case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ... 94 KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]): 95 size = sizeof(__uint128_t); 96 break; 97 98 case KVM_REG_ARM_CORE_REG(fp_regs.fpsr): 99 case KVM_REG_ARM_CORE_REG(fp_regs.fpcr): 100 size = sizeof(__u32); 101 break; 102 103 default: 104 return -EINVAL; 105 } 106 107 if (!IS_ALIGNED(off, size / sizeof(__u32))) 108 return -EINVAL; 109 110 /* 111 * The KVM_REG_ARM64_SVE regs must be used instead of 112 * KVM_REG_ARM_CORE for accessing the FPSIMD V-registers on 113 * SVE-enabled vcpus: 114 */ 115 if (vcpu_has_sve(vcpu) && core_reg_offset_is_vreg(off)) 116 return -EINVAL; 117 118 return size; 119 } 120 121 static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 122 { 123 u64 off = core_reg_offset_from_id(reg->id); 124 int size = core_reg_size_from_offset(vcpu, off); 125 126 if (size < 0) 127 return NULL; 128 129 if (KVM_REG_SIZE(reg->id) != size) 130 return NULL; 131 132 switch (off) { 133 case KVM_REG_ARM_CORE_REG(regs.regs[0]) ... 134 KVM_REG_ARM_CORE_REG(regs.regs[30]): 135 off -= KVM_REG_ARM_CORE_REG(regs.regs[0]); 136 off /= 2; 137 return &vcpu->arch.ctxt.regs.regs[off]; 138 139 case KVM_REG_ARM_CORE_REG(regs.sp): 140 return &vcpu->arch.ctxt.regs.sp; 141 142 case KVM_REG_ARM_CORE_REG(regs.pc): 143 return &vcpu->arch.ctxt.regs.pc; 144 145 case KVM_REG_ARM_CORE_REG(regs.pstate): 146 return &vcpu->arch.ctxt.regs.pstate; 147 148 case KVM_REG_ARM_CORE_REG(sp_el1): 149 return __ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1); 150 151 case KVM_REG_ARM_CORE_REG(elr_el1): 152 return __ctxt_sys_reg(&vcpu->arch.ctxt, ELR_EL1); 153 154 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_EL1]): 155 return __ctxt_sys_reg(&vcpu->arch.ctxt, SPSR_EL1); 156 157 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_ABT]): 158 return &vcpu->arch.ctxt.spsr_abt; 159 160 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_UND]): 161 return &vcpu->arch.ctxt.spsr_und; 162 163 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_IRQ]): 164 return &vcpu->arch.ctxt.spsr_irq; 165 166 case KVM_REG_ARM_CORE_REG(spsr[KVM_SPSR_FIQ]): 167 return &vcpu->arch.ctxt.spsr_fiq; 168 169 case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ... 170 KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]): 171 off -= KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]); 172 off /= 4; 173 return &vcpu->arch.ctxt.fp_regs.vregs[off]; 174 175 case KVM_REG_ARM_CORE_REG(fp_regs.fpsr): 176 return &vcpu->arch.ctxt.fp_regs.fpsr; 177 178 case KVM_REG_ARM_CORE_REG(fp_regs.fpcr): 179 return &vcpu->arch.ctxt.fp_regs.fpcr; 180 181 default: 182 return NULL; 183 } 184 } 185 186 static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 187 { 188 /* 189 * Because the kvm_regs structure is a mix of 32, 64 and 190 * 128bit fields, we index it as if it was a 32bit 191 * array. Hence below, nr_regs is the number of entries, and 192 * off the index in the "array". 193 */ 194 __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr; 195 int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32); 196 void *addr; 197 u32 off; 198 199 /* Our ID is an index into the kvm_regs struct. */ 200 off = core_reg_offset_from_id(reg->id); 201 if (off >= nr_regs || 202 (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs) 203 return -ENOENT; 204 205 addr = core_reg_addr(vcpu, reg); 206 if (!addr) 207 return -EINVAL; 208 209 if (copy_to_user(uaddr, addr, KVM_REG_SIZE(reg->id))) 210 return -EFAULT; 211 212 return 0; 213 } 214 215 static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 216 { 217 __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr; 218 int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32); 219 __uint128_t tmp; 220 void *valp = &tmp, *addr; 221 u64 off; 222 int err = 0; 223 224 /* Our ID is an index into the kvm_regs struct. */ 225 off = core_reg_offset_from_id(reg->id); 226 if (off >= nr_regs || 227 (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs) 228 return -ENOENT; 229 230 addr = core_reg_addr(vcpu, reg); 231 if (!addr) 232 return -EINVAL; 233 234 if (KVM_REG_SIZE(reg->id) > sizeof(tmp)) 235 return -EINVAL; 236 237 if (copy_from_user(valp, uaddr, KVM_REG_SIZE(reg->id))) { 238 err = -EFAULT; 239 goto out; 240 } 241 242 if (off == KVM_REG_ARM_CORE_REG(regs.pstate)) { 243 u64 mode = (*(u64 *)valp) & PSR_AA32_MODE_MASK; 244 switch (mode) { 245 case PSR_AA32_MODE_USR: 246 if (!kvm_supports_32bit_el0()) 247 return -EINVAL; 248 break; 249 case PSR_AA32_MODE_FIQ: 250 case PSR_AA32_MODE_IRQ: 251 case PSR_AA32_MODE_SVC: 252 case PSR_AA32_MODE_ABT: 253 case PSR_AA32_MODE_UND: 254 case PSR_AA32_MODE_SYS: 255 if (!vcpu_el1_is_32bit(vcpu)) 256 return -EINVAL; 257 break; 258 case PSR_MODE_EL2h: 259 case PSR_MODE_EL2t: 260 if (!vcpu_has_nv(vcpu)) 261 return -EINVAL; 262 fallthrough; 263 case PSR_MODE_EL0t: 264 case PSR_MODE_EL1t: 265 case PSR_MODE_EL1h: 266 if (vcpu_el1_is_32bit(vcpu)) 267 return -EINVAL; 268 break; 269 default: 270 err = -EINVAL; 271 goto out; 272 } 273 } 274 275 memcpy(addr, valp, KVM_REG_SIZE(reg->id)); 276 277 if (*vcpu_cpsr(vcpu) & PSR_MODE32_BIT) { 278 int i, nr_reg; 279 280 switch (*vcpu_cpsr(vcpu) & PSR_AA32_MODE_MASK) { 281 /* 282 * Either we are dealing with user mode, and only the 283 * first 15 registers (+ PC) must be narrowed to 32bit. 284 * AArch32 r0-r14 conveniently map to AArch64 x0-x14. 285 */ 286 case PSR_AA32_MODE_USR: 287 case PSR_AA32_MODE_SYS: 288 nr_reg = 15; 289 break; 290 291 /* 292 * Otherwise, this is a privileged mode, and *all* the 293 * registers must be narrowed to 32bit. 294 */ 295 default: 296 nr_reg = 31; 297 break; 298 } 299 300 for (i = 0; i < nr_reg; i++) 301 vcpu_set_reg(vcpu, i, (u32)vcpu_get_reg(vcpu, i)); 302 303 *vcpu_pc(vcpu) = (u32)*vcpu_pc(vcpu); 304 } 305 out: 306 return err; 307 } 308 309 #define vq_word(vq) (((vq) - SVE_VQ_MIN) / 64) 310 #define vq_mask(vq) ((u64)1 << ((vq) - SVE_VQ_MIN) % 64) 311 #define vq_present(vqs, vq) (!!((vqs)[vq_word(vq)] & vq_mask(vq))) 312 313 static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 314 { 315 unsigned int max_vq, vq; 316 u64 vqs[KVM_ARM64_SVE_VLS_WORDS]; 317 318 if (!vcpu_has_sve(vcpu)) 319 return -ENOENT; 320 321 if (WARN_ON(!sve_vl_valid(vcpu->arch.sve_max_vl))) 322 return -EINVAL; 323 324 memset(vqs, 0, sizeof(vqs)); 325 326 max_vq = vcpu_sve_max_vq(vcpu); 327 for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq) 328 if (sve_vq_available(vq)) 329 vqs[vq_word(vq)] |= vq_mask(vq); 330 331 if (copy_to_user((void __user *)reg->addr, vqs, sizeof(vqs))) 332 return -EFAULT; 333 334 return 0; 335 } 336 337 static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 338 { 339 unsigned int max_vq, vq; 340 u64 vqs[KVM_ARM64_SVE_VLS_WORDS]; 341 342 if (!vcpu_has_sve(vcpu)) 343 return -ENOENT; 344 345 if (kvm_arm_vcpu_sve_finalized(vcpu)) 346 return -EPERM; /* too late! */ 347 348 if (WARN_ON(vcpu->arch.sve_state)) 349 return -EINVAL; 350 351 if (copy_from_user(vqs, (const void __user *)reg->addr, sizeof(vqs))) 352 return -EFAULT; 353 354 max_vq = 0; 355 for (vq = SVE_VQ_MIN; vq <= SVE_VQ_MAX; ++vq) 356 if (vq_present(vqs, vq)) 357 max_vq = vq; 358 359 if (max_vq > sve_vq_from_vl(kvm_sve_max_vl)) 360 return -EINVAL; 361 362 /* 363 * Vector lengths supported by the host can't currently be 364 * hidden from the guest individually: instead we can only set a 365 * maximum via ZCR_EL2.LEN. So, make sure the available vector 366 * lengths match the set requested exactly up to the requested 367 * maximum: 368 */ 369 for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq) 370 if (vq_present(vqs, vq) != sve_vq_available(vq)) 371 return -EINVAL; 372 373 /* Can't run with no vector lengths at all: */ 374 if (max_vq < SVE_VQ_MIN) 375 return -EINVAL; 376 377 /* vcpu->arch.sve_state will be alloc'd by kvm_vcpu_finalize_sve() */ 378 vcpu->arch.sve_max_vl = sve_vl_from_vq(max_vq); 379 380 return 0; 381 } 382 383 #define SVE_REG_SLICE_SHIFT 0 384 #define SVE_REG_SLICE_BITS 5 385 #define SVE_REG_ID_SHIFT (SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS) 386 #define SVE_REG_ID_BITS 5 387 388 #define SVE_REG_SLICE_MASK \ 389 GENMASK(SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS - 1, \ 390 SVE_REG_SLICE_SHIFT) 391 #define SVE_REG_ID_MASK \ 392 GENMASK(SVE_REG_ID_SHIFT + SVE_REG_ID_BITS - 1, SVE_REG_ID_SHIFT) 393 394 #define SVE_NUM_SLICES (1 << SVE_REG_SLICE_BITS) 395 396 #define KVM_SVE_ZREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_ZREG(0, 0)) 397 #define KVM_SVE_PREG_SIZE KVM_REG_SIZE(KVM_REG_ARM64_SVE_PREG(0, 0)) 398 399 /* 400 * Number of register slices required to cover each whole SVE register. 401 * NOTE: Only the first slice every exists, for now. 402 * If you are tempted to modify this, you must also rework sve_reg_to_region() 403 * to match: 404 */ 405 #define vcpu_sve_slices(vcpu) 1 406 407 /* Bounds of a single SVE register slice within vcpu->arch.sve_state */ 408 struct sve_state_reg_region { 409 unsigned int koffset; /* offset into sve_state in kernel memory */ 410 unsigned int klen; /* length in kernel memory */ 411 unsigned int upad; /* extra trailing padding in user memory */ 412 }; 413 414 /* 415 * Validate SVE register ID and get sanitised bounds for user/kernel SVE 416 * register copy 417 */ 418 static int sve_reg_to_region(struct sve_state_reg_region *region, 419 struct kvm_vcpu *vcpu, 420 const struct kvm_one_reg *reg) 421 { 422 /* reg ID ranges for Z- registers */ 423 const u64 zreg_id_min = KVM_REG_ARM64_SVE_ZREG(0, 0); 424 const u64 zreg_id_max = KVM_REG_ARM64_SVE_ZREG(SVE_NUM_ZREGS - 1, 425 SVE_NUM_SLICES - 1); 426 427 /* reg ID ranges for P- registers and FFR (which are contiguous) */ 428 const u64 preg_id_min = KVM_REG_ARM64_SVE_PREG(0, 0); 429 const u64 preg_id_max = KVM_REG_ARM64_SVE_FFR(SVE_NUM_SLICES - 1); 430 431 unsigned int vq; 432 unsigned int reg_num; 433 434 unsigned int reqoffset, reqlen; /* User-requested offset and length */ 435 unsigned int maxlen; /* Maximum permitted length */ 436 437 size_t sve_state_size; 438 439 const u64 last_preg_id = KVM_REG_ARM64_SVE_PREG(SVE_NUM_PREGS - 1, 440 SVE_NUM_SLICES - 1); 441 442 /* Verify that the P-regs and FFR really do have contiguous IDs: */ 443 BUILD_BUG_ON(KVM_REG_ARM64_SVE_FFR(0) != last_preg_id + 1); 444 445 /* Verify that we match the UAPI header: */ 446 BUILD_BUG_ON(SVE_NUM_SLICES != KVM_ARM64_SVE_MAX_SLICES); 447 448 reg_num = (reg->id & SVE_REG_ID_MASK) >> SVE_REG_ID_SHIFT; 449 450 if (reg->id >= zreg_id_min && reg->id <= zreg_id_max) { 451 if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0) 452 return -ENOENT; 453 454 vq = vcpu_sve_max_vq(vcpu); 455 456 reqoffset = SVE_SIG_ZREG_OFFSET(vq, reg_num) - 457 SVE_SIG_REGS_OFFSET; 458 reqlen = KVM_SVE_ZREG_SIZE; 459 maxlen = SVE_SIG_ZREG_SIZE(vq); 460 } else if (reg->id >= preg_id_min && reg->id <= preg_id_max) { 461 if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0) 462 return -ENOENT; 463 464 vq = vcpu_sve_max_vq(vcpu); 465 466 reqoffset = SVE_SIG_PREG_OFFSET(vq, reg_num) - 467 SVE_SIG_REGS_OFFSET; 468 reqlen = KVM_SVE_PREG_SIZE; 469 maxlen = SVE_SIG_PREG_SIZE(vq); 470 } else { 471 return -EINVAL; 472 } 473 474 sve_state_size = vcpu_sve_state_size(vcpu); 475 if (WARN_ON(!sve_state_size)) 476 return -EINVAL; 477 478 region->koffset = array_index_nospec(reqoffset, sve_state_size); 479 region->klen = min(maxlen, reqlen); 480 region->upad = reqlen - region->klen; 481 482 return 0; 483 } 484 485 static int get_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 486 { 487 int ret; 488 struct sve_state_reg_region region; 489 char __user *uptr = (char __user *)reg->addr; 490 491 /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */ 492 if (reg->id == KVM_REG_ARM64_SVE_VLS) 493 return get_sve_vls(vcpu, reg); 494 495 /* Try to interpret reg ID as an architectural SVE register... */ 496 ret = sve_reg_to_region(®ion, vcpu, reg); 497 if (ret) 498 return ret; 499 500 if (!kvm_arm_vcpu_sve_finalized(vcpu)) 501 return -EPERM; 502 503 if (copy_to_user(uptr, vcpu->arch.sve_state + region.koffset, 504 region.klen) || 505 clear_user(uptr + region.klen, region.upad)) 506 return -EFAULT; 507 508 return 0; 509 } 510 511 static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 512 { 513 int ret; 514 struct sve_state_reg_region region; 515 const char __user *uptr = (const char __user *)reg->addr; 516 517 /* Handle the KVM_REG_ARM64_SVE_VLS pseudo-reg as a special case: */ 518 if (reg->id == KVM_REG_ARM64_SVE_VLS) 519 return set_sve_vls(vcpu, reg); 520 521 /* Try to interpret reg ID as an architectural SVE register... */ 522 ret = sve_reg_to_region(®ion, vcpu, reg); 523 if (ret) 524 return ret; 525 526 if (!kvm_arm_vcpu_sve_finalized(vcpu)) 527 return -EPERM; 528 529 if (copy_from_user(vcpu->arch.sve_state + region.koffset, uptr, 530 region.klen)) 531 return -EFAULT; 532 533 return 0; 534 } 535 536 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 537 { 538 return -EINVAL; 539 } 540 541 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 542 { 543 return -EINVAL; 544 } 545 546 static int copy_core_reg_indices(const struct kvm_vcpu *vcpu, 547 u64 __user *uindices) 548 { 549 unsigned int i; 550 int n = 0; 551 552 for (i = 0; i < sizeof(struct kvm_regs) / sizeof(__u32); i++) { 553 u64 reg = KVM_REG_ARM64 | KVM_REG_ARM_CORE | i; 554 int size = core_reg_size_from_offset(vcpu, i); 555 556 if (size < 0) 557 continue; 558 559 switch (size) { 560 case sizeof(__u32): 561 reg |= KVM_REG_SIZE_U32; 562 break; 563 564 case sizeof(__u64): 565 reg |= KVM_REG_SIZE_U64; 566 break; 567 568 case sizeof(__uint128_t): 569 reg |= KVM_REG_SIZE_U128; 570 break; 571 572 default: 573 WARN_ON(1); 574 continue; 575 } 576 577 if (uindices) { 578 if (put_user(reg, uindices)) 579 return -EFAULT; 580 uindices++; 581 } 582 583 n++; 584 } 585 586 return n; 587 } 588 589 static unsigned long num_core_regs(const struct kvm_vcpu *vcpu) 590 { 591 return copy_core_reg_indices(vcpu, NULL); 592 } 593 594 static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu) 595 { 596 const unsigned int slices = vcpu_sve_slices(vcpu); 597 598 if (!vcpu_has_sve(vcpu)) 599 return 0; 600 601 /* Policed by KVM_GET_REG_LIST: */ 602 WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu)); 603 604 return slices * (SVE_NUM_PREGS + SVE_NUM_ZREGS + 1 /* FFR */) 605 + 1; /* KVM_REG_ARM64_SVE_VLS */ 606 } 607 608 static int copy_sve_reg_indices(const struct kvm_vcpu *vcpu, 609 u64 __user *uindices) 610 { 611 const unsigned int slices = vcpu_sve_slices(vcpu); 612 u64 reg; 613 unsigned int i, n; 614 int num_regs = 0; 615 616 if (!vcpu_has_sve(vcpu)) 617 return 0; 618 619 /* Policed by KVM_GET_REG_LIST: */ 620 WARN_ON(!kvm_arm_vcpu_sve_finalized(vcpu)); 621 622 /* 623 * Enumerate this first, so that userspace can save/restore in 624 * the order reported by KVM_GET_REG_LIST: 625 */ 626 reg = KVM_REG_ARM64_SVE_VLS; 627 if (put_user(reg, uindices++)) 628 return -EFAULT; 629 ++num_regs; 630 631 for (i = 0; i < slices; i++) { 632 for (n = 0; n < SVE_NUM_ZREGS; n++) { 633 reg = KVM_REG_ARM64_SVE_ZREG(n, i); 634 if (put_user(reg, uindices++)) 635 return -EFAULT; 636 num_regs++; 637 } 638 639 for (n = 0; n < SVE_NUM_PREGS; n++) { 640 reg = KVM_REG_ARM64_SVE_PREG(n, i); 641 if (put_user(reg, uindices++)) 642 return -EFAULT; 643 num_regs++; 644 } 645 646 reg = KVM_REG_ARM64_SVE_FFR(i); 647 if (put_user(reg, uindices++)) 648 return -EFAULT; 649 num_regs++; 650 } 651 652 return num_regs; 653 } 654 655 /** 656 * kvm_arm_num_regs - how many registers do we present via KVM_GET_ONE_REG 657 * @vcpu: the vCPU pointer 658 * 659 * This is for all registers. 660 */ 661 unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu) 662 { 663 unsigned long res = 0; 664 665 res += num_core_regs(vcpu); 666 res += num_sve_regs(vcpu); 667 res += kvm_arm_num_sys_reg_descs(vcpu); 668 res += kvm_arm_get_fw_num_regs(vcpu); 669 670 return res; 671 } 672 673 /** 674 * kvm_arm_copy_reg_indices - get indices of all registers. 675 * @vcpu: the vCPU pointer 676 * @uindices: register list to copy 677 * 678 * We do core registers right here, then we append system regs. 679 */ 680 int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) 681 { 682 int ret; 683 684 ret = copy_core_reg_indices(vcpu, uindices); 685 if (ret < 0) 686 return ret; 687 uindices += ret; 688 689 ret = copy_sve_reg_indices(vcpu, uindices); 690 if (ret < 0) 691 return ret; 692 uindices += ret; 693 694 ret = kvm_arm_copy_fw_reg_indices(vcpu, uindices); 695 if (ret < 0) 696 return ret; 697 uindices += kvm_arm_get_fw_num_regs(vcpu); 698 699 return kvm_arm_copy_sys_reg_indices(vcpu, uindices); 700 } 701 702 int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 703 { 704 /* We currently use nothing arch-specific in upper 32 bits */ 705 if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) 706 return -EINVAL; 707 708 switch (reg->id & KVM_REG_ARM_COPROC_MASK) { 709 case KVM_REG_ARM_CORE: return get_core_reg(vcpu, reg); 710 case KVM_REG_ARM_FW: 711 case KVM_REG_ARM_FW_FEAT_BMAP: 712 return kvm_arm_get_fw_reg(vcpu, reg); 713 case KVM_REG_ARM64_SVE: return get_sve_reg(vcpu, reg); 714 } 715 716 return kvm_arm_sys_reg_get_reg(vcpu, reg); 717 } 718 719 int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 720 { 721 /* We currently use nothing arch-specific in upper 32 bits */ 722 if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) 723 return -EINVAL; 724 725 switch (reg->id & KVM_REG_ARM_COPROC_MASK) { 726 case KVM_REG_ARM_CORE: return set_core_reg(vcpu, reg); 727 case KVM_REG_ARM_FW: 728 case KVM_REG_ARM_FW_FEAT_BMAP: 729 return kvm_arm_set_fw_reg(vcpu, reg); 730 case KVM_REG_ARM64_SVE: return set_sve_reg(vcpu, reg); 731 } 732 733 return kvm_arm_sys_reg_set_reg(vcpu, reg); 734 } 735 736 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 737 struct kvm_sregs *sregs) 738 { 739 return -EINVAL; 740 } 741 742 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 743 struct kvm_sregs *sregs) 744 { 745 return -EINVAL; 746 } 747 748 int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, 749 struct kvm_vcpu_events *events) 750 { 751 events->exception.serror_has_esr = cpus_have_final_cap(ARM64_HAS_RAS_EXTN); 752 events->exception.serror_pending = (vcpu->arch.hcr_el2 & HCR_VSE) || 753 vcpu_get_flag(vcpu, NESTED_SERROR_PENDING); 754 755 if (events->exception.serror_pending && events->exception.serror_has_esr) 756 events->exception.serror_esr = vcpu_get_vsesr(vcpu); 757 758 /* 759 * We never return a pending ext_dabt here because we deliver it to 760 * the virtual CPU directly when setting the event and it's no longer 761 * 'pending' at this point. 762 */ 763 764 return 0; 765 } 766 767 static void commit_pending_events(struct kvm_vcpu *vcpu) 768 { 769 if (!vcpu_get_flag(vcpu, PENDING_EXCEPTION)) 770 return; 771 772 /* 773 * Reset the MMIO emulation state to avoid stepping PC after emulating 774 * the exception entry. 775 */ 776 vcpu->mmio_needed = false; 777 kvm_call_hyp(__kvm_adjust_pc, vcpu); 778 } 779 780 int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, 781 struct kvm_vcpu_events *events) 782 { 783 bool serror_pending = events->exception.serror_pending; 784 bool has_esr = events->exception.serror_has_esr; 785 bool ext_dabt_pending = events->exception.ext_dabt_pending; 786 u64 esr = events->exception.serror_esr; 787 int ret = 0; 788 789 /* 790 * Immediately commit the pending SEA to the vCPU's architectural 791 * state which is necessary since we do not return a pending SEA 792 * to userspace via KVM_GET_VCPU_EVENTS. 793 */ 794 if (ext_dabt_pending) { 795 ret = kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu)); 796 commit_pending_events(vcpu); 797 } 798 799 if (ret < 0) 800 return ret; 801 802 if (!serror_pending) 803 return 0; 804 805 if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && has_esr) 806 return -EINVAL; 807 808 if (has_esr && (esr & ~ESR_ELx_ISS_MASK)) 809 return -EINVAL; 810 811 if (has_esr) 812 ret = kvm_inject_serror_esr(vcpu, esr); 813 else 814 ret = kvm_inject_serror(vcpu); 815 816 /* 817 * We could've decided that the SError is due for immediate software 818 * injection; commit the exception in case userspace decides it wants 819 * to inject more exceptions for some strange reason. 820 */ 821 commit_pending_events(vcpu); 822 return (ret < 0) ? ret : 0; 823 } 824 825 u32 __attribute_const__ kvm_target_cpu(void) 826 { 827 unsigned long implementor = read_cpuid_implementor(); 828 unsigned long part_number = read_cpuid_part_number(); 829 830 switch (implementor) { 831 case ARM_CPU_IMP_ARM: 832 switch (part_number) { 833 case ARM_CPU_PART_AEM_V8: 834 return KVM_ARM_TARGET_AEM_V8; 835 case ARM_CPU_PART_FOUNDATION: 836 return KVM_ARM_TARGET_FOUNDATION_V8; 837 case ARM_CPU_PART_CORTEX_A53: 838 return KVM_ARM_TARGET_CORTEX_A53; 839 case ARM_CPU_PART_CORTEX_A57: 840 return KVM_ARM_TARGET_CORTEX_A57; 841 } 842 break; 843 case ARM_CPU_IMP_APM: 844 switch (part_number) { 845 case APM_CPU_PART_XGENE: 846 return KVM_ARM_TARGET_XGENE_POTENZA; 847 } 848 break; 849 } 850 851 /* Return a default generic target */ 852 return KVM_ARM_TARGET_GENERIC_V8; 853 } 854 855 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 856 { 857 return -EINVAL; 858 } 859 860 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 861 { 862 return -EINVAL; 863 } 864 865 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 866 struct kvm_translation *tr) 867 { 868 return -EINVAL; 869 } 870 871 /** 872 * kvm_arch_vcpu_ioctl_set_guest_debug - set up guest debugging 873 * @vcpu: the vCPU pointer 874 * @dbg: the ioctl data buffer 875 * 876 * This sets up and enables the VM for guest debugging. Userspace 877 * passes in a control flag to enable different debug types and 878 * potentially other architecture specific information in the rest of 879 * the structure. 880 */ 881 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 882 struct kvm_guest_debug *dbg) 883 { 884 trace_kvm_set_guest_debug(vcpu, dbg->control); 885 886 if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) 887 return -EINVAL; 888 889 if (!(dbg->control & KVM_GUESTDBG_ENABLE)) { 890 vcpu->guest_debug = 0; 891 vcpu_clear_flag(vcpu, HOST_SS_ACTIVE_PENDING); 892 return 0; 893 } 894 895 vcpu->guest_debug = dbg->control; 896 897 /* Hardware assisted Break and Watch points */ 898 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW) 899 vcpu->arch.external_debug_state = dbg->arch; 900 901 return 0; 902 } 903 904 int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu, 905 struct kvm_device_attr *attr) 906 { 907 int ret; 908 909 switch (attr->group) { 910 case KVM_ARM_VCPU_PMU_V3_CTRL: 911 mutex_lock(&vcpu->kvm->arch.config_lock); 912 ret = kvm_arm_pmu_v3_set_attr(vcpu, attr); 913 mutex_unlock(&vcpu->kvm->arch.config_lock); 914 break; 915 case KVM_ARM_VCPU_TIMER_CTRL: 916 ret = kvm_arm_timer_set_attr(vcpu, attr); 917 break; 918 case KVM_ARM_VCPU_PVTIME_CTRL: 919 ret = kvm_arm_pvtime_set_attr(vcpu, attr); 920 break; 921 default: 922 ret = -ENXIO; 923 break; 924 } 925 926 return ret; 927 } 928 929 int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu, 930 struct kvm_device_attr *attr) 931 { 932 int ret; 933 934 switch (attr->group) { 935 case KVM_ARM_VCPU_PMU_V3_CTRL: 936 ret = kvm_arm_pmu_v3_get_attr(vcpu, attr); 937 break; 938 case KVM_ARM_VCPU_TIMER_CTRL: 939 ret = kvm_arm_timer_get_attr(vcpu, attr); 940 break; 941 case KVM_ARM_VCPU_PVTIME_CTRL: 942 ret = kvm_arm_pvtime_get_attr(vcpu, attr); 943 break; 944 default: 945 ret = -ENXIO; 946 break; 947 } 948 949 return ret; 950 } 951 952 int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu, 953 struct kvm_device_attr *attr) 954 { 955 int ret; 956 957 switch (attr->group) { 958 case KVM_ARM_VCPU_PMU_V3_CTRL: 959 ret = kvm_arm_pmu_v3_has_attr(vcpu, attr); 960 break; 961 case KVM_ARM_VCPU_TIMER_CTRL: 962 ret = kvm_arm_timer_has_attr(vcpu, attr); 963 break; 964 case KVM_ARM_VCPU_PVTIME_CTRL: 965 ret = kvm_arm_pvtime_has_attr(vcpu, attr); 966 break; 967 default: 968 ret = -ENXIO; 969 break; 970 } 971 972 return ret; 973 } 974 975 int kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm, 976 struct kvm_arm_copy_mte_tags *copy_tags) 977 { 978 gpa_t guest_ipa = copy_tags->guest_ipa; 979 size_t length = copy_tags->length; 980 void __user *tags = copy_tags->addr; 981 gpa_t gfn; 982 bool write = !(copy_tags->flags & KVM_ARM_TAGS_FROM_GUEST); 983 int ret = 0; 984 985 if (!kvm_has_mte(kvm)) 986 return -EINVAL; 987 988 if (copy_tags->reserved[0] || copy_tags->reserved[1]) 989 return -EINVAL; 990 991 if (copy_tags->flags & ~KVM_ARM_TAGS_FROM_GUEST) 992 return -EINVAL; 993 994 if (length & ~PAGE_MASK || guest_ipa & ~PAGE_MASK) 995 return -EINVAL; 996 997 /* Lengths above INT_MAX cannot be represented in the return value */ 998 if (length > INT_MAX) 999 return -EINVAL; 1000 1001 gfn = gpa_to_gfn(guest_ipa); 1002 1003 mutex_lock(&kvm->slots_lock); 1004 1005 if (write && atomic_read(&kvm->nr_memslots_dirty_logging)) { 1006 ret = -EBUSY; 1007 goto out; 1008 } 1009 1010 while (length > 0) { 1011 struct page *page = __gfn_to_page(kvm, gfn, write); 1012 void *maddr; 1013 unsigned long num_tags; 1014 struct folio *folio; 1015 1016 if (!page) { 1017 ret = -EFAULT; 1018 goto out; 1019 } 1020 1021 if (!pfn_to_online_page(page_to_pfn(page))) { 1022 /* Reject ZONE_DEVICE memory */ 1023 kvm_release_page_unused(page); 1024 ret = -EFAULT; 1025 goto out; 1026 } 1027 folio = page_folio(page); 1028 maddr = page_address(page); 1029 1030 if (!write) { 1031 if ((folio_test_hugetlb(folio) && 1032 folio_test_hugetlb_mte_tagged(folio)) || 1033 page_mte_tagged(page)) 1034 num_tags = mte_copy_tags_to_user(tags, maddr, 1035 MTE_GRANULES_PER_PAGE); 1036 else 1037 /* No tags in memory, so write zeros */ 1038 num_tags = MTE_GRANULES_PER_PAGE - 1039 clear_user(tags, MTE_GRANULES_PER_PAGE); 1040 kvm_release_page_clean(page); 1041 } else { 1042 /* 1043 * Only locking to serialise with a concurrent 1044 * __set_ptes() in the VMM but still overriding the 1045 * tags, hence ignoring the return value. 1046 */ 1047 if (folio_test_hugetlb(folio)) 1048 folio_try_hugetlb_mte_tagging(folio); 1049 else 1050 try_page_mte_tagging(page); 1051 num_tags = mte_copy_tags_from_user(maddr, tags, 1052 MTE_GRANULES_PER_PAGE); 1053 1054 /* uaccess failed, don't leave stale tags */ 1055 if (num_tags != MTE_GRANULES_PER_PAGE) 1056 mte_clear_page_tags(maddr); 1057 if (folio_test_hugetlb(folio)) 1058 folio_set_hugetlb_mte_tagged(folio); 1059 else 1060 set_page_mte_tagged(page); 1061 1062 kvm_release_page_dirty(page); 1063 } 1064 1065 if (num_tags != MTE_GRANULES_PER_PAGE) { 1066 ret = -EFAULT; 1067 goto out; 1068 } 1069 1070 gfn++; 1071 tags += num_tags; 1072 length -= PAGE_SIZE; 1073 } 1074 1075 out: 1076 mutex_unlock(&kvm->slots_lock); 1077 /* If some data has been copied report the number of bytes copied */ 1078 if (length != copy_tags->length) 1079 return copy_tags->length - length; 1080 return ret; 1081 } 1082