1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 4 * Copyright (C) 2023 Ventana Micro Systems Inc. 5 * 6 * Authors: 7 * Anup Patel <apatel@ventanamicro.com> 8 */ 9 10 #include <linux/bitops.h> 11 #include <linux/errno.h> 12 #include <linux/err.h> 13 #include <linux/nospec.h> 14 #include <linux/uaccess.h> 15 #include <linux/kvm_host.h> 16 #include <asm/cacheflush.h> 17 #include <asm/cpufeature.h> 18 #include <asm/kvm_isa.h> 19 #include <asm/kvm_vcpu_vector.h> 20 21 #define KVM_RISCV_BASE_ISA_MASK GENMASK(25, 0) 22 23 void kvm_riscv_vcpu_setup_isa(struct kvm_vcpu *vcpu) 24 { 25 unsigned long guest_ext, i; 26 27 for (i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { 28 if (__kvm_riscv_isa_check_host(i, &guest_ext)) 29 continue; 30 if (kvm_riscv_isa_enable_allowed(i)) 31 set_bit(guest_ext, vcpu->arch.isa); 32 } 33 } 34 35 static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu, 36 const struct kvm_one_reg *reg) 37 { 38 unsigned long __user *uaddr = 39 (unsigned long __user *)(unsigned long)reg->addr; 40 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 41 KVM_REG_SIZE_MASK | 42 KVM_REG_RISCV_CONFIG); 43 unsigned long reg_val; 44 45 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 46 return -EINVAL; 47 48 switch (reg_num) { 49 case KVM_REG_RISCV_CONFIG_REG(isa): 50 reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK; 51 break; 52 case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): 53 reg_val = (kvm_riscv_isa_check_host(ZICBOM)) ? 0 : riscv_cbom_block_size; 54 break; 55 case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): 56 reg_val = (kvm_riscv_isa_check_host(ZICBOZ)) ? 0 : riscv_cboz_block_size; 57 break; 58 case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): 59 reg_val = (kvm_riscv_isa_check_host(ZICBOP)) ? 0 : riscv_cbop_block_size; 60 break; 61 case KVM_REG_RISCV_CONFIG_REG(mvendorid): 62 reg_val = vcpu->arch.mvendorid; 63 break; 64 case KVM_REG_RISCV_CONFIG_REG(marchid): 65 reg_val = vcpu->arch.marchid; 66 break; 67 case KVM_REG_RISCV_CONFIG_REG(mimpid): 68 reg_val = vcpu->arch.mimpid; 69 break; 70 case KVM_REG_RISCV_CONFIG_REG(satp_mode): 71 reg_val = satp_mode >> SATP_MODE_SHIFT; 72 break; 73 default: 74 return -ENOENT; 75 } 76 77 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 78 return -EFAULT; 79 80 return 0; 81 } 82 83 static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu, 84 const struct kvm_one_reg *reg) 85 { 86 unsigned long __user *uaddr = 87 (unsigned long __user *)(unsigned long)reg->addr; 88 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 89 KVM_REG_SIZE_MASK | 90 KVM_REG_RISCV_CONFIG); 91 unsigned long i, isa_ext, reg_val; 92 93 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 94 return -EINVAL; 95 96 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 97 return -EFAULT; 98 99 switch (reg_num) { 100 case KVM_REG_RISCV_CONFIG_REG(isa): 101 /* 102 * This ONE REG interface is only defined for 103 * single letter extensions. 104 */ 105 if (fls(reg_val) >= RISCV_ISA_EXT_BASE) 106 return -EINVAL; 107 108 /* 109 * Return early (i.e. do nothing) if reg_val is the same 110 * value retrievable via kvm_riscv_vcpu_get_reg_config(). 111 */ 112 if (reg_val == (vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK)) 113 break; 114 115 if (!vcpu->arch.ran_atleast_once) { 116 /* Ignore the enable/disable request for certain extensions */ 117 for (i = 0; i < RISCV_ISA_EXT_BASE; i++) { 118 isa_ext = kvm_riscv_base2isa_ext(i); 119 if (isa_ext >= KVM_RISCV_ISA_EXT_MAX) { 120 reg_val &= ~BIT(i); 121 continue; 122 } 123 if (!kvm_riscv_isa_enable_allowed(isa_ext)) 124 if (reg_val & BIT(i)) 125 reg_val &= ~BIT(i); 126 if (!kvm_riscv_isa_disable_allowed(isa_ext)) 127 if (!(reg_val & BIT(i))) 128 reg_val |= BIT(i); 129 } 130 reg_val &= riscv_isa_extension_base(NULL); 131 /* Do not modify anything beyond single letter extensions */ 132 reg_val = (vcpu->arch.isa[0] & ~KVM_RISCV_BASE_ISA_MASK) | 133 (reg_val & KVM_RISCV_BASE_ISA_MASK); 134 vcpu->arch.isa[0] = reg_val; 135 kvm_riscv_vcpu_fp_reset(vcpu); 136 } else { 137 return -EBUSY; 138 } 139 break; 140 case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): 141 if (reg_val && reg_val != riscv_cbom_block_size) 142 return -EINVAL; 143 break; 144 case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): 145 if (reg_val && reg_val != riscv_cboz_block_size) 146 return -EINVAL; 147 break; 148 case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): 149 if (reg_val && reg_val != riscv_cbop_block_size) 150 return -EINVAL; 151 break; 152 case KVM_REG_RISCV_CONFIG_REG(mvendorid): 153 if (reg_val == vcpu->arch.mvendorid) 154 break; 155 if (!vcpu->arch.ran_atleast_once) 156 vcpu->arch.mvendorid = reg_val; 157 else 158 return -EBUSY; 159 break; 160 case KVM_REG_RISCV_CONFIG_REG(marchid): 161 if (reg_val == vcpu->arch.marchid) 162 break; 163 if (!vcpu->arch.ran_atleast_once) 164 vcpu->arch.marchid = reg_val; 165 else 166 return -EBUSY; 167 break; 168 case KVM_REG_RISCV_CONFIG_REG(mimpid): 169 if (reg_val == vcpu->arch.mimpid) 170 break; 171 if (!vcpu->arch.ran_atleast_once) 172 vcpu->arch.mimpid = reg_val; 173 else 174 return -EBUSY; 175 break; 176 case KVM_REG_RISCV_CONFIG_REG(satp_mode): 177 if (reg_val != (satp_mode >> SATP_MODE_SHIFT)) 178 return -EINVAL; 179 break; 180 default: 181 return -ENOENT; 182 } 183 184 return 0; 185 } 186 187 static int kvm_riscv_vcpu_get_reg_core(struct kvm_vcpu *vcpu, 188 const struct kvm_one_reg *reg) 189 { 190 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 191 unsigned long __user *uaddr = 192 (unsigned long __user *)(unsigned long)reg->addr; 193 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 194 KVM_REG_SIZE_MASK | 195 KVM_REG_RISCV_CORE); 196 unsigned long regs_max = sizeof(struct kvm_riscv_core) / sizeof(unsigned long); 197 unsigned long reg_val; 198 199 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 200 return -EINVAL; 201 if (reg_num >= regs_max) 202 return -ENOENT; 203 204 reg_num = array_index_nospec(reg_num, regs_max); 205 206 if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc)) 207 reg_val = cntx->sepc; 208 else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num && 209 reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6)) 210 reg_val = ((unsigned long *)cntx)[reg_num]; 211 else if (reg_num == KVM_REG_RISCV_CORE_REG(mode)) 212 reg_val = (cntx->sstatus & SR_SPP) ? 213 KVM_RISCV_MODE_S : KVM_RISCV_MODE_U; 214 else 215 return -ENOENT; 216 217 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 218 return -EFAULT; 219 220 return 0; 221 } 222 223 static int kvm_riscv_vcpu_set_reg_core(struct kvm_vcpu *vcpu, 224 const struct kvm_one_reg *reg) 225 { 226 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 227 unsigned long __user *uaddr = 228 (unsigned long __user *)(unsigned long)reg->addr; 229 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 230 KVM_REG_SIZE_MASK | 231 KVM_REG_RISCV_CORE); 232 unsigned long regs_max = sizeof(struct kvm_riscv_core) / sizeof(unsigned long); 233 unsigned long reg_val; 234 235 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 236 return -EINVAL; 237 if (reg_num >= regs_max) 238 return -ENOENT; 239 240 reg_num = array_index_nospec(reg_num, regs_max); 241 242 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 243 return -EFAULT; 244 245 if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc)) 246 cntx->sepc = reg_val; 247 else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num && 248 reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6)) 249 ((unsigned long *)cntx)[reg_num] = reg_val; 250 else if (reg_num == KVM_REG_RISCV_CORE_REG(mode)) { 251 if (reg_val == KVM_RISCV_MODE_S) 252 cntx->sstatus |= SR_SPP; 253 else 254 cntx->sstatus &= ~SR_SPP; 255 } else 256 return -ENOENT; 257 258 return 0; 259 } 260 261 static int kvm_riscv_vcpu_general_get_csr(struct kvm_vcpu *vcpu, 262 unsigned long reg_num, 263 unsigned long *out_val) 264 { 265 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 266 unsigned long regs_max = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); 267 268 if (reg_num >= regs_max) 269 return -ENOENT; 270 271 reg_num = array_index_nospec(reg_num, regs_max); 272 273 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { 274 kvm_riscv_vcpu_flush_interrupts(vcpu); 275 *out_val = hvip_to_vsip(csr->hvip); 276 *out_val |= csr->hvip & ~IRQ_LOCAL_MASK; 277 } else 278 *out_val = ((unsigned long *)csr)[reg_num]; 279 280 return 0; 281 } 282 283 static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, 284 unsigned long reg_num, 285 unsigned long reg_val) 286 { 287 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 288 unsigned long regs_max = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); 289 unsigned long flags; 290 291 if (reg_num >= regs_max) 292 return -ENOENT; 293 294 reg_num = array_index_nospec(reg_num, regs_max); 295 296 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) 297 reg_val = vsip_to_hvip(reg_val); 298 299 ((unsigned long *)csr)[reg_num] = reg_val; 300 301 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { 302 raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); 303 vcpu->arch.irqs_pending_mask[0] = 0; 304 raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); 305 } 306 307 return 0; 308 } 309 310 static inline int kvm_riscv_vcpu_smstateen_set_csr(struct kvm_vcpu *vcpu, 311 unsigned long reg_num, 312 unsigned long reg_val) 313 { 314 struct kvm_vcpu_smstateen_csr *csr = &vcpu->arch.smstateen_csr; 315 unsigned long regs_max = sizeof(struct kvm_riscv_smstateen_csr) / 316 sizeof(unsigned long); 317 318 if (!riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) 319 return -ENOENT; 320 if (reg_num >= regs_max) 321 return -ENOENT; 322 323 reg_num = array_index_nospec(reg_num, regs_max); 324 325 ((unsigned long *)csr)[reg_num] = reg_val; 326 return 0; 327 } 328 329 static int kvm_riscv_vcpu_smstateen_get_csr(struct kvm_vcpu *vcpu, 330 unsigned long reg_num, 331 unsigned long *out_val) 332 { 333 struct kvm_vcpu_smstateen_csr *csr = &vcpu->arch.smstateen_csr; 334 unsigned long regs_max = sizeof(struct kvm_riscv_smstateen_csr) / 335 sizeof(unsigned long); 336 337 if (!riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) 338 return -ENOENT; 339 if (reg_num >= regs_max) 340 return -ENOENT; 341 342 reg_num = array_index_nospec(reg_num, regs_max); 343 344 *out_val = ((unsigned long *)csr)[reg_num]; 345 return 0; 346 } 347 348 static inline int kvm_riscv_vcpu_zicfiss_set_csr(struct kvm_vcpu *vcpu, 349 unsigned long reg_num, 350 unsigned long reg_val) 351 { 352 struct kvm_vcpu_zicfiss_csr *csr = &vcpu->arch.zicfiss_csr; 353 unsigned long regs_max = sizeof(struct kvm_vcpu_zicfiss_csr) / 354 sizeof(unsigned long); 355 356 if (!riscv_isa_extension_available(vcpu->arch.isa, ZICFISS)) 357 return -ENOENT; 358 if (reg_num >= regs_max) 359 return -ENOENT; 360 361 reg_num = array_index_nospec(reg_num, regs_max); 362 363 ((unsigned long *)csr)[reg_num] = reg_val; 364 return 0; 365 } 366 367 static int kvm_riscv_vcpu_zicfiss_get_csr(struct kvm_vcpu *vcpu, 368 unsigned long reg_num, 369 unsigned long *out_val) 370 { 371 struct kvm_vcpu_zicfiss_csr *csr = &vcpu->arch.zicfiss_csr; 372 unsigned long regs_max = sizeof(struct kvm_vcpu_zicfiss_csr) / 373 sizeof(unsigned long); 374 375 if (!riscv_isa_extension_available(vcpu->arch.isa, ZICFISS)) 376 return -ENOENT; 377 if (reg_num >= regs_max) 378 return -ENOENT; 379 380 reg_num = array_index_nospec(reg_num, regs_max); 381 382 *out_val = ((unsigned long *)csr)[reg_num]; 383 return 0; 384 } 385 386 static int kvm_riscv_vcpu_get_reg_csr(struct kvm_vcpu *vcpu, 387 const struct kvm_one_reg *reg) 388 { 389 int rc; 390 unsigned long __user *uaddr = 391 (unsigned long __user *)(unsigned long)reg->addr; 392 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 393 KVM_REG_SIZE_MASK | 394 KVM_REG_RISCV_CSR); 395 unsigned long reg_val, reg_subtype; 396 397 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 398 return -EINVAL; 399 400 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 401 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 402 switch (reg_subtype) { 403 case KVM_REG_RISCV_CSR_GENERAL: 404 rc = kvm_riscv_vcpu_general_get_csr(vcpu, reg_num, ®_val); 405 break; 406 case KVM_REG_RISCV_CSR_AIA: 407 rc = kvm_riscv_vcpu_aia_get_csr(vcpu, reg_num, ®_val); 408 break; 409 case KVM_REG_RISCV_CSR_SMSTATEEN: 410 rc = kvm_riscv_vcpu_smstateen_get_csr(vcpu, reg_num, ®_val); 411 break; 412 case KVM_REG_RISCV_CSR_ZICFISS: 413 rc = kvm_riscv_vcpu_zicfiss_get_csr(vcpu, reg_num, ®_val); 414 break; 415 default: 416 rc = -ENOENT; 417 break; 418 } 419 if (rc) 420 return rc; 421 422 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 423 return -EFAULT; 424 425 return 0; 426 } 427 428 static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu, 429 const struct kvm_one_reg *reg) 430 { 431 int rc; 432 unsigned long __user *uaddr = 433 (unsigned long __user *)(unsigned long)reg->addr; 434 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 435 KVM_REG_SIZE_MASK | 436 KVM_REG_RISCV_CSR); 437 unsigned long reg_val, reg_subtype; 438 439 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 440 return -EINVAL; 441 442 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 443 return -EFAULT; 444 445 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 446 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 447 switch (reg_subtype) { 448 case KVM_REG_RISCV_CSR_GENERAL: 449 rc = kvm_riscv_vcpu_general_set_csr(vcpu, reg_num, reg_val); 450 break; 451 case KVM_REG_RISCV_CSR_AIA: 452 rc = kvm_riscv_vcpu_aia_set_csr(vcpu, reg_num, reg_val); 453 break; 454 case KVM_REG_RISCV_CSR_SMSTATEEN: 455 rc = kvm_riscv_vcpu_smstateen_set_csr(vcpu, reg_num, reg_val); 456 break; 457 case KVM_REG_RISCV_CSR_ZICFISS: 458 rc = kvm_riscv_vcpu_zicfiss_set_csr(vcpu, reg_num, reg_val); 459 break; 460 default: 461 rc = -ENOENT; 462 break; 463 } 464 if (rc) 465 return rc; 466 467 vcpu->arch.csr_dirty = true; 468 469 return 0; 470 } 471 472 static int riscv_vcpu_get_isa_ext_single(struct kvm_vcpu *vcpu, 473 unsigned long reg_num, 474 unsigned long *reg_val) 475 { 476 unsigned long guest_ext; 477 int ret; 478 479 ret = __kvm_riscv_isa_check_host(reg_num, &guest_ext); 480 if (ret) 481 return ret; 482 483 *reg_val = 0; 484 if (__riscv_isa_extension_available(vcpu->arch.isa, guest_ext)) 485 *reg_val = 1; /* Mark the given extension as available */ 486 487 return 0; 488 } 489 490 static int riscv_vcpu_set_isa_ext_single(struct kvm_vcpu *vcpu, 491 unsigned long reg_num, 492 unsigned long reg_val) 493 { 494 unsigned long guest_ext; 495 int ret; 496 497 ret = __kvm_riscv_isa_check_host(reg_num, &guest_ext); 498 if (ret) 499 return ret; 500 501 if (reg_val == test_bit(guest_ext, vcpu->arch.isa)) 502 return 0; 503 504 if (!vcpu->arch.ran_atleast_once) { 505 /* 506 * All multi-letter extension and a few single letter 507 * extension can be disabled 508 */ 509 if (reg_val == 1 && 510 kvm_riscv_isa_enable_allowed(reg_num)) 511 set_bit(guest_ext, vcpu->arch.isa); 512 else if (!reg_val && 513 kvm_riscv_isa_disable_allowed(reg_num)) 514 clear_bit(guest_ext, vcpu->arch.isa); 515 else 516 return -EINVAL; 517 kvm_riscv_vcpu_fp_reset(vcpu); 518 } else { 519 return -EBUSY; 520 } 521 522 return 0; 523 } 524 525 static int riscv_vcpu_get_isa_ext_multi(struct kvm_vcpu *vcpu, 526 unsigned long reg_num, 527 unsigned long *reg_val) 528 { 529 unsigned long i, ext_id, ext_val; 530 531 if (reg_num > KVM_REG_RISCV_ISA_MULTI_REG_LAST) 532 return -ENOENT; 533 534 for (i = 0; i < BITS_PER_LONG; i++) { 535 ext_id = i + reg_num * BITS_PER_LONG; 536 if (ext_id >= KVM_RISCV_ISA_EXT_MAX) 537 break; 538 539 ext_val = 0; 540 riscv_vcpu_get_isa_ext_single(vcpu, ext_id, &ext_val); 541 if (ext_val) 542 *reg_val |= KVM_REG_RISCV_ISA_MULTI_MASK(ext_id); 543 } 544 545 return 0; 546 } 547 548 static int riscv_vcpu_set_isa_ext_multi(struct kvm_vcpu *vcpu, 549 unsigned long reg_num, 550 unsigned long reg_val, bool enable) 551 { 552 unsigned long i, ext_id; 553 554 if (reg_num > KVM_REG_RISCV_ISA_MULTI_REG_LAST) 555 return -ENOENT; 556 557 for_each_set_bit(i, ®_val, BITS_PER_LONG) { 558 ext_id = i + reg_num * BITS_PER_LONG; 559 if (ext_id >= KVM_RISCV_ISA_EXT_MAX) 560 break; 561 562 riscv_vcpu_set_isa_ext_single(vcpu, ext_id, enable); 563 } 564 565 return 0; 566 } 567 568 static int kvm_riscv_vcpu_get_reg_isa_ext(struct kvm_vcpu *vcpu, 569 const struct kvm_one_reg *reg) 570 { 571 int rc; 572 unsigned long __user *uaddr = 573 (unsigned long __user *)(unsigned long)reg->addr; 574 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 575 KVM_REG_SIZE_MASK | 576 KVM_REG_RISCV_ISA_EXT); 577 unsigned long reg_val, reg_subtype; 578 579 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 580 return -EINVAL; 581 582 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 583 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 584 585 reg_val = 0; 586 switch (reg_subtype) { 587 case KVM_REG_RISCV_ISA_SINGLE: 588 rc = riscv_vcpu_get_isa_ext_single(vcpu, reg_num, ®_val); 589 break; 590 case KVM_REG_RISCV_ISA_MULTI_EN: 591 case KVM_REG_RISCV_ISA_MULTI_DIS: 592 rc = riscv_vcpu_get_isa_ext_multi(vcpu, reg_num, ®_val); 593 if (!rc && reg_subtype == KVM_REG_RISCV_ISA_MULTI_DIS) 594 reg_val = ~reg_val; 595 break; 596 default: 597 rc = -ENOENT; 598 } 599 if (rc) 600 return rc; 601 602 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 603 return -EFAULT; 604 605 return 0; 606 } 607 608 static int kvm_riscv_vcpu_set_reg_isa_ext(struct kvm_vcpu *vcpu, 609 const struct kvm_one_reg *reg) 610 { 611 unsigned long __user *uaddr = 612 (unsigned long __user *)(unsigned long)reg->addr; 613 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 614 KVM_REG_SIZE_MASK | 615 KVM_REG_RISCV_ISA_EXT); 616 unsigned long reg_val, reg_subtype; 617 618 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 619 return -EINVAL; 620 621 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 622 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 623 624 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 625 return -EFAULT; 626 627 switch (reg_subtype) { 628 case KVM_REG_RISCV_ISA_SINGLE: 629 return riscv_vcpu_set_isa_ext_single(vcpu, reg_num, reg_val); 630 case KVM_REG_RISCV_ISA_MULTI_EN: 631 return riscv_vcpu_set_isa_ext_multi(vcpu, reg_num, reg_val, true); 632 case KVM_REG_RISCV_ISA_MULTI_DIS: 633 return riscv_vcpu_set_isa_ext_multi(vcpu, reg_num, reg_val, false); 634 default: 635 return -ENOENT; 636 } 637 638 return 0; 639 } 640 641 static int copy_config_reg_indices(const struct kvm_vcpu *vcpu, 642 u64 __user *uindices) 643 { 644 int n = 0; 645 646 for (int i = 0; i < sizeof(struct kvm_riscv_config)/sizeof(unsigned long); 647 i++) { 648 u64 size; 649 u64 reg; 650 651 size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 652 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CONFIG | i; 653 654 if (uindices) { 655 if (put_user(reg, uindices)) 656 return -EFAULT; 657 uindices++; 658 } 659 660 n++; 661 } 662 663 return n; 664 } 665 666 static unsigned long num_config_regs(const struct kvm_vcpu *vcpu) 667 { 668 return copy_config_reg_indices(vcpu, NULL); 669 } 670 671 static inline unsigned long num_core_regs(void) 672 { 673 return sizeof(struct kvm_riscv_core) / sizeof(unsigned long); 674 } 675 676 static int copy_core_reg_indices(u64 __user *uindices) 677 { 678 int n = num_core_regs(); 679 680 for (int i = 0; i < n; i++) { 681 u64 size = IS_ENABLED(CONFIG_32BIT) ? 682 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 683 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CORE | i; 684 685 if (uindices) { 686 if (put_user(reg, uindices)) 687 return -EFAULT; 688 uindices++; 689 } 690 } 691 692 return n; 693 } 694 695 static inline unsigned long num_csr_regs(const struct kvm_vcpu *vcpu) 696 { 697 unsigned long n = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); 698 699 if (riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) 700 n += sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); 701 if (riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) 702 n += sizeof(struct kvm_riscv_smstateen_csr) / sizeof(unsigned long); 703 if (riscv_isa_extension_available(vcpu->arch.isa, ZICFISS)) 704 n += sizeof(struct kvm_riscv_zicfiss_csr) / sizeof(unsigned long); 705 706 return n; 707 } 708 709 static int copy_csr_reg_indices(const struct kvm_vcpu *vcpu, 710 u64 __user *uindices) 711 { 712 int n1 = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); 713 int n2 = 0, n3 = 0, n4 = 0; 714 715 /* copy general csr regs */ 716 for (int i = 0; i < n1; i++) { 717 u64 size = IS_ENABLED(CONFIG_32BIT) ? 718 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 719 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR | 720 KVM_REG_RISCV_CSR_GENERAL | i; 721 722 if (uindices) { 723 if (put_user(reg, uindices)) 724 return -EFAULT; 725 uindices++; 726 } 727 } 728 729 /* copy AIA csr regs */ 730 if (riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) { 731 n2 = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); 732 733 for (int i = 0; i < n2; i++) { 734 u64 size = IS_ENABLED(CONFIG_32BIT) ? 735 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 736 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR | 737 KVM_REG_RISCV_CSR_AIA | i; 738 739 if (uindices) { 740 if (put_user(reg, uindices)) 741 return -EFAULT; 742 uindices++; 743 } 744 } 745 } 746 747 /* copy Smstateen csr regs */ 748 if (riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) { 749 n3 = sizeof(struct kvm_riscv_smstateen_csr) / sizeof(unsigned long); 750 751 for (int i = 0; i < n3; i++) { 752 u64 size = IS_ENABLED(CONFIG_32BIT) ? 753 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 754 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR | 755 KVM_REG_RISCV_CSR_SMSTATEEN | i; 756 757 if (uindices) { 758 if (put_user(reg, uindices)) 759 return -EFAULT; 760 uindices++; 761 } 762 } 763 } 764 765 /* copy Zicfiss csr regs */ 766 if (riscv_isa_extension_available(vcpu->arch.isa, ZICFISS)) { 767 n4 = sizeof(struct kvm_riscv_zicfiss_csr) / sizeof(unsigned long); 768 769 for (int i = 0; i < n4; i++) { 770 u64 size = IS_ENABLED(CONFIG_32BIT) ? 771 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 772 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR | 773 KVM_REG_RISCV_CSR_ZICFISS | i; 774 775 if (uindices) { 776 if (put_user(reg, uindices)) 777 return -EFAULT; 778 uindices++; 779 } 780 } 781 } 782 783 return n1 + n2 + n3 + n4; 784 } 785 786 static inline unsigned long num_timer_regs(void) 787 { 788 return sizeof(struct kvm_riscv_timer) / sizeof(u64); 789 } 790 791 static int copy_timer_reg_indices(u64 __user *uindices) 792 { 793 int n = num_timer_regs(); 794 795 for (int i = 0; i < n; i++) { 796 u64 reg = KVM_REG_RISCV | KVM_REG_SIZE_U64 | 797 KVM_REG_RISCV_TIMER | i; 798 799 if (uindices) { 800 if (put_user(reg, uindices)) 801 return -EFAULT; 802 uindices++; 803 } 804 } 805 806 return n; 807 } 808 809 static inline unsigned long num_fp_f_regs(const struct kvm_vcpu *vcpu) 810 { 811 const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 812 813 if (riscv_isa_extension_available(vcpu->arch.isa, F)) 814 return sizeof(cntx->fp.f) / sizeof(u32); 815 else 816 return 0; 817 } 818 819 static int copy_fp_f_reg_indices(const struct kvm_vcpu *vcpu, 820 u64 __user *uindices) 821 { 822 int n = num_fp_f_regs(vcpu); 823 824 for (int i = 0; i < n; i++) { 825 u64 reg = KVM_REG_RISCV | KVM_REG_SIZE_U32 | 826 KVM_REG_RISCV_FP_F | i; 827 828 if (uindices) { 829 if (put_user(reg, uindices)) 830 return -EFAULT; 831 uindices++; 832 } 833 } 834 835 return n; 836 } 837 838 static inline unsigned long num_fp_d_regs(const struct kvm_vcpu *vcpu) 839 { 840 const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 841 842 if (riscv_isa_extension_available(vcpu->arch.isa, D)) 843 return sizeof(cntx->fp.d.f) / sizeof(u64) + 1; 844 else 845 return 0; 846 } 847 848 static int copy_fp_d_reg_indices(const struct kvm_vcpu *vcpu, 849 u64 __user *uindices) 850 { 851 int i; 852 int n = num_fp_d_regs(vcpu); 853 u64 reg; 854 855 /* copy fp.d.f indices */ 856 for (i = 0; i < n-1; i++) { 857 reg = KVM_REG_RISCV | KVM_REG_SIZE_U64 | 858 KVM_REG_RISCV_FP_D | i; 859 860 if (uindices) { 861 if (put_user(reg, uindices)) 862 return -EFAULT; 863 uindices++; 864 } 865 } 866 867 /* copy fp.d.fcsr indices */ 868 reg = KVM_REG_RISCV | KVM_REG_SIZE_U32 | KVM_REG_RISCV_FP_D | i; 869 if (uindices) { 870 if (put_user(reg, uindices)) 871 return -EFAULT; 872 uindices++; 873 } 874 875 return n; 876 } 877 878 static int copy_isa_ext_reg_indices(const struct kvm_vcpu *vcpu, 879 u64 __user *uindices) 880 { 881 unsigned long guest_ext; 882 unsigned int n = 0; 883 884 for (int i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { 885 u64 size = IS_ENABLED(CONFIG_32BIT) ? 886 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 887 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_ISA_EXT | i; 888 889 if (__kvm_riscv_isa_check_host(i, &guest_ext)) 890 continue; 891 892 if (uindices) { 893 if (put_user(reg, uindices)) 894 return -EFAULT; 895 uindices++; 896 } 897 898 n++; 899 } 900 901 return n; 902 } 903 904 static inline unsigned long num_isa_ext_regs(const struct kvm_vcpu *vcpu) 905 { 906 return copy_isa_ext_reg_indices(vcpu, NULL); 907 } 908 909 static unsigned long num_sbi_ext_regs(struct kvm_vcpu *vcpu) 910 { 911 return kvm_riscv_vcpu_reg_indices_sbi_ext(vcpu, NULL); 912 } 913 914 static inline unsigned long num_sbi_regs(struct kvm_vcpu *vcpu) 915 { 916 return kvm_riscv_vcpu_reg_indices_sbi(vcpu, NULL); 917 } 918 919 static inline unsigned long num_vector_regs(const struct kvm_vcpu *vcpu) 920 { 921 if (!riscv_isa_extension_available(vcpu->arch.isa, V)) 922 return 0; 923 924 /* vstart, vl, vtype, vcsr, vlenb and 32 vector regs */ 925 return 37; 926 } 927 928 static int copy_vector_reg_indices(const struct kvm_vcpu *vcpu, 929 u64 __user *uindices) 930 { 931 const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 932 int n = num_vector_regs(vcpu); 933 u64 reg, size; 934 int i; 935 936 if (n == 0) 937 return 0; 938 939 /* copy vstart, vl, vtype, vcsr and vlenb */ 940 size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 941 for (i = 0; i < 5; i++) { 942 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_VECTOR | i; 943 944 if (uindices) { 945 if (put_user(reg, uindices)) 946 return -EFAULT; 947 uindices++; 948 } 949 } 950 951 /* vector_regs have a variable 'vlenb' size */ 952 size = __builtin_ctzl(cntx->vector.vlenb); 953 size <<= KVM_REG_SIZE_SHIFT; 954 for (i = 0; i < 32; i++) { 955 reg = KVM_REG_RISCV | KVM_REG_RISCV_VECTOR | size | 956 KVM_REG_RISCV_VECTOR_REG(i); 957 958 if (uindices) { 959 if (put_user(reg, uindices)) 960 return -EFAULT; 961 uindices++; 962 } 963 } 964 965 return n; 966 } 967 968 /* 969 * kvm_riscv_vcpu_num_regs - how many registers do we present via KVM_GET/SET_ONE_REG 970 * 971 * This is for all registers. 972 */ 973 unsigned long kvm_riscv_vcpu_num_regs(struct kvm_vcpu *vcpu) 974 { 975 unsigned long res = 0; 976 977 res += num_config_regs(vcpu); 978 res += num_core_regs(); 979 res += num_csr_regs(vcpu); 980 res += num_timer_regs(); 981 res += num_fp_f_regs(vcpu); 982 res += num_fp_d_regs(vcpu); 983 res += num_vector_regs(vcpu); 984 res += num_isa_ext_regs(vcpu); 985 res += num_sbi_ext_regs(vcpu); 986 res += num_sbi_regs(vcpu); 987 988 return res; 989 } 990 991 /* 992 * kvm_riscv_vcpu_copy_reg_indices - get indices of all registers. 993 */ 994 int kvm_riscv_vcpu_copy_reg_indices(struct kvm_vcpu *vcpu, 995 u64 __user *uindices) 996 { 997 int ret; 998 999 ret = copy_config_reg_indices(vcpu, uindices); 1000 if (ret < 0) 1001 return ret; 1002 uindices += ret; 1003 1004 ret = copy_core_reg_indices(uindices); 1005 if (ret < 0) 1006 return ret; 1007 uindices += ret; 1008 1009 ret = copy_csr_reg_indices(vcpu, uindices); 1010 if (ret < 0) 1011 return ret; 1012 uindices += ret; 1013 1014 ret = copy_timer_reg_indices(uindices); 1015 if (ret < 0) 1016 return ret; 1017 uindices += ret; 1018 1019 ret = copy_fp_f_reg_indices(vcpu, uindices); 1020 if (ret < 0) 1021 return ret; 1022 uindices += ret; 1023 1024 ret = copy_fp_d_reg_indices(vcpu, uindices); 1025 if (ret < 0) 1026 return ret; 1027 uindices += ret; 1028 1029 ret = copy_vector_reg_indices(vcpu, uindices); 1030 if (ret < 0) 1031 return ret; 1032 uindices += ret; 1033 1034 ret = copy_isa_ext_reg_indices(vcpu, uindices); 1035 if (ret < 0) 1036 return ret; 1037 uindices += ret; 1038 1039 ret = kvm_riscv_vcpu_reg_indices_sbi_ext(vcpu, uindices); 1040 if (ret < 0) 1041 return ret; 1042 uindices += ret; 1043 1044 ret = kvm_riscv_vcpu_reg_indices_sbi(vcpu, uindices); 1045 if (ret < 0) 1046 return ret; 1047 uindices += ret; 1048 1049 return 0; 1050 } 1051 1052 int kvm_riscv_vcpu_set_reg(struct kvm_vcpu *vcpu, 1053 const struct kvm_one_reg *reg) 1054 { 1055 switch (reg->id & KVM_REG_RISCV_TYPE_MASK) { 1056 case KVM_REG_RISCV_CONFIG: 1057 return kvm_riscv_vcpu_set_reg_config(vcpu, reg); 1058 case KVM_REG_RISCV_CORE: 1059 return kvm_riscv_vcpu_set_reg_core(vcpu, reg); 1060 case KVM_REG_RISCV_CSR: 1061 return kvm_riscv_vcpu_set_reg_csr(vcpu, reg); 1062 case KVM_REG_RISCV_TIMER: 1063 return kvm_riscv_vcpu_set_reg_timer(vcpu, reg); 1064 case KVM_REG_RISCV_FP_F: 1065 return kvm_riscv_vcpu_set_reg_fp(vcpu, reg, 1066 KVM_REG_RISCV_FP_F); 1067 case KVM_REG_RISCV_FP_D: 1068 return kvm_riscv_vcpu_set_reg_fp(vcpu, reg, 1069 KVM_REG_RISCV_FP_D); 1070 case KVM_REG_RISCV_VECTOR: 1071 return kvm_riscv_vcpu_set_reg_vector(vcpu, reg); 1072 case KVM_REG_RISCV_ISA_EXT: 1073 return kvm_riscv_vcpu_set_reg_isa_ext(vcpu, reg); 1074 case KVM_REG_RISCV_SBI_EXT: 1075 return kvm_riscv_vcpu_set_reg_sbi_ext(vcpu, reg); 1076 case KVM_REG_RISCV_SBI_STATE: 1077 return kvm_riscv_vcpu_set_reg_sbi(vcpu, reg); 1078 default: 1079 break; 1080 } 1081 1082 return -ENOENT; 1083 } 1084 1085 int kvm_riscv_vcpu_get_reg(struct kvm_vcpu *vcpu, 1086 const struct kvm_one_reg *reg) 1087 { 1088 switch (reg->id & KVM_REG_RISCV_TYPE_MASK) { 1089 case KVM_REG_RISCV_CONFIG: 1090 return kvm_riscv_vcpu_get_reg_config(vcpu, reg); 1091 case KVM_REG_RISCV_CORE: 1092 return kvm_riscv_vcpu_get_reg_core(vcpu, reg); 1093 case KVM_REG_RISCV_CSR: 1094 return kvm_riscv_vcpu_get_reg_csr(vcpu, reg); 1095 case KVM_REG_RISCV_TIMER: 1096 return kvm_riscv_vcpu_get_reg_timer(vcpu, reg); 1097 case KVM_REG_RISCV_FP_F: 1098 return kvm_riscv_vcpu_get_reg_fp(vcpu, reg, 1099 KVM_REG_RISCV_FP_F); 1100 case KVM_REG_RISCV_FP_D: 1101 return kvm_riscv_vcpu_get_reg_fp(vcpu, reg, 1102 KVM_REG_RISCV_FP_D); 1103 case KVM_REG_RISCV_VECTOR: 1104 return kvm_riscv_vcpu_get_reg_vector(vcpu, reg); 1105 case KVM_REG_RISCV_ISA_EXT: 1106 return kvm_riscv_vcpu_get_reg_isa_ext(vcpu, reg); 1107 case KVM_REG_RISCV_SBI_EXT: 1108 return kvm_riscv_vcpu_get_reg_sbi_ext(vcpu, reg); 1109 case KVM_REG_RISCV_SBI_STATE: 1110 return kvm_riscv_vcpu_get_reg_sbi(vcpu, reg); 1111 default: 1112 break; 1113 } 1114 1115 return -ENOENT; 1116 } 1117