1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 4 * Copyright (C) 2023 Ventana Micro Systems Inc. 5 * 6 * Authors: 7 * Anup Patel <apatel@ventanamicro.com> 8 */ 9 10 #include <linux/bitops.h> 11 #include <linux/errno.h> 12 #include <linux/err.h> 13 #include <linux/nospec.h> 14 #include <linux/uaccess.h> 15 #include <linux/kvm_host.h> 16 #include <asm/cacheflush.h> 17 #include <asm/cpufeature.h> 18 #include <asm/kvm_isa.h> 19 #include <asm/kvm_vcpu_vector.h> 20 21 #define KVM_RISCV_BASE_ISA_MASK GENMASK(25, 0) 22 23 void kvm_riscv_vcpu_setup_isa(struct kvm_vcpu *vcpu) 24 { 25 unsigned long guest_ext, i; 26 27 for (i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { 28 if (__kvm_riscv_isa_check_host(i, &guest_ext)) 29 continue; 30 if (kvm_riscv_isa_enable_allowed(i)) 31 set_bit(guest_ext, vcpu->arch.isa); 32 } 33 } 34 35 static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu, 36 const struct kvm_one_reg *reg) 37 { 38 unsigned long __user *uaddr = 39 (unsigned long __user *)(unsigned long)reg->addr; 40 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 41 KVM_REG_SIZE_MASK | 42 KVM_REG_RISCV_CONFIG); 43 unsigned long reg_val; 44 45 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 46 return -EINVAL; 47 48 switch (reg_num) { 49 case KVM_REG_RISCV_CONFIG_REG(isa): 50 reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK; 51 break; 52 case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): 53 reg_val = (kvm_riscv_isa_check_host(ZICBOM)) ? 0 : riscv_cbom_block_size; 54 break; 55 case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): 56 reg_val = (kvm_riscv_isa_check_host(ZICBOZ)) ? 0 : riscv_cboz_block_size; 57 break; 58 case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): 59 reg_val = (kvm_riscv_isa_check_host(ZICBOP)) ? 0 : riscv_cbop_block_size; 60 break; 61 case KVM_REG_RISCV_CONFIG_REG(mvendorid): 62 reg_val = vcpu->arch.mvendorid; 63 break; 64 case KVM_REG_RISCV_CONFIG_REG(marchid): 65 reg_val = vcpu->arch.marchid; 66 break; 67 case KVM_REG_RISCV_CONFIG_REG(mimpid): 68 reg_val = vcpu->arch.mimpid; 69 break; 70 case KVM_REG_RISCV_CONFIG_REG(satp_mode): 71 reg_val = satp_mode >> SATP_MODE_SHIFT; 72 break; 73 default: 74 return -ENOENT; 75 } 76 77 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 78 return -EFAULT; 79 80 return 0; 81 } 82 83 static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu, 84 const struct kvm_one_reg *reg) 85 { 86 unsigned long __user *uaddr = 87 (unsigned long __user *)(unsigned long)reg->addr; 88 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 89 KVM_REG_SIZE_MASK | 90 KVM_REG_RISCV_CONFIG); 91 unsigned long i, isa_ext, reg_val; 92 93 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 94 return -EINVAL; 95 96 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 97 return -EFAULT; 98 99 switch (reg_num) { 100 case KVM_REG_RISCV_CONFIG_REG(isa): 101 /* 102 * This ONE REG interface is only defined for 103 * single letter extensions. 104 */ 105 if (fls(reg_val) >= RISCV_ISA_EXT_BASE) 106 return -EINVAL; 107 108 /* 109 * Return early (i.e. do nothing) if reg_val is the same 110 * value retrievable via kvm_riscv_vcpu_get_reg_config(). 111 */ 112 if (reg_val == (vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK)) 113 break; 114 115 if (!vcpu->arch.ran_atleast_once) { 116 /* Ignore the enable/disable request for certain extensions */ 117 for (i = 0; i < RISCV_ISA_EXT_BASE; i++) { 118 isa_ext = kvm_riscv_base2isa_ext(i); 119 if (isa_ext >= KVM_RISCV_ISA_EXT_MAX) { 120 reg_val &= ~BIT(i); 121 continue; 122 } 123 if (!kvm_riscv_isa_enable_allowed(isa_ext)) 124 if (reg_val & BIT(i)) 125 reg_val &= ~BIT(i); 126 if (!kvm_riscv_isa_disable_allowed(isa_ext)) 127 if (!(reg_val & BIT(i))) 128 reg_val |= BIT(i); 129 } 130 reg_val &= riscv_isa_extension_base(NULL); 131 /* Do not modify anything beyond single letter extensions */ 132 reg_val = (vcpu->arch.isa[0] & ~KVM_RISCV_BASE_ISA_MASK) | 133 (reg_val & KVM_RISCV_BASE_ISA_MASK); 134 vcpu->arch.isa[0] = reg_val; 135 kvm_riscv_vcpu_fp_reset(vcpu); 136 } else { 137 return -EBUSY; 138 } 139 break; 140 case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): 141 if (reg_val && reg_val != riscv_cbom_block_size) 142 return -EINVAL; 143 break; 144 case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): 145 if (reg_val && reg_val != riscv_cboz_block_size) 146 return -EINVAL; 147 break; 148 case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): 149 if (reg_val && reg_val != riscv_cbop_block_size) 150 return -EINVAL; 151 break; 152 case KVM_REG_RISCV_CONFIG_REG(mvendorid): 153 if (reg_val == vcpu->arch.mvendorid) 154 break; 155 if (!vcpu->arch.ran_atleast_once) 156 vcpu->arch.mvendorid = reg_val; 157 else 158 return -EBUSY; 159 break; 160 case KVM_REG_RISCV_CONFIG_REG(marchid): 161 if (reg_val == vcpu->arch.marchid) 162 break; 163 if (!vcpu->arch.ran_atleast_once) 164 vcpu->arch.marchid = reg_val; 165 else 166 return -EBUSY; 167 break; 168 case KVM_REG_RISCV_CONFIG_REG(mimpid): 169 if (reg_val == vcpu->arch.mimpid) 170 break; 171 if (!vcpu->arch.ran_atleast_once) 172 vcpu->arch.mimpid = reg_val; 173 else 174 return -EBUSY; 175 break; 176 case KVM_REG_RISCV_CONFIG_REG(satp_mode): 177 if (reg_val != (satp_mode >> SATP_MODE_SHIFT)) 178 return -EINVAL; 179 break; 180 default: 181 return -ENOENT; 182 } 183 184 return 0; 185 } 186 187 static int kvm_riscv_vcpu_get_reg_core(struct kvm_vcpu *vcpu, 188 const struct kvm_one_reg *reg) 189 { 190 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 191 unsigned long __user *uaddr = 192 (unsigned long __user *)(unsigned long)reg->addr; 193 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 194 KVM_REG_SIZE_MASK | 195 KVM_REG_RISCV_CORE); 196 unsigned long regs_max = sizeof(struct kvm_riscv_core) / sizeof(unsigned long); 197 unsigned long reg_val; 198 199 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 200 return -EINVAL; 201 if (reg_num >= regs_max) 202 return -ENOENT; 203 204 reg_num = array_index_nospec(reg_num, regs_max); 205 206 if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc)) 207 reg_val = cntx->sepc; 208 else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num && 209 reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6)) 210 reg_val = ((unsigned long *)cntx)[reg_num]; 211 else if (reg_num == KVM_REG_RISCV_CORE_REG(mode)) 212 reg_val = (cntx->sstatus & SR_SPP) ? 213 KVM_RISCV_MODE_S : KVM_RISCV_MODE_U; 214 else 215 return -ENOENT; 216 217 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 218 return -EFAULT; 219 220 return 0; 221 } 222 223 static int kvm_riscv_vcpu_set_reg_core(struct kvm_vcpu *vcpu, 224 const struct kvm_one_reg *reg) 225 { 226 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 227 unsigned long __user *uaddr = 228 (unsigned long __user *)(unsigned long)reg->addr; 229 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 230 KVM_REG_SIZE_MASK | 231 KVM_REG_RISCV_CORE); 232 unsigned long regs_max = sizeof(struct kvm_riscv_core) / sizeof(unsigned long); 233 unsigned long reg_val; 234 235 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 236 return -EINVAL; 237 if (reg_num >= regs_max) 238 return -ENOENT; 239 240 reg_num = array_index_nospec(reg_num, regs_max); 241 242 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 243 return -EFAULT; 244 245 if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc)) 246 cntx->sepc = reg_val; 247 else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num && 248 reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6)) 249 ((unsigned long *)cntx)[reg_num] = reg_val; 250 else if (reg_num == KVM_REG_RISCV_CORE_REG(mode)) { 251 if (reg_val == KVM_RISCV_MODE_S) 252 cntx->sstatus |= SR_SPP; 253 else 254 cntx->sstatus &= ~SR_SPP; 255 } else 256 return -ENOENT; 257 258 return 0; 259 } 260 261 static int kvm_riscv_vcpu_general_get_csr(struct kvm_vcpu *vcpu, 262 unsigned long reg_num, 263 unsigned long *out_val) 264 { 265 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 266 unsigned long regs_max = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); 267 268 if (reg_num >= regs_max) 269 return -ENOENT; 270 271 reg_num = array_index_nospec(reg_num, regs_max); 272 273 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { 274 kvm_riscv_vcpu_flush_interrupts(vcpu); 275 *out_val = (csr->hvip >> VSIP_TO_HVIP_SHIFT) & VSIP_VALID_MASK; 276 *out_val |= csr->hvip & ~IRQ_LOCAL_MASK; 277 } else 278 *out_val = ((unsigned long *)csr)[reg_num]; 279 280 return 0; 281 } 282 283 static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, 284 unsigned long reg_num, 285 unsigned long reg_val) 286 { 287 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 288 unsigned long regs_max = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); 289 unsigned long flags; 290 291 if (reg_num >= regs_max) 292 return -ENOENT; 293 294 reg_num = array_index_nospec(reg_num, regs_max); 295 296 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { 297 reg_val &= VSIP_VALID_MASK; 298 reg_val <<= VSIP_TO_HVIP_SHIFT; 299 } 300 301 ((unsigned long *)csr)[reg_num] = reg_val; 302 303 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { 304 raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); 305 vcpu->arch.irqs_pending_mask[0] = 0; 306 raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); 307 } 308 309 return 0; 310 } 311 312 static inline int kvm_riscv_vcpu_smstateen_set_csr(struct kvm_vcpu *vcpu, 313 unsigned long reg_num, 314 unsigned long reg_val) 315 { 316 struct kvm_vcpu_smstateen_csr *csr = &vcpu->arch.smstateen_csr; 317 unsigned long regs_max = sizeof(struct kvm_riscv_smstateen_csr) / 318 sizeof(unsigned long); 319 320 if (!riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) 321 return -ENOENT; 322 if (reg_num >= regs_max) 323 return -ENOENT; 324 325 reg_num = array_index_nospec(reg_num, regs_max); 326 327 ((unsigned long *)csr)[reg_num] = reg_val; 328 return 0; 329 } 330 331 static int kvm_riscv_vcpu_smstateen_get_csr(struct kvm_vcpu *vcpu, 332 unsigned long reg_num, 333 unsigned long *out_val) 334 { 335 struct kvm_vcpu_smstateen_csr *csr = &vcpu->arch.smstateen_csr; 336 unsigned long regs_max = sizeof(struct kvm_riscv_smstateen_csr) / 337 sizeof(unsigned long); 338 339 if (!riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) 340 return -ENOENT; 341 if (reg_num >= regs_max) 342 return -ENOENT; 343 344 reg_num = array_index_nospec(reg_num, regs_max); 345 346 *out_val = ((unsigned long *)csr)[reg_num]; 347 return 0; 348 } 349 350 static inline int kvm_riscv_vcpu_zicfiss_set_csr(struct kvm_vcpu *vcpu, 351 unsigned long reg_num, 352 unsigned long reg_val) 353 { 354 struct kvm_vcpu_zicfiss_csr *csr = &vcpu->arch.zicfiss_csr; 355 unsigned long regs_max = sizeof(struct kvm_vcpu_zicfiss_csr) / 356 sizeof(unsigned long); 357 358 if (!riscv_isa_extension_available(vcpu->arch.isa, ZICFISS)) 359 return -ENOENT; 360 if (reg_num >= regs_max) 361 return -ENOENT; 362 363 reg_num = array_index_nospec(reg_num, regs_max); 364 365 ((unsigned long *)csr)[reg_num] = reg_val; 366 return 0; 367 } 368 369 static int kvm_riscv_vcpu_zicfiss_get_csr(struct kvm_vcpu *vcpu, 370 unsigned long reg_num, 371 unsigned long *out_val) 372 { 373 struct kvm_vcpu_zicfiss_csr *csr = &vcpu->arch.zicfiss_csr; 374 unsigned long regs_max = sizeof(struct kvm_vcpu_zicfiss_csr) / 375 sizeof(unsigned long); 376 377 if (!riscv_isa_extension_available(vcpu->arch.isa, ZICFISS)) 378 return -ENOENT; 379 if (reg_num >= regs_max) 380 return -ENOENT; 381 382 reg_num = array_index_nospec(reg_num, regs_max); 383 384 *out_val = ((unsigned long *)csr)[reg_num]; 385 return 0; 386 } 387 388 static int kvm_riscv_vcpu_get_reg_csr(struct kvm_vcpu *vcpu, 389 const struct kvm_one_reg *reg) 390 { 391 int rc; 392 unsigned long __user *uaddr = 393 (unsigned long __user *)(unsigned long)reg->addr; 394 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 395 KVM_REG_SIZE_MASK | 396 KVM_REG_RISCV_CSR); 397 unsigned long reg_val, reg_subtype; 398 399 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 400 return -EINVAL; 401 402 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 403 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 404 switch (reg_subtype) { 405 case KVM_REG_RISCV_CSR_GENERAL: 406 rc = kvm_riscv_vcpu_general_get_csr(vcpu, reg_num, ®_val); 407 break; 408 case KVM_REG_RISCV_CSR_AIA: 409 rc = kvm_riscv_vcpu_aia_get_csr(vcpu, reg_num, ®_val); 410 break; 411 case KVM_REG_RISCV_CSR_SMSTATEEN: 412 rc = kvm_riscv_vcpu_smstateen_get_csr(vcpu, reg_num, ®_val); 413 break; 414 case KVM_REG_RISCV_CSR_ZICFISS: 415 rc = kvm_riscv_vcpu_zicfiss_get_csr(vcpu, reg_num, ®_val); 416 break; 417 default: 418 rc = -ENOENT; 419 break; 420 } 421 if (rc) 422 return rc; 423 424 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 425 return -EFAULT; 426 427 return 0; 428 } 429 430 static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu, 431 const struct kvm_one_reg *reg) 432 { 433 int rc; 434 unsigned long __user *uaddr = 435 (unsigned long __user *)(unsigned long)reg->addr; 436 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 437 KVM_REG_SIZE_MASK | 438 KVM_REG_RISCV_CSR); 439 unsigned long reg_val, reg_subtype; 440 441 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 442 return -EINVAL; 443 444 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 445 return -EFAULT; 446 447 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 448 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 449 switch (reg_subtype) { 450 case KVM_REG_RISCV_CSR_GENERAL: 451 rc = kvm_riscv_vcpu_general_set_csr(vcpu, reg_num, reg_val); 452 break; 453 case KVM_REG_RISCV_CSR_AIA: 454 rc = kvm_riscv_vcpu_aia_set_csr(vcpu, reg_num, reg_val); 455 break; 456 case KVM_REG_RISCV_CSR_SMSTATEEN: 457 rc = kvm_riscv_vcpu_smstateen_set_csr(vcpu, reg_num, reg_val); 458 break; 459 case KVM_REG_RISCV_CSR_ZICFISS: 460 rc = kvm_riscv_vcpu_zicfiss_set_csr(vcpu, reg_num, reg_val); 461 break; 462 default: 463 rc = -ENOENT; 464 break; 465 } 466 if (rc) 467 return rc; 468 469 vcpu->arch.csr_dirty = true; 470 471 return 0; 472 } 473 474 static int riscv_vcpu_get_isa_ext_single(struct kvm_vcpu *vcpu, 475 unsigned long reg_num, 476 unsigned long *reg_val) 477 { 478 unsigned long guest_ext; 479 int ret; 480 481 ret = __kvm_riscv_isa_check_host(reg_num, &guest_ext); 482 if (ret) 483 return ret; 484 485 *reg_val = 0; 486 if (__riscv_isa_extension_available(vcpu->arch.isa, guest_ext)) 487 *reg_val = 1; /* Mark the given extension as available */ 488 489 return 0; 490 } 491 492 static int riscv_vcpu_set_isa_ext_single(struct kvm_vcpu *vcpu, 493 unsigned long reg_num, 494 unsigned long reg_val) 495 { 496 unsigned long guest_ext; 497 int ret; 498 499 ret = __kvm_riscv_isa_check_host(reg_num, &guest_ext); 500 if (ret) 501 return ret; 502 503 if (reg_val == test_bit(guest_ext, vcpu->arch.isa)) 504 return 0; 505 506 if (!vcpu->arch.ran_atleast_once) { 507 /* 508 * All multi-letter extension and a few single letter 509 * extension can be disabled 510 */ 511 if (reg_val == 1 && 512 kvm_riscv_isa_enable_allowed(reg_num)) 513 set_bit(guest_ext, vcpu->arch.isa); 514 else if (!reg_val && 515 kvm_riscv_isa_disable_allowed(reg_num)) 516 clear_bit(guest_ext, vcpu->arch.isa); 517 else 518 return -EINVAL; 519 kvm_riscv_vcpu_fp_reset(vcpu); 520 } else { 521 return -EBUSY; 522 } 523 524 return 0; 525 } 526 527 static int riscv_vcpu_get_isa_ext_multi(struct kvm_vcpu *vcpu, 528 unsigned long reg_num, 529 unsigned long *reg_val) 530 { 531 unsigned long i, ext_id, ext_val; 532 533 if (reg_num > KVM_REG_RISCV_ISA_MULTI_REG_LAST) 534 return -ENOENT; 535 536 for (i = 0; i < BITS_PER_LONG; i++) { 537 ext_id = i + reg_num * BITS_PER_LONG; 538 if (ext_id >= KVM_RISCV_ISA_EXT_MAX) 539 break; 540 541 ext_val = 0; 542 riscv_vcpu_get_isa_ext_single(vcpu, ext_id, &ext_val); 543 if (ext_val) 544 *reg_val |= KVM_REG_RISCV_ISA_MULTI_MASK(ext_id); 545 } 546 547 return 0; 548 } 549 550 static int riscv_vcpu_set_isa_ext_multi(struct kvm_vcpu *vcpu, 551 unsigned long reg_num, 552 unsigned long reg_val, bool enable) 553 { 554 unsigned long i, ext_id; 555 556 if (reg_num > KVM_REG_RISCV_ISA_MULTI_REG_LAST) 557 return -ENOENT; 558 559 for_each_set_bit(i, ®_val, BITS_PER_LONG) { 560 ext_id = i + reg_num * BITS_PER_LONG; 561 if (ext_id >= KVM_RISCV_ISA_EXT_MAX) 562 break; 563 564 riscv_vcpu_set_isa_ext_single(vcpu, ext_id, enable); 565 } 566 567 return 0; 568 } 569 570 static int kvm_riscv_vcpu_get_reg_isa_ext(struct kvm_vcpu *vcpu, 571 const struct kvm_one_reg *reg) 572 { 573 int rc; 574 unsigned long __user *uaddr = 575 (unsigned long __user *)(unsigned long)reg->addr; 576 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 577 KVM_REG_SIZE_MASK | 578 KVM_REG_RISCV_ISA_EXT); 579 unsigned long reg_val, reg_subtype; 580 581 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 582 return -EINVAL; 583 584 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 585 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 586 587 reg_val = 0; 588 switch (reg_subtype) { 589 case KVM_REG_RISCV_ISA_SINGLE: 590 rc = riscv_vcpu_get_isa_ext_single(vcpu, reg_num, ®_val); 591 break; 592 case KVM_REG_RISCV_ISA_MULTI_EN: 593 case KVM_REG_RISCV_ISA_MULTI_DIS: 594 rc = riscv_vcpu_get_isa_ext_multi(vcpu, reg_num, ®_val); 595 if (!rc && reg_subtype == KVM_REG_RISCV_ISA_MULTI_DIS) 596 reg_val = ~reg_val; 597 break; 598 default: 599 rc = -ENOENT; 600 } 601 if (rc) 602 return rc; 603 604 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 605 return -EFAULT; 606 607 return 0; 608 } 609 610 static int kvm_riscv_vcpu_set_reg_isa_ext(struct kvm_vcpu *vcpu, 611 const struct kvm_one_reg *reg) 612 { 613 unsigned long __user *uaddr = 614 (unsigned long __user *)(unsigned long)reg->addr; 615 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 616 KVM_REG_SIZE_MASK | 617 KVM_REG_RISCV_ISA_EXT); 618 unsigned long reg_val, reg_subtype; 619 620 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 621 return -EINVAL; 622 623 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 624 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 625 626 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 627 return -EFAULT; 628 629 switch (reg_subtype) { 630 case KVM_REG_RISCV_ISA_SINGLE: 631 return riscv_vcpu_set_isa_ext_single(vcpu, reg_num, reg_val); 632 case KVM_REG_RISCV_ISA_MULTI_EN: 633 return riscv_vcpu_set_isa_ext_multi(vcpu, reg_num, reg_val, true); 634 case KVM_REG_RISCV_ISA_MULTI_DIS: 635 return riscv_vcpu_set_isa_ext_multi(vcpu, reg_num, reg_val, false); 636 default: 637 return -ENOENT; 638 } 639 640 return 0; 641 } 642 643 static int copy_config_reg_indices(const struct kvm_vcpu *vcpu, 644 u64 __user *uindices) 645 { 646 int n = 0; 647 648 for (int i = 0; i < sizeof(struct kvm_riscv_config)/sizeof(unsigned long); 649 i++) { 650 u64 size; 651 u64 reg; 652 653 size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 654 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CONFIG | i; 655 656 if (uindices) { 657 if (put_user(reg, uindices)) 658 return -EFAULT; 659 uindices++; 660 } 661 662 n++; 663 } 664 665 return n; 666 } 667 668 static unsigned long num_config_regs(const struct kvm_vcpu *vcpu) 669 { 670 return copy_config_reg_indices(vcpu, NULL); 671 } 672 673 static inline unsigned long num_core_regs(void) 674 { 675 return sizeof(struct kvm_riscv_core) / sizeof(unsigned long); 676 } 677 678 static int copy_core_reg_indices(u64 __user *uindices) 679 { 680 int n = num_core_regs(); 681 682 for (int i = 0; i < n; i++) { 683 u64 size = IS_ENABLED(CONFIG_32BIT) ? 684 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 685 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CORE | i; 686 687 if (uindices) { 688 if (put_user(reg, uindices)) 689 return -EFAULT; 690 uindices++; 691 } 692 } 693 694 return n; 695 } 696 697 static inline unsigned long num_csr_regs(const struct kvm_vcpu *vcpu) 698 { 699 unsigned long n = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); 700 701 if (riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) 702 n += sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); 703 if (riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) 704 n += sizeof(struct kvm_riscv_smstateen_csr) / sizeof(unsigned long); 705 if (riscv_isa_extension_available(vcpu->arch.isa, ZICFISS)) 706 n += sizeof(struct kvm_riscv_zicfiss_csr) / sizeof(unsigned long); 707 708 return n; 709 } 710 711 static int copy_csr_reg_indices(const struct kvm_vcpu *vcpu, 712 u64 __user *uindices) 713 { 714 int n1 = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); 715 int n2 = 0, n3 = 0, n4 = 0; 716 717 /* copy general csr regs */ 718 for (int i = 0; i < n1; i++) { 719 u64 size = IS_ENABLED(CONFIG_32BIT) ? 720 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 721 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR | 722 KVM_REG_RISCV_CSR_GENERAL | i; 723 724 if (uindices) { 725 if (put_user(reg, uindices)) 726 return -EFAULT; 727 uindices++; 728 } 729 } 730 731 /* copy AIA csr regs */ 732 if (riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) { 733 n2 = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); 734 735 for (int i = 0; i < n2; i++) { 736 u64 size = IS_ENABLED(CONFIG_32BIT) ? 737 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 738 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR | 739 KVM_REG_RISCV_CSR_AIA | i; 740 741 if (uindices) { 742 if (put_user(reg, uindices)) 743 return -EFAULT; 744 uindices++; 745 } 746 } 747 } 748 749 /* copy Smstateen csr regs */ 750 if (riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) { 751 n3 = sizeof(struct kvm_riscv_smstateen_csr) / sizeof(unsigned long); 752 753 for (int i = 0; i < n3; i++) { 754 u64 size = IS_ENABLED(CONFIG_32BIT) ? 755 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 756 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR | 757 KVM_REG_RISCV_CSR_SMSTATEEN | i; 758 759 if (uindices) { 760 if (put_user(reg, uindices)) 761 return -EFAULT; 762 uindices++; 763 } 764 } 765 } 766 767 /* copy Zicfiss csr regs */ 768 if (riscv_isa_extension_available(vcpu->arch.isa, ZICFISS)) { 769 n4 = sizeof(struct kvm_riscv_zicfiss_csr) / sizeof(unsigned long); 770 771 for (int i = 0; i < n4; i++) { 772 u64 size = IS_ENABLED(CONFIG_32BIT) ? 773 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 774 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR | 775 KVM_REG_RISCV_CSR_ZICFISS | i; 776 777 if (uindices) { 778 if (put_user(reg, uindices)) 779 return -EFAULT; 780 uindices++; 781 } 782 } 783 } 784 785 return n1 + n2 + n3 + n4; 786 } 787 788 static inline unsigned long num_timer_regs(void) 789 { 790 return sizeof(struct kvm_riscv_timer) / sizeof(u64); 791 } 792 793 static int copy_timer_reg_indices(u64 __user *uindices) 794 { 795 int n = num_timer_regs(); 796 797 for (int i = 0; i < n; i++) { 798 u64 reg = KVM_REG_RISCV | KVM_REG_SIZE_U64 | 799 KVM_REG_RISCV_TIMER | i; 800 801 if (uindices) { 802 if (put_user(reg, uindices)) 803 return -EFAULT; 804 uindices++; 805 } 806 } 807 808 return n; 809 } 810 811 static inline unsigned long num_fp_f_regs(const struct kvm_vcpu *vcpu) 812 { 813 const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 814 815 if (riscv_isa_extension_available(vcpu->arch.isa, F)) 816 return sizeof(cntx->fp.f) / sizeof(u32); 817 else 818 return 0; 819 } 820 821 static int copy_fp_f_reg_indices(const struct kvm_vcpu *vcpu, 822 u64 __user *uindices) 823 { 824 int n = num_fp_f_regs(vcpu); 825 826 for (int i = 0; i < n; i++) { 827 u64 reg = KVM_REG_RISCV | KVM_REG_SIZE_U32 | 828 KVM_REG_RISCV_FP_F | i; 829 830 if (uindices) { 831 if (put_user(reg, uindices)) 832 return -EFAULT; 833 uindices++; 834 } 835 } 836 837 return n; 838 } 839 840 static inline unsigned long num_fp_d_regs(const struct kvm_vcpu *vcpu) 841 { 842 const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 843 844 if (riscv_isa_extension_available(vcpu->arch.isa, D)) 845 return sizeof(cntx->fp.d.f) / sizeof(u64) + 1; 846 else 847 return 0; 848 } 849 850 static int copy_fp_d_reg_indices(const struct kvm_vcpu *vcpu, 851 u64 __user *uindices) 852 { 853 int i; 854 int n = num_fp_d_regs(vcpu); 855 u64 reg; 856 857 /* copy fp.d.f indices */ 858 for (i = 0; i < n-1; i++) { 859 reg = KVM_REG_RISCV | KVM_REG_SIZE_U64 | 860 KVM_REG_RISCV_FP_D | i; 861 862 if (uindices) { 863 if (put_user(reg, uindices)) 864 return -EFAULT; 865 uindices++; 866 } 867 } 868 869 /* copy fp.d.fcsr indices */ 870 reg = KVM_REG_RISCV | KVM_REG_SIZE_U32 | KVM_REG_RISCV_FP_D | i; 871 if (uindices) { 872 if (put_user(reg, uindices)) 873 return -EFAULT; 874 uindices++; 875 } 876 877 return n; 878 } 879 880 static int copy_isa_ext_reg_indices(const struct kvm_vcpu *vcpu, 881 u64 __user *uindices) 882 { 883 unsigned long guest_ext; 884 unsigned int n = 0; 885 886 for (int i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { 887 u64 size = IS_ENABLED(CONFIG_32BIT) ? 888 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 889 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_ISA_EXT | i; 890 891 if (__kvm_riscv_isa_check_host(i, &guest_ext)) 892 continue; 893 894 if (uindices) { 895 if (put_user(reg, uindices)) 896 return -EFAULT; 897 uindices++; 898 } 899 900 n++; 901 } 902 903 return n; 904 } 905 906 static inline unsigned long num_isa_ext_regs(const struct kvm_vcpu *vcpu) 907 { 908 return copy_isa_ext_reg_indices(vcpu, NULL); 909 } 910 911 static unsigned long num_sbi_ext_regs(struct kvm_vcpu *vcpu) 912 { 913 return kvm_riscv_vcpu_reg_indices_sbi_ext(vcpu, NULL); 914 } 915 916 static inline unsigned long num_sbi_regs(struct kvm_vcpu *vcpu) 917 { 918 return kvm_riscv_vcpu_reg_indices_sbi(vcpu, NULL); 919 } 920 921 static inline unsigned long num_vector_regs(const struct kvm_vcpu *vcpu) 922 { 923 if (!riscv_isa_extension_available(vcpu->arch.isa, V)) 924 return 0; 925 926 /* vstart, vl, vtype, vcsr, vlenb and 32 vector regs */ 927 return 37; 928 } 929 930 static int copy_vector_reg_indices(const struct kvm_vcpu *vcpu, 931 u64 __user *uindices) 932 { 933 const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 934 int n = num_vector_regs(vcpu); 935 u64 reg, size; 936 int i; 937 938 if (n == 0) 939 return 0; 940 941 /* copy vstart, vl, vtype, vcsr and vlenb */ 942 size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 943 for (i = 0; i < 5; i++) { 944 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_VECTOR | i; 945 946 if (uindices) { 947 if (put_user(reg, uindices)) 948 return -EFAULT; 949 uindices++; 950 } 951 } 952 953 /* vector_regs have a variable 'vlenb' size */ 954 size = __builtin_ctzl(cntx->vector.vlenb); 955 size <<= KVM_REG_SIZE_SHIFT; 956 for (i = 0; i < 32; i++) { 957 reg = KVM_REG_RISCV | KVM_REG_RISCV_VECTOR | size | 958 KVM_REG_RISCV_VECTOR_REG(i); 959 960 if (uindices) { 961 if (put_user(reg, uindices)) 962 return -EFAULT; 963 uindices++; 964 } 965 } 966 967 return n; 968 } 969 970 /* 971 * kvm_riscv_vcpu_num_regs - how many registers do we present via KVM_GET/SET_ONE_REG 972 * 973 * This is for all registers. 974 */ 975 unsigned long kvm_riscv_vcpu_num_regs(struct kvm_vcpu *vcpu) 976 { 977 unsigned long res = 0; 978 979 res += num_config_regs(vcpu); 980 res += num_core_regs(); 981 res += num_csr_regs(vcpu); 982 res += num_timer_regs(); 983 res += num_fp_f_regs(vcpu); 984 res += num_fp_d_regs(vcpu); 985 res += num_vector_regs(vcpu); 986 res += num_isa_ext_regs(vcpu); 987 res += num_sbi_ext_regs(vcpu); 988 res += num_sbi_regs(vcpu); 989 990 return res; 991 } 992 993 /* 994 * kvm_riscv_vcpu_copy_reg_indices - get indices of all registers. 995 */ 996 int kvm_riscv_vcpu_copy_reg_indices(struct kvm_vcpu *vcpu, 997 u64 __user *uindices) 998 { 999 int ret; 1000 1001 ret = copy_config_reg_indices(vcpu, uindices); 1002 if (ret < 0) 1003 return ret; 1004 uindices += ret; 1005 1006 ret = copy_core_reg_indices(uindices); 1007 if (ret < 0) 1008 return ret; 1009 uindices += ret; 1010 1011 ret = copy_csr_reg_indices(vcpu, uindices); 1012 if (ret < 0) 1013 return ret; 1014 uindices += ret; 1015 1016 ret = copy_timer_reg_indices(uindices); 1017 if (ret < 0) 1018 return ret; 1019 uindices += ret; 1020 1021 ret = copy_fp_f_reg_indices(vcpu, uindices); 1022 if (ret < 0) 1023 return ret; 1024 uindices += ret; 1025 1026 ret = copy_fp_d_reg_indices(vcpu, uindices); 1027 if (ret < 0) 1028 return ret; 1029 uindices += ret; 1030 1031 ret = copy_vector_reg_indices(vcpu, uindices); 1032 if (ret < 0) 1033 return ret; 1034 uindices += ret; 1035 1036 ret = copy_isa_ext_reg_indices(vcpu, uindices); 1037 if (ret < 0) 1038 return ret; 1039 uindices += ret; 1040 1041 ret = kvm_riscv_vcpu_reg_indices_sbi_ext(vcpu, uindices); 1042 if (ret < 0) 1043 return ret; 1044 uindices += ret; 1045 1046 ret = kvm_riscv_vcpu_reg_indices_sbi(vcpu, uindices); 1047 if (ret < 0) 1048 return ret; 1049 uindices += ret; 1050 1051 return 0; 1052 } 1053 1054 int kvm_riscv_vcpu_set_reg(struct kvm_vcpu *vcpu, 1055 const struct kvm_one_reg *reg) 1056 { 1057 switch (reg->id & KVM_REG_RISCV_TYPE_MASK) { 1058 case KVM_REG_RISCV_CONFIG: 1059 return kvm_riscv_vcpu_set_reg_config(vcpu, reg); 1060 case KVM_REG_RISCV_CORE: 1061 return kvm_riscv_vcpu_set_reg_core(vcpu, reg); 1062 case KVM_REG_RISCV_CSR: 1063 return kvm_riscv_vcpu_set_reg_csr(vcpu, reg); 1064 case KVM_REG_RISCV_TIMER: 1065 return kvm_riscv_vcpu_set_reg_timer(vcpu, reg); 1066 case KVM_REG_RISCV_FP_F: 1067 return kvm_riscv_vcpu_set_reg_fp(vcpu, reg, 1068 KVM_REG_RISCV_FP_F); 1069 case KVM_REG_RISCV_FP_D: 1070 return kvm_riscv_vcpu_set_reg_fp(vcpu, reg, 1071 KVM_REG_RISCV_FP_D); 1072 case KVM_REG_RISCV_VECTOR: 1073 return kvm_riscv_vcpu_set_reg_vector(vcpu, reg); 1074 case KVM_REG_RISCV_ISA_EXT: 1075 return kvm_riscv_vcpu_set_reg_isa_ext(vcpu, reg); 1076 case KVM_REG_RISCV_SBI_EXT: 1077 return kvm_riscv_vcpu_set_reg_sbi_ext(vcpu, reg); 1078 case KVM_REG_RISCV_SBI_STATE: 1079 return kvm_riscv_vcpu_set_reg_sbi(vcpu, reg); 1080 default: 1081 break; 1082 } 1083 1084 return -ENOENT; 1085 } 1086 1087 int kvm_riscv_vcpu_get_reg(struct kvm_vcpu *vcpu, 1088 const struct kvm_one_reg *reg) 1089 { 1090 switch (reg->id & KVM_REG_RISCV_TYPE_MASK) { 1091 case KVM_REG_RISCV_CONFIG: 1092 return kvm_riscv_vcpu_get_reg_config(vcpu, reg); 1093 case KVM_REG_RISCV_CORE: 1094 return kvm_riscv_vcpu_get_reg_core(vcpu, reg); 1095 case KVM_REG_RISCV_CSR: 1096 return kvm_riscv_vcpu_get_reg_csr(vcpu, reg); 1097 case KVM_REG_RISCV_TIMER: 1098 return kvm_riscv_vcpu_get_reg_timer(vcpu, reg); 1099 case KVM_REG_RISCV_FP_F: 1100 return kvm_riscv_vcpu_get_reg_fp(vcpu, reg, 1101 KVM_REG_RISCV_FP_F); 1102 case KVM_REG_RISCV_FP_D: 1103 return kvm_riscv_vcpu_get_reg_fp(vcpu, reg, 1104 KVM_REG_RISCV_FP_D); 1105 case KVM_REG_RISCV_VECTOR: 1106 return kvm_riscv_vcpu_get_reg_vector(vcpu, reg); 1107 case KVM_REG_RISCV_ISA_EXT: 1108 return kvm_riscv_vcpu_get_reg_isa_ext(vcpu, reg); 1109 case KVM_REG_RISCV_SBI_EXT: 1110 return kvm_riscv_vcpu_get_reg_sbi_ext(vcpu, reg); 1111 case KVM_REG_RISCV_SBI_STATE: 1112 return kvm_riscv_vcpu_get_reg_sbi(vcpu, reg); 1113 default: 1114 break; 1115 } 1116 1117 return -ENOENT; 1118 } 1119