1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 4 * Copyright (C) 2023 Ventana Micro Systems Inc. 5 * 6 * Authors: 7 * Anup Patel <apatel@ventanamicro.com> 8 */ 9 10 #include <linux/bitops.h> 11 #include <linux/errno.h> 12 #include <linux/err.h> 13 #include <linux/nospec.h> 14 #include <linux/uaccess.h> 15 #include <linux/kvm_host.h> 16 #include <asm/cacheflush.h> 17 #include <asm/cpufeature.h> 18 #include <asm/kvm_isa.h> 19 #include <asm/kvm_vcpu_vector.h> 20 21 #define KVM_RISCV_BASE_ISA_MASK GENMASK(25, 0) 22 23 void kvm_riscv_vcpu_setup_isa(struct kvm_vcpu *vcpu) 24 { 25 unsigned long guest_ext, i; 26 27 for (i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { 28 if (__kvm_riscv_isa_check_host(i, &guest_ext)) 29 continue; 30 if (kvm_riscv_isa_enable_allowed(i)) 31 set_bit(guest_ext, vcpu->arch.isa); 32 } 33 } 34 35 static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu, 36 const struct kvm_one_reg *reg) 37 { 38 unsigned long __user *uaddr = 39 (unsigned long __user *)(unsigned long)reg->addr; 40 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 41 KVM_REG_SIZE_MASK | 42 KVM_REG_RISCV_CONFIG); 43 unsigned long reg_val; 44 45 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 46 return -EINVAL; 47 48 switch (reg_num) { 49 case KVM_REG_RISCV_CONFIG_REG(isa): 50 reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK; 51 break; 52 case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): 53 reg_val = (kvm_riscv_isa_check_host(ZICBOM)) ? 0 : riscv_cbom_block_size; 54 break; 55 case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): 56 reg_val = (kvm_riscv_isa_check_host(ZICBOZ)) ? 0 : riscv_cboz_block_size; 57 break; 58 case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): 59 reg_val = (kvm_riscv_isa_check_host(ZICBOP)) ? 0 : riscv_cbop_block_size; 60 break; 61 case KVM_REG_RISCV_CONFIG_REG(mvendorid): 62 reg_val = vcpu->arch.mvendorid; 63 break; 64 case KVM_REG_RISCV_CONFIG_REG(marchid): 65 reg_val = vcpu->arch.marchid; 66 break; 67 case KVM_REG_RISCV_CONFIG_REG(mimpid): 68 reg_val = vcpu->arch.mimpid; 69 break; 70 case KVM_REG_RISCV_CONFIG_REG(satp_mode): 71 reg_val = satp_mode >> SATP_MODE_SHIFT; 72 break; 73 default: 74 return -ENOENT; 75 } 76 77 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 78 return -EFAULT; 79 80 return 0; 81 } 82 83 static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu, 84 const struct kvm_one_reg *reg) 85 { 86 unsigned long __user *uaddr = 87 (unsigned long __user *)(unsigned long)reg->addr; 88 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 89 KVM_REG_SIZE_MASK | 90 KVM_REG_RISCV_CONFIG); 91 unsigned long i, isa_ext, reg_val; 92 93 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 94 return -EINVAL; 95 96 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 97 return -EFAULT; 98 99 switch (reg_num) { 100 case KVM_REG_RISCV_CONFIG_REG(isa): 101 /* 102 * This ONE REG interface is only defined for 103 * single letter extensions. 104 */ 105 if (fls(reg_val) >= RISCV_ISA_EXT_BASE) 106 return -EINVAL; 107 108 /* 109 * Return early (i.e. do nothing) if reg_val is the same 110 * value retrievable via kvm_riscv_vcpu_get_reg_config(). 111 */ 112 if (reg_val == (vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK)) 113 break; 114 115 if (!vcpu->arch.ran_atleast_once) { 116 /* Ignore the enable/disable request for certain extensions */ 117 for (i = 0; i < RISCV_ISA_EXT_BASE; i++) { 118 isa_ext = kvm_riscv_base2isa_ext(i); 119 if (isa_ext >= KVM_RISCV_ISA_EXT_MAX) { 120 reg_val &= ~BIT(i); 121 continue; 122 } 123 if (!kvm_riscv_isa_enable_allowed(isa_ext)) 124 if (reg_val & BIT(i)) 125 reg_val &= ~BIT(i); 126 if (!kvm_riscv_isa_disable_allowed(isa_ext)) 127 if (!(reg_val & BIT(i))) 128 reg_val |= BIT(i); 129 } 130 reg_val &= riscv_isa_extension_base(NULL); 131 /* Do not modify anything beyond single letter extensions */ 132 reg_val = (vcpu->arch.isa[0] & ~KVM_RISCV_BASE_ISA_MASK) | 133 (reg_val & KVM_RISCV_BASE_ISA_MASK); 134 vcpu->arch.isa[0] = reg_val; 135 kvm_riscv_vcpu_fp_reset(vcpu); 136 } else { 137 return -EBUSY; 138 } 139 break; 140 case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): 141 if (reg_val && reg_val != riscv_cbom_block_size) 142 return -EINVAL; 143 break; 144 case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): 145 if (reg_val && reg_val != riscv_cboz_block_size) 146 return -EINVAL; 147 break; 148 case KVM_REG_RISCV_CONFIG_REG(zicbop_block_size): 149 if (reg_val && reg_val != riscv_cbop_block_size) 150 return -EINVAL; 151 break; 152 case KVM_REG_RISCV_CONFIG_REG(mvendorid): 153 if (reg_val == vcpu->arch.mvendorid) 154 break; 155 if (!vcpu->arch.ran_atleast_once) 156 vcpu->arch.mvendorid = reg_val; 157 else 158 return -EBUSY; 159 break; 160 case KVM_REG_RISCV_CONFIG_REG(marchid): 161 if (reg_val == vcpu->arch.marchid) 162 break; 163 if (!vcpu->arch.ran_atleast_once) 164 vcpu->arch.marchid = reg_val; 165 else 166 return -EBUSY; 167 break; 168 case KVM_REG_RISCV_CONFIG_REG(mimpid): 169 if (reg_val == vcpu->arch.mimpid) 170 break; 171 if (!vcpu->arch.ran_atleast_once) 172 vcpu->arch.mimpid = reg_val; 173 else 174 return -EBUSY; 175 break; 176 case KVM_REG_RISCV_CONFIG_REG(satp_mode): 177 if (reg_val != (satp_mode >> SATP_MODE_SHIFT)) 178 return -EINVAL; 179 break; 180 default: 181 return -ENOENT; 182 } 183 184 return 0; 185 } 186 187 static int kvm_riscv_vcpu_get_reg_core(struct kvm_vcpu *vcpu, 188 const struct kvm_one_reg *reg) 189 { 190 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 191 unsigned long __user *uaddr = 192 (unsigned long __user *)(unsigned long)reg->addr; 193 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 194 KVM_REG_SIZE_MASK | 195 KVM_REG_RISCV_CORE); 196 unsigned long regs_max = sizeof(struct kvm_riscv_core) / sizeof(unsigned long); 197 unsigned long reg_val; 198 199 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 200 return -EINVAL; 201 if (reg_num >= regs_max) 202 return -ENOENT; 203 204 reg_num = array_index_nospec(reg_num, regs_max); 205 206 if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc)) 207 reg_val = cntx->sepc; 208 else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num && 209 reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6)) 210 reg_val = ((unsigned long *)cntx)[reg_num]; 211 else if (reg_num == KVM_REG_RISCV_CORE_REG(mode)) 212 reg_val = (cntx->sstatus & SR_SPP) ? 213 KVM_RISCV_MODE_S : KVM_RISCV_MODE_U; 214 else 215 return -ENOENT; 216 217 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 218 return -EFAULT; 219 220 return 0; 221 } 222 223 static int kvm_riscv_vcpu_set_reg_core(struct kvm_vcpu *vcpu, 224 const struct kvm_one_reg *reg) 225 { 226 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 227 unsigned long __user *uaddr = 228 (unsigned long __user *)(unsigned long)reg->addr; 229 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 230 KVM_REG_SIZE_MASK | 231 KVM_REG_RISCV_CORE); 232 unsigned long regs_max = sizeof(struct kvm_riscv_core) / sizeof(unsigned long); 233 unsigned long reg_val; 234 235 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 236 return -EINVAL; 237 if (reg_num >= regs_max) 238 return -ENOENT; 239 240 reg_num = array_index_nospec(reg_num, regs_max); 241 242 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 243 return -EFAULT; 244 245 if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc)) 246 cntx->sepc = reg_val; 247 else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num && 248 reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6)) 249 ((unsigned long *)cntx)[reg_num] = reg_val; 250 else if (reg_num == KVM_REG_RISCV_CORE_REG(mode)) { 251 if (reg_val == KVM_RISCV_MODE_S) 252 cntx->sstatus |= SR_SPP; 253 else 254 cntx->sstatus &= ~SR_SPP; 255 } else 256 return -ENOENT; 257 258 return 0; 259 } 260 261 static int kvm_riscv_vcpu_general_get_csr(struct kvm_vcpu *vcpu, 262 unsigned long reg_num, 263 unsigned long *out_val) 264 { 265 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 266 unsigned long regs_max = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); 267 268 if (reg_num >= regs_max) 269 return -ENOENT; 270 271 reg_num = array_index_nospec(reg_num, regs_max); 272 273 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { 274 kvm_riscv_vcpu_flush_interrupts(vcpu); 275 *out_val = (csr->hvip >> VSIP_TO_HVIP_SHIFT) & VSIP_VALID_MASK; 276 *out_val |= csr->hvip & ~IRQ_LOCAL_MASK; 277 } else 278 *out_val = ((unsigned long *)csr)[reg_num]; 279 280 return 0; 281 } 282 283 static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, 284 unsigned long reg_num, 285 unsigned long reg_val) 286 { 287 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 288 unsigned long regs_max = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); 289 unsigned long flags; 290 291 if (reg_num >= regs_max) 292 return -ENOENT; 293 294 reg_num = array_index_nospec(reg_num, regs_max); 295 296 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { 297 reg_val &= VSIP_VALID_MASK; 298 reg_val <<= VSIP_TO_HVIP_SHIFT; 299 } 300 301 ((unsigned long *)csr)[reg_num] = reg_val; 302 303 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { 304 raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); 305 vcpu->arch.irqs_pending_mask[0] = 0; 306 raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); 307 } 308 309 return 0; 310 } 311 312 static inline int kvm_riscv_vcpu_smstateen_set_csr(struct kvm_vcpu *vcpu, 313 unsigned long reg_num, 314 unsigned long reg_val) 315 { 316 struct kvm_vcpu_smstateen_csr *csr = &vcpu->arch.smstateen_csr; 317 unsigned long regs_max = sizeof(struct kvm_riscv_smstateen_csr) / 318 sizeof(unsigned long); 319 320 if (!riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) 321 return -ENOENT; 322 if (reg_num >= regs_max) 323 return -ENOENT; 324 325 reg_num = array_index_nospec(reg_num, regs_max); 326 327 ((unsigned long *)csr)[reg_num] = reg_val; 328 return 0; 329 } 330 331 static int kvm_riscv_vcpu_smstateen_get_csr(struct kvm_vcpu *vcpu, 332 unsigned long reg_num, 333 unsigned long *out_val) 334 { 335 struct kvm_vcpu_smstateen_csr *csr = &vcpu->arch.smstateen_csr; 336 unsigned long regs_max = sizeof(struct kvm_riscv_smstateen_csr) / 337 sizeof(unsigned long); 338 339 if (!riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) 340 return -ENOENT; 341 if (reg_num >= regs_max) 342 return -ENOENT; 343 344 reg_num = array_index_nospec(reg_num, regs_max); 345 346 *out_val = ((unsigned long *)csr)[reg_num]; 347 return 0; 348 } 349 350 static int kvm_riscv_vcpu_get_reg_csr(struct kvm_vcpu *vcpu, 351 const struct kvm_one_reg *reg) 352 { 353 int rc; 354 unsigned long __user *uaddr = 355 (unsigned long __user *)(unsigned long)reg->addr; 356 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 357 KVM_REG_SIZE_MASK | 358 KVM_REG_RISCV_CSR); 359 unsigned long reg_val, reg_subtype; 360 361 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 362 return -EINVAL; 363 364 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 365 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 366 switch (reg_subtype) { 367 case KVM_REG_RISCV_CSR_GENERAL: 368 rc = kvm_riscv_vcpu_general_get_csr(vcpu, reg_num, ®_val); 369 break; 370 case KVM_REG_RISCV_CSR_AIA: 371 rc = kvm_riscv_vcpu_aia_get_csr(vcpu, reg_num, ®_val); 372 break; 373 case KVM_REG_RISCV_CSR_SMSTATEEN: 374 rc = kvm_riscv_vcpu_smstateen_get_csr(vcpu, reg_num, ®_val); 375 break; 376 default: 377 rc = -ENOENT; 378 break; 379 } 380 if (rc) 381 return rc; 382 383 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 384 return -EFAULT; 385 386 return 0; 387 } 388 389 static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu, 390 const struct kvm_one_reg *reg) 391 { 392 int rc; 393 unsigned long __user *uaddr = 394 (unsigned long __user *)(unsigned long)reg->addr; 395 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 396 KVM_REG_SIZE_MASK | 397 KVM_REG_RISCV_CSR); 398 unsigned long reg_val, reg_subtype; 399 400 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 401 return -EINVAL; 402 403 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 404 return -EFAULT; 405 406 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 407 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 408 switch (reg_subtype) { 409 case KVM_REG_RISCV_CSR_GENERAL: 410 rc = kvm_riscv_vcpu_general_set_csr(vcpu, reg_num, reg_val); 411 break; 412 case KVM_REG_RISCV_CSR_AIA: 413 rc = kvm_riscv_vcpu_aia_set_csr(vcpu, reg_num, reg_val); 414 break; 415 case KVM_REG_RISCV_CSR_SMSTATEEN: 416 rc = kvm_riscv_vcpu_smstateen_set_csr(vcpu, reg_num, reg_val); 417 break; 418 default: 419 rc = -ENOENT; 420 break; 421 } 422 if (rc) 423 return rc; 424 425 vcpu->arch.csr_dirty = true; 426 427 return 0; 428 } 429 430 static int riscv_vcpu_get_isa_ext_single(struct kvm_vcpu *vcpu, 431 unsigned long reg_num, 432 unsigned long *reg_val) 433 { 434 unsigned long guest_ext; 435 int ret; 436 437 ret = __kvm_riscv_isa_check_host(reg_num, &guest_ext); 438 if (ret) 439 return ret; 440 441 *reg_val = 0; 442 if (__riscv_isa_extension_available(vcpu->arch.isa, guest_ext)) 443 *reg_val = 1; /* Mark the given extension as available */ 444 445 return 0; 446 } 447 448 static int riscv_vcpu_set_isa_ext_single(struct kvm_vcpu *vcpu, 449 unsigned long reg_num, 450 unsigned long reg_val) 451 { 452 unsigned long guest_ext; 453 int ret; 454 455 ret = __kvm_riscv_isa_check_host(reg_num, &guest_ext); 456 if (ret) 457 return ret; 458 459 if (reg_val == test_bit(guest_ext, vcpu->arch.isa)) 460 return 0; 461 462 if (!vcpu->arch.ran_atleast_once) { 463 /* 464 * All multi-letter extension and a few single letter 465 * extension can be disabled 466 */ 467 if (reg_val == 1 && 468 kvm_riscv_isa_enable_allowed(reg_num)) 469 set_bit(guest_ext, vcpu->arch.isa); 470 else if (!reg_val && 471 kvm_riscv_isa_disable_allowed(reg_num)) 472 clear_bit(guest_ext, vcpu->arch.isa); 473 else 474 return -EINVAL; 475 kvm_riscv_vcpu_fp_reset(vcpu); 476 } else { 477 return -EBUSY; 478 } 479 480 return 0; 481 } 482 483 static int riscv_vcpu_get_isa_ext_multi(struct kvm_vcpu *vcpu, 484 unsigned long reg_num, 485 unsigned long *reg_val) 486 { 487 unsigned long i, ext_id, ext_val; 488 489 if (reg_num > KVM_REG_RISCV_ISA_MULTI_REG_LAST) 490 return -ENOENT; 491 492 for (i = 0; i < BITS_PER_LONG; i++) { 493 ext_id = i + reg_num * BITS_PER_LONG; 494 if (ext_id >= KVM_RISCV_ISA_EXT_MAX) 495 break; 496 497 ext_val = 0; 498 riscv_vcpu_get_isa_ext_single(vcpu, ext_id, &ext_val); 499 if (ext_val) 500 *reg_val |= KVM_REG_RISCV_ISA_MULTI_MASK(ext_id); 501 } 502 503 return 0; 504 } 505 506 static int riscv_vcpu_set_isa_ext_multi(struct kvm_vcpu *vcpu, 507 unsigned long reg_num, 508 unsigned long reg_val, bool enable) 509 { 510 unsigned long i, ext_id; 511 512 if (reg_num > KVM_REG_RISCV_ISA_MULTI_REG_LAST) 513 return -ENOENT; 514 515 for_each_set_bit(i, ®_val, BITS_PER_LONG) { 516 ext_id = i + reg_num * BITS_PER_LONG; 517 if (ext_id >= KVM_RISCV_ISA_EXT_MAX) 518 break; 519 520 riscv_vcpu_set_isa_ext_single(vcpu, ext_id, enable); 521 } 522 523 return 0; 524 } 525 526 static int kvm_riscv_vcpu_get_reg_isa_ext(struct kvm_vcpu *vcpu, 527 const struct kvm_one_reg *reg) 528 { 529 int rc; 530 unsigned long __user *uaddr = 531 (unsigned long __user *)(unsigned long)reg->addr; 532 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 533 KVM_REG_SIZE_MASK | 534 KVM_REG_RISCV_ISA_EXT); 535 unsigned long reg_val, reg_subtype; 536 537 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 538 return -EINVAL; 539 540 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 541 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 542 543 reg_val = 0; 544 switch (reg_subtype) { 545 case KVM_REG_RISCV_ISA_SINGLE: 546 rc = riscv_vcpu_get_isa_ext_single(vcpu, reg_num, ®_val); 547 break; 548 case KVM_REG_RISCV_ISA_MULTI_EN: 549 case KVM_REG_RISCV_ISA_MULTI_DIS: 550 rc = riscv_vcpu_get_isa_ext_multi(vcpu, reg_num, ®_val); 551 if (!rc && reg_subtype == KVM_REG_RISCV_ISA_MULTI_DIS) 552 reg_val = ~reg_val; 553 break; 554 default: 555 rc = -ENOENT; 556 } 557 if (rc) 558 return rc; 559 560 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 561 return -EFAULT; 562 563 return 0; 564 } 565 566 static int kvm_riscv_vcpu_set_reg_isa_ext(struct kvm_vcpu *vcpu, 567 const struct kvm_one_reg *reg) 568 { 569 unsigned long __user *uaddr = 570 (unsigned long __user *)(unsigned long)reg->addr; 571 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 572 KVM_REG_SIZE_MASK | 573 KVM_REG_RISCV_ISA_EXT); 574 unsigned long reg_val, reg_subtype; 575 576 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 577 return -EINVAL; 578 579 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 580 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 581 582 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 583 return -EFAULT; 584 585 switch (reg_subtype) { 586 case KVM_REG_RISCV_ISA_SINGLE: 587 return riscv_vcpu_set_isa_ext_single(vcpu, reg_num, reg_val); 588 case KVM_REG_RISCV_ISA_MULTI_EN: 589 return riscv_vcpu_set_isa_ext_multi(vcpu, reg_num, reg_val, true); 590 case KVM_REG_RISCV_ISA_MULTI_DIS: 591 return riscv_vcpu_set_isa_ext_multi(vcpu, reg_num, reg_val, false); 592 default: 593 return -ENOENT; 594 } 595 596 return 0; 597 } 598 599 static int copy_config_reg_indices(const struct kvm_vcpu *vcpu, 600 u64 __user *uindices) 601 { 602 int n = 0; 603 604 for (int i = 0; i < sizeof(struct kvm_riscv_config)/sizeof(unsigned long); 605 i++) { 606 u64 size; 607 u64 reg; 608 609 size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 610 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CONFIG | i; 611 612 if (uindices) { 613 if (put_user(reg, uindices)) 614 return -EFAULT; 615 uindices++; 616 } 617 618 n++; 619 } 620 621 return n; 622 } 623 624 static unsigned long num_config_regs(const struct kvm_vcpu *vcpu) 625 { 626 return copy_config_reg_indices(vcpu, NULL); 627 } 628 629 static inline unsigned long num_core_regs(void) 630 { 631 return sizeof(struct kvm_riscv_core) / sizeof(unsigned long); 632 } 633 634 static int copy_core_reg_indices(u64 __user *uindices) 635 { 636 int n = num_core_regs(); 637 638 for (int i = 0; i < n; i++) { 639 u64 size = IS_ENABLED(CONFIG_32BIT) ? 640 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 641 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CORE | i; 642 643 if (uindices) { 644 if (put_user(reg, uindices)) 645 return -EFAULT; 646 uindices++; 647 } 648 } 649 650 return n; 651 } 652 653 static inline unsigned long num_csr_regs(const struct kvm_vcpu *vcpu) 654 { 655 unsigned long n = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); 656 657 if (riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) 658 n += sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); 659 if (riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) 660 n += sizeof(struct kvm_riscv_smstateen_csr) / sizeof(unsigned long); 661 662 return n; 663 } 664 665 static int copy_csr_reg_indices(const struct kvm_vcpu *vcpu, 666 u64 __user *uindices) 667 { 668 int n1 = sizeof(struct kvm_riscv_csr) / sizeof(unsigned long); 669 int n2 = 0, n3 = 0; 670 671 /* copy general csr regs */ 672 for (int i = 0; i < n1; i++) { 673 u64 size = IS_ENABLED(CONFIG_32BIT) ? 674 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 675 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR | 676 KVM_REG_RISCV_CSR_GENERAL | i; 677 678 if (uindices) { 679 if (put_user(reg, uindices)) 680 return -EFAULT; 681 uindices++; 682 } 683 } 684 685 /* copy AIA csr regs */ 686 if (riscv_isa_extension_available(vcpu->arch.isa, SSAIA)) { 687 n2 = sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long); 688 689 for (int i = 0; i < n2; i++) { 690 u64 size = IS_ENABLED(CONFIG_32BIT) ? 691 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 692 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR | 693 KVM_REG_RISCV_CSR_AIA | i; 694 695 if (uindices) { 696 if (put_user(reg, uindices)) 697 return -EFAULT; 698 uindices++; 699 } 700 } 701 } 702 703 /* copy Smstateen csr regs */ 704 if (riscv_isa_extension_available(vcpu->arch.isa, SMSTATEEN)) { 705 n3 = sizeof(struct kvm_riscv_smstateen_csr) / sizeof(unsigned long); 706 707 for (int i = 0; i < n3; i++) { 708 u64 size = IS_ENABLED(CONFIG_32BIT) ? 709 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 710 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_CSR | 711 KVM_REG_RISCV_CSR_SMSTATEEN | i; 712 713 if (uindices) { 714 if (put_user(reg, uindices)) 715 return -EFAULT; 716 uindices++; 717 } 718 } 719 } 720 721 return n1 + n2 + n3; 722 } 723 724 static inline unsigned long num_timer_regs(void) 725 { 726 return sizeof(struct kvm_riscv_timer) / sizeof(u64); 727 } 728 729 static int copy_timer_reg_indices(u64 __user *uindices) 730 { 731 int n = num_timer_regs(); 732 733 for (int i = 0; i < n; i++) { 734 u64 reg = KVM_REG_RISCV | KVM_REG_SIZE_U64 | 735 KVM_REG_RISCV_TIMER | i; 736 737 if (uindices) { 738 if (put_user(reg, uindices)) 739 return -EFAULT; 740 uindices++; 741 } 742 } 743 744 return n; 745 } 746 747 static inline unsigned long num_fp_f_regs(const struct kvm_vcpu *vcpu) 748 { 749 const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 750 751 if (riscv_isa_extension_available(vcpu->arch.isa, f)) 752 return sizeof(cntx->fp.f) / sizeof(u32); 753 else 754 return 0; 755 } 756 757 static int copy_fp_f_reg_indices(const struct kvm_vcpu *vcpu, 758 u64 __user *uindices) 759 { 760 int n = num_fp_f_regs(vcpu); 761 762 for (int i = 0; i < n; i++) { 763 u64 reg = KVM_REG_RISCV | KVM_REG_SIZE_U32 | 764 KVM_REG_RISCV_FP_F | i; 765 766 if (uindices) { 767 if (put_user(reg, uindices)) 768 return -EFAULT; 769 uindices++; 770 } 771 } 772 773 return n; 774 } 775 776 static inline unsigned long num_fp_d_regs(const struct kvm_vcpu *vcpu) 777 { 778 const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 779 780 if (riscv_isa_extension_available(vcpu->arch.isa, d)) 781 return sizeof(cntx->fp.d.f) / sizeof(u64) + 1; 782 else 783 return 0; 784 } 785 786 static int copy_fp_d_reg_indices(const struct kvm_vcpu *vcpu, 787 u64 __user *uindices) 788 { 789 int i; 790 int n = num_fp_d_regs(vcpu); 791 u64 reg; 792 793 /* copy fp.d.f indices */ 794 for (i = 0; i < n-1; i++) { 795 reg = KVM_REG_RISCV | KVM_REG_SIZE_U64 | 796 KVM_REG_RISCV_FP_D | i; 797 798 if (uindices) { 799 if (put_user(reg, uindices)) 800 return -EFAULT; 801 uindices++; 802 } 803 } 804 805 /* copy fp.d.fcsr indices */ 806 reg = KVM_REG_RISCV | KVM_REG_SIZE_U32 | KVM_REG_RISCV_FP_D | i; 807 if (uindices) { 808 if (put_user(reg, uindices)) 809 return -EFAULT; 810 uindices++; 811 } 812 813 return n; 814 } 815 816 static int copy_isa_ext_reg_indices(const struct kvm_vcpu *vcpu, 817 u64 __user *uindices) 818 { 819 unsigned long guest_ext; 820 unsigned int n = 0; 821 822 for (int i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { 823 u64 size = IS_ENABLED(CONFIG_32BIT) ? 824 KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 825 u64 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_ISA_EXT | i; 826 827 if (__kvm_riscv_isa_check_host(i, &guest_ext)) 828 continue; 829 830 if (uindices) { 831 if (put_user(reg, uindices)) 832 return -EFAULT; 833 uindices++; 834 } 835 836 n++; 837 } 838 839 return n; 840 } 841 842 static inline unsigned long num_isa_ext_regs(const struct kvm_vcpu *vcpu) 843 { 844 return copy_isa_ext_reg_indices(vcpu, NULL); 845 } 846 847 static unsigned long num_sbi_ext_regs(struct kvm_vcpu *vcpu) 848 { 849 return kvm_riscv_vcpu_reg_indices_sbi_ext(vcpu, NULL); 850 } 851 852 static inline unsigned long num_sbi_regs(struct kvm_vcpu *vcpu) 853 { 854 return kvm_riscv_vcpu_reg_indices_sbi(vcpu, NULL); 855 } 856 857 static inline unsigned long num_vector_regs(const struct kvm_vcpu *vcpu) 858 { 859 if (!riscv_isa_extension_available(vcpu->arch.isa, v)) 860 return 0; 861 862 /* vstart, vl, vtype, vcsr, vlenb and 32 vector regs */ 863 return 37; 864 } 865 866 static int copy_vector_reg_indices(const struct kvm_vcpu *vcpu, 867 u64 __user *uindices) 868 { 869 const struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 870 int n = num_vector_regs(vcpu); 871 u64 reg, size; 872 int i; 873 874 if (n == 0) 875 return 0; 876 877 /* copy vstart, vl, vtype, vcsr and vlenb */ 878 size = IS_ENABLED(CONFIG_32BIT) ? KVM_REG_SIZE_U32 : KVM_REG_SIZE_U64; 879 for (i = 0; i < 5; i++) { 880 reg = KVM_REG_RISCV | size | KVM_REG_RISCV_VECTOR | i; 881 882 if (uindices) { 883 if (put_user(reg, uindices)) 884 return -EFAULT; 885 uindices++; 886 } 887 } 888 889 /* vector_regs have a variable 'vlenb' size */ 890 size = __builtin_ctzl(cntx->vector.vlenb); 891 size <<= KVM_REG_SIZE_SHIFT; 892 for (i = 0; i < 32; i++) { 893 reg = KVM_REG_RISCV | KVM_REG_RISCV_VECTOR | size | 894 KVM_REG_RISCV_VECTOR_REG(i); 895 896 if (uindices) { 897 if (put_user(reg, uindices)) 898 return -EFAULT; 899 uindices++; 900 } 901 } 902 903 return n; 904 } 905 906 /* 907 * kvm_riscv_vcpu_num_regs - how many registers do we present via KVM_GET/SET_ONE_REG 908 * 909 * This is for all registers. 910 */ 911 unsigned long kvm_riscv_vcpu_num_regs(struct kvm_vcpu *vcpu) 912 { 913 unsigned long res = 0; 914 915 res += num_config_regs(vcpu); 916 res += num_core_regs(); 917 res += num_csr_regs(vcpu); 918 res += num_timer_regs(); 919 res += num_fp_f_regs(vcpu); 920 res += num_fp_d_regs(vcpu); 921 res += num_vector_regs(vcpu); 922 res += num_isa_ext_regs(vcpu); 923 res += num_sbi_ext_regs(vcpu); 924 res += num_sbi_regs(vcpu); 925 926 return res; 927 } 928 929 /* 930 * kvm_riscv_vcpu_copy_reg_indices - get indices of all registers. 931 */ 932 int kvm_riscv_vcpu_copy_reg_indices(struct kvm_vcpu *vcpu, 933 u64 __user *uindices) 934 { 935 int ret; 936 937 ret = copy_config_reg_indices(vcpu, uindices); 938 if (ret < 0) 939 return ret; 940 uindices += ret; 941 942 ret = copy_core_reg_indices(uindices); 943 if (ret < 0) 944 return ret; 945 uindices += ret; 946 947 ret = copy_csr_reg_indices(vcpu, uindices); 948 if (ret < 0) 949 return ret; 950 uindices += ret; 951 952 ret = copy_timer_reg_indices(uindices); 953 if (ret < 0) 954 return ret; 955 uindices += ret; 956 957 ret = copy_fp_f_reg_indices(vcpu, uindices); 958 if (ret < 0) 959 return ret; 960 uindices += ret; 961 962 ret = copy_fp_d_reg_indices(vcpu, uindices); 963 if (ret < 0) 964 return ret; 965 uindices += ret; 966 967 ret = copy_vector_reg_indices(vcpu, uindices); 968 if (ret < 0) 969 return ret; 970 uindices += ret; 971 972 ret = copy_isa_ext_reg_indices(vcpu, uindices); 973 if (ret < 0) 974 return ret; 975 uindices += ret; 976 977 ret = kvm_riscv_vcpu_reg_indices_sbi_ext(vcpu, uindices); 978 if (ret < 0) 979 return ret; 980 uindices += ret; 981 982 ret = kvm_riscv_vcpu_reg_indices_sbi(vcpu, uindices); 983 if (ret < 0) 984 return ret; 985 uindices += ret; 986 987 return 0; 988 } 989 990 int kvm_riscv_vcpu_set_reg(struct kvm_vcpu *vcpu, 991 const struct kvm_one_reg *reg) 992 { 993 switch (reg->id & KVM_REG_RISCV_TYPE_MASK) { 994 case KVM_REG_RISCV_CONFIG: 995 return kvm_riscv_vcpu_set_reg_config(vcpu, reg); 996 case KVM_REG_RISCV_CORE: 997 return kvm_riscv_vcpu_set_reg_core(vcpu, reg); 998 case KVM_REG_RISCV_CSR: 999 return kvm_riscv_vcpu_set_reg_csr(vcpu, reg); 1000 case KVM_REG_RISCV_TIMER: 1001 return kvm_riscv_vcpu_set_reg_timer(vcpu, reg); 1002 case KVM_REG_RISCV_FP_F: 1003 return kvm_riscv_vcpu_set_reg_fp(vcpu, reg, 1004 KVM_REG_RISCV_FP_F); 1005 case KVM_REG_RISCV_FP_D: 1006 return kvm_riscv_vcpu_set_reg_fp(vcpu, reg, 1007 KVM_REG_RISCV_FP_D); 1008 case KVM_REG_RISCV_VECTOR: 1009 return kvm_riscv_vcpu_set_reg_vector(vcpu, reg); 1010 case KVM_REG_RISCV_ISA_EXT: 1011 return kvm_riscv_vcpu_set_reg_isa_ext(vcpu, reg); 1012 case KVM_REG_RISCV_SBI_EXT: 1013 return kvm_riscv_vcpu_set_reg_sbi_ext(vcpu, reg); 1014 case KVM_REG_RISCV_SBI_STATE: 1015 return kvm_riscv_vcpu_set_reg_sbi(vcpu, reg); 1016 default: 1017 break; 1018 } 1019 1020 return -ENOENT; 1021 } 1022 1023 int kvm_riscv_vcpu_get_reg(struct kvm_vcpu *vcpu, 1024 const struct kvm_one_reg *reg) 1025 { 1026 switch (reg->id & KVM_REG_RISCV_TYPE_MASK) { 1027 case KVM_REG_RISCV_CONFIG: 1028 return kvm_riscv_vcpu_get_reg_config(vcpu, reg); 1029 case KVM_REG_RISCV_CORE: 1030 return kvm_riscv_vcpu_get_reg_core(vcpu, reg); 1031 case KVM_REG_RISCV_CSR: 1032 return kvm_riscv_vcpu_get_reg_csr(vcpu, reg); 1033 case KVM_REG_RISCV_TIMER: 1034 return kvm_riscv_vcpu_get_reg_timer(vcpu, reg); 1035 case KVM_REG_RISCV_FP_F: 1036 return kvm_riscv_vcpu_get_reg_fp(vcpu, reg, 1037 KVM_REG_RISCV_FP_F); 1038 case KVM_REG_RISCV_FP_D: 1039 return kvm_riscv_vcpu_get_reg_fp(vcpu, reg, 1040 KVM_REG_RISCV_FP_D); 1041 case KVM_REG_RISCV_VECTOR: 1042 return kvm_riscv_vcpu_get_reg_vector(vcpu, reg); 1043 case KVM_REG_RISCV_ISA_EXT: 1044 return kvm_riscv_vcpu_get_reg_isa_ext(vcpu, reg); 1045 case KVM_REG_RISCV_SBI_EXT: 1046 return kvm_riscv_vcpu_get_reg_sbi_ext(vcpu, reg); 1047 case KVM_REG_RISCV_SBI_STATE: 1048 return kvm_riscv_vcpu_get_reg_sbi(vcpu, reg); 1049 default: 1050 break; 1051 } 1052 1053 return -ENOENT; 1054 } 1055