1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022 SiFive 4 * 5 * Authors: 6 * Vincent Chen <vincent.chen@sifive.com> 7 * Greentime Hu <greentime.hu@sifive.com> 8 */ 9 10 #include <linux/errno.h> 11 #include <linux/err.h> 12 #include <linux/kvm_host.h> 13 #include <linux/nospec.h> 14 #include <linux/uaccess.h> 15 #include <asm/cpufeature.h> 16 #include <asm/kvm_isa.h> 17 #include <asm/kvm_vcpu_vector.h> 18 #include <asm/vector.h> 19 20 #ifdef CONFIG_RISCV_ISA_V 21 void kvm_riscv_vcpu_vector_reset(struct kvm_vcpu *vcpu) 22 { 23 unsigned long *isa = vcpu->arch.isa; 24 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 25 26 cntx->sstatus &= ~SR_VS; 27 28 cntx->vector.vlenb = riscv_v_vsize / 32; 29 30 if (riscv_isa_extension_available(isa, v)) { 31 cntx->sstatus |= SR_VS_INITIAL; 32 WARN_ON(!cntx->vector.datap); 33 memset(cntx->vector.datap, 0, riscv_v_vsize); 34 } else { 35 cntx->sstatus |= SR_VS_OFF; 36 } 37 } 38 39 static void kvm_riscv_vcpu_vector_clean(struct kvm_cpu_context *cntx) 40 { 41 cntx->sstatus &= ~SR_VS; 42 cntx->sstatus |= SR_VS_CLEAN; 43 } 44 45 void kvm_riscv_vcpu_guest_vector_save(struct kvm_cpu_context *cntx, 46 unsigned long *isa) 47 { 48 if ((cntx->sstatus & SR_VS) == SR_VS_DIRTY) { 49 if (riscv_isa_extension_available(isa, v)) 50 __kvm_riscv_vector_save(cntx); 51 kvm_riscv_vcpu_vector_clean(cntx); 52 } 53 } 54 55 void kvm_riscv_vcpu_guest_vector_restore(struct kvm_cpu_context *cntx, 56 unsigned long *isa) 57 { 58 if ((cntx->sstatus & SR_VS) != SR_VS_OFF) { 59 if (riscv_isa_extension_available(isa, v)) 60 __kvm_riscv_vector_restore(cntx); 61 kvm_riscv_vcpu_vector_clean(cntx); 62 } 63 } 64 65 void kvm_riscv_vcpu_host_vector_save(struct kvm_cpu_context *cntx) 66 { 67 /* No need to check host sstatus as it can be modified outside */ 68 if (!kvm_riscv_isa_check_host(V)) 69 __kvm_riscv_vector_save(cntx); 70 } 71 72 void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx) 73 { 74 if (!kvm_riscv_isa_check_host(V)) 75 __kvm_riscv_vector_restore(cntx); 76 } 77 78 int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu) 79 { 80 vcpu->arch.guest_context.vector.datap = kzalloc(riscv_v_vsize, GFP_KERNEL); 81 if (!vcpu->arch.guest_context.vector.datap) 82 return -ENOMEM; 83 84 vcpu->arch.host_context.vector.datap = kzalloc(riscv_v_vsize, GFP_KERNEL); 85 if (!vcpu->arch.host_context.vector.datap) { 86 kfree(vcpu->arch.guest_context.vector.datap); 87 vcpu->arch.guest_context.vector.datap = NULL; 88 return -ENOMEM; 89 } 90 91 return 0; 92 } 93 94 void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu) 95 { 96 kfree(vcpu->arch.guest_context.vector.datap); 97 kfree(vcpu->arch.host_context.vector.datap); 98 } 99 #endif 100 101 static int kvm_riscv_vcpu_vreg_addr(struct kvm_vcpu *vcpu, 102 unsigned long reg_num, 103 size_t reg_size, 104 void **reg_addr) 105 { 106 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 107 size_t vlenb = riscv_v_vsize / 32; 108 109 if (reg_num < KVM_REG_RISCV_VECTOR_REG(0)) { 110 if (reg_size != sizeof(unsigned long)) 111 return -EINVAL; 112 switch (reg_num) { 113 case KVM_REG_RISCV_VECTOR_CSR_REG(vstart): 114 *reg_addr = &cntx->vector.vstart; 115 break; 116 case KVM_REG_RISCV_VECTOR_CSR_REG(vl): 117 *reg_addr = &cntx->vector.vl; 118 break; 119 case KVM_REG_RISCV_VECTOR_CSR_REG(vtype): 120 *reg_addr = &cntx->vector.vtype; 121 break; 122 case KVM_REG_RISCV_VECTOR_CSR_REG(vcsr): 123 *reg_addr = &cntx->vector.vcsr; 124 break; 125 case KVM_REG_RISCV_VECTOR_CSR_REG(vlenb): 126 *reg_addr = &cntx->vector.vlenb; 127 break; 128 case KVM_REG_RISCV_VECTOR_CSR_REG(datap): 129 default: 130 return -ENOENT; 131 } 132 } else if (reg_num <= KVM_REG_RISCV_VECTOR_REG(31)) { 133 unsigned long reg_offset; 134 135 if (reg_size != vlenb) 136 return -EINVAL; 137 WARN_ON(!cntx->vector.datap); 138 /* 139 * The reg_num is derived from the userspace-provided ONE_REG 140 * id. Sanitize it with array_index_nospec() to prevent 141 * speculative out-of-bounds access to the vector register 142 * buffer (32 vector registers: v0..v31). 143 */ 144 reg_offset = array_index_nospec( 145 reg_num - KVM_REG_RISCV_VECTOR_REG(0), 32); 146 *reg_addr = cntx->vector.datap + reg_offset * vlenb; 147 } else { 148 return -ENOENT; 149 } 150 151 return 0; 152 } 153 154 int kvm_riscv_vcpu_get_reg_vector(struct kvm_vcpu *vcpu, 155 const struct kvm_one_reg *reg) 156 { 157 unsigned long *isa = vcpu->arch.isa; 158 unsigned long __user *uaddr = 159 (unsigned long __user *)(unsigned long)reg->addr; 160 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 161 KVM_REG_SIZE_MASK | 162 KVM_REG_RISCV_VECTOR); 163 size_t reg_size = KVM_REG_SIZE(reg->id); 164 void *reg_addr; 165 int rc; 166 167 if (!riscv_isa_extension_available(isa, v)) 168 return -ENOENT; 169 170 rc = kvm_riscv_vcpu_vreg_addr(vcpu, reg_num, reg_size, ®_addr); 171 if (rc) 172 return rc; 173 174 if (copy_to_user(uaddr, reg_addr, reg_size)) 175 return -EFAULT; 176 177 return 0; 178 } 179 180 int kvm_riscv_vcpu_set_reg_vector(struct kvm_vcpu *vcpu, 181 const struct kvm_one_reg *reg) 182 { 183 unsigned long *isa = vcpu->arch.isa; 184 unsigned long __user *uaddr = 185 (unsigned long __user *)(unsigned long)reg->addr; 186 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 187 KVM_REG_SIZE_MASK | 188 KVM_REG_RISCV_VECTOR); 189 size_t reg_size = KVM_REG_SIZE(reg->id); 190 void *reg_addr; 191 int rc; 192 193 if (!riscv_isa_extension_available(isa, v)) 194 return -ENOENT; 195 196 if (reg_num == KVM_REG_RISCV_VECTOR_CSR_REG(vlenb)) { 197 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 198 unsigned long reg_val; 199 200 if (reg_size != sizeof(reg_val)) 201 return -EINVAL; 202 if (copy_from_user(®_val, uaddr, reg_size)) 203 return -EFAULT; 204 if (reg_val != cntx->vector.vlenb) 205 return -EINVAL; 206 207 return 0; 208 } 209 210 rc = kvm_riscv_vcpu_vreg_addr(vcpu, reg_num, reg_size, ®_addr); 211 if (rc) 212 return rc; 213 214 if (copy_from_user(reg_addr, uaddr, reg_size)) 215 return -EFAULT; 216 217 return 0; 218 } 219