1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022 SiFive 4 * 5 * Authors: 6 * Vincent Chen <vincent.chen@sifive.com> 7 * Greentime Hu <greentime.hu@sifive.com> 8 */ 9 10 #include <linux/errno.h> 11 #include <linux/err.h> 12 #include <linux/kvm_host.h> 13 #include <linux/nospec.h> 14 #include <linux/uaccess.h> 15 #include <asm/cpufeature.h> 16 #include <asm/kvm_isa.h> 17 #include <asm/kvm_vcpu_vector.h> 18 #include <asm/vector.h> 19 20 #ifdef CONFIG_RISCV_ISA_V 21 void kvm_riscv_vcpu_vector_reset(struct kvm_vcpu *vcpu) 22 { 23 unsigned long *isa = vcpu->arch.isa; 24 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 25 26 cntx->sstatus &= ~SR_VS; 27 28 cntx->vector.vlenb = riscv_v_vsize / 32; 29 30 if (riscv_isa_extension_available(isa, V)) { 31 cntx->sstatus |= SR_VS_INITIAL; 32 WARN_ON(!cntx->vector.datap); 33 memset(cntx->vector.datap, 0, riscv_v_vsize); 34 } else { 35 cntx->sstatus |= SR_VS_OFF; 36 } 37 } 38 39 static void kvm_riscv_vcpu_vector_clean(struct kvm_cpu_context *cntx) 40 { 41 cntx->sstatus &= ~SR_VS; 42 cntx->sstatus |= SR_VS_CLEAN; 43 } 44 45 void kvm_riscv_vcpu_guest_vector_save(struct kvm_cpu_context *cntx, 46 unsigned long *isa) 47 { 48 if ((cntx->sstatus & SR_VS) == SR_VS_DIRTY) { 49 if (riscv_isa_extension_available(isa, V)) 50 __kvm_riscv_vector_save(cntx); 51 kvm_riscv_vcpu_vector_clean(cntx); 52 } 53 } 54 55 void kvm_riscv_vcpu_guest_vector_restore(struct kvm_cpu_context *cntx, 56 unsigned long *isa) 57 { 58 if ((cntx->sstatus & SR_VS) != SR_VS_OFF) { 59 if (riscv_isa_extension_available(isa, V)) 60 riscv_v_flags_set(riscv_v_flags() | RISCV_V_VCPU_NEED_RESTORE); 61 } 62 } 63 64 void kvm_riscv_vcpu_host_vector_save(struct kvm_cpu_context *cntx) 65 { 66 /* No need to check host sstatus as it can be modified outside */ 67 if (!kvm_riscv_isa_check_host(V)) 68 __kvm_riscv_vector_save(cntx); 69 } 70 71 void kvm_riscv_vcpu_host_vector_restore(struct kvm_cpu_context *cntx) 72 { 73 if (!kvm_riscv_isa_check_host(V)) 74 __kvm_riscv_vector_restore(cntx); 75 riscv_v_flags_set(riscv_v_flags() & ~(RISCV_V_VCPU_CTX | RISCV_V_VCPU_NEED_RESTORE)); 76 } 77 78 int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu) 79 { 80 vcpu->arch.guest_context.vector.datap = kzalloc(riscv_v_vsize, GFP_KERNEL_ACCOUNT); 81 if (!vcpu->arch.guest_context.vector.datap) 82 return -ENOMEM; 83 84 vcpu->arch.host_context.vector.datap = kzalloc(riscv_v_vsize, GFP_KERNEL_ACCOUNT); 85 if (!vcpu->arch.host_context.vector.datap) { 86 kfree(vcpu->arch.guest_context.vector.datap); 87 vcpu->arch.guest_context.vector.datap = NULL; 88 return -ENOMEM; 89 } 90 91 return 0; 92 } 93 94 void kvm_riscv_vcpu_free_vector_context(struct kvm_vcpu *vcpu) 95 { 96 kfree(vcpu->arch.guest_context.vector.datap); 97 kfree(vcpu->arch.host_context.vector.datap); 98 } 99 100 void kvm_riscv_vcpu_flush_vector(void) 101 { 102 struct kvm_vcpu *vcpu = *this_cpu_ptr(kvm_get_running_vcpus()); 103 104 /* 105 * Only reached from __riscv_flush_vector_context() when RISCV_V_VCPU_CTX is set, which 106 * always have kvm_get_running_vcpus non-NULL. 107 */ 108 if (WARN_ON_ONCE(!vcpu)) 109 return; 110 111 kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context, vcpu->arch.isa); 112 113 if ((vcpu->arch.guest_context.sstatus & SR_VS) != SR_VS_OFF) 114 riscv_v_flags_set(riscv_v_flags() | RISCV_V_VCPU_NEED_RESTORE); 115 } 116 117 #endif 118 119 static int kvm_riscv_vcpu_vreg_addr(struct kvm_vcpu *vcpu, 120 unsigned long reg_num, 121 size_t reg_size, 122 void **reg_addr) 123 { 124 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 125 size_t vlenb = riscv_v_vsize / 32; 126 127 if (reg_num < KVM_REG_RISCV_VECTOR_REG(0)) { 128 if (reg_size != sizeof(unsigned long)) 129 return -EINVAL; 130 switch (reg_num) { 131 case KVM_REG_RISCV_VECTOR_CSR_REG(vstart): 132 *reg_addr = &cntx->vector.vstart; 133 break; 134 case KVM_REG_RISCV_VECTOR_CSR_REG(vl): 135 *reg_addr = &cntx->vector.vl; 136 break; 137 case KVM_REG_RISCV_VECTOR_CSR_REG(vtype): 138 *reg_addr = &cntx->vector.vtype; 139 break; 140 case KVM_REG_RISCV_VECTOR_CSR_REG(vcsr): 141 *reg_addr = &cntx->vector.vcsr; 142 break; 143 case KVM_REG_RISCV_VECTOR_CSR_REG(vlenb): 144 *reg_addr = &cntx->vector.vlenb; 145 break; 146 case KVM_REG_RISCV_VECTOR_CSR_REG(datap): 147 default: 148 return -ENOENT; 149 } 150 } else if (reg_num <= KVM_REG_RISCV_VECTOR_REG(31)) { 151 unsigned long reg_offset; 152 153 if (reg_size != vlenb) 154 return -EINVAL; 155 WARN_ON(!cntx->vector.datap); 156 /* 157 * The reg_num is derived from the userspace-provided ONE_REG 158 * id. Sanitize it with array_index_nospec() to prevent 159 * speculative out-of-bounds access to the vector register 160 * buffer (32 vector registers: v0..v31). 161 */ 162 reg_offset = array_index_nospec( 163 reg_num - KVM_REG_RISCV_VECTOR_REG(0), 32); 164 *reg_addr = cntx->vector.datap + reg_offset * vlenb; 165 } else { 166 return -ENOENT; 167 } 168 169 return 0; 170 } 171 172 int kvm_riscv_vcpu_get_reg_vector(struct kvm_vcpu *vcpu, 173 const struct kvm_one_reg *reg) 174 { 175 unsigned long *isa = vcpu->arch.isa; 176 unsigned long __user *uaddr = 177 (unsigned long __user *)(unsigned long)reg->addr; 178 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 179 KVM_REG_SIZE_MASK | 180 KVM_REG_RISCV_VECTOR); 181 size_t reg_size = KVM_REG_SIZE(reg->id); 182 void *reg_addr; 183 int rc; 184 185 if (!riscv_isa_extension_available(isa, V)) 186 return -ENOENT; 187 188 rc = kvm_riscv_vcpu_vreg_addr(vcpu, reg_num, reg_size, ®_addr); 189 if (rc) 190 return rc; 191 192 if (copy_to_user(uaddr, reg_addr, reg_size)) 193 return -EFAULT; 194 195 return 0; 196 } 197 198 int kvm_riscv_vcpu_set_reg_vector(struct kvm_vcpu *vcpu, 199 const struct kvm_one_reg *reg) 200 { 201 unsigned long *isa = vcpu->arch.isa; 202 unsigned long __user *uaddr = 203 (unsigned long __user *)(unsigned long)reg->addr; 204 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 205 KVM_REG_SIZE_MASK | 206 KVM_REG_RISCV_VECTOR); 207 size_t reg_size = KVM_REG_SIZE(reg->id); 208 void *reg_addr; 209 int rc; 210 211 if (!riscv_isa_extension_available(isa, V)) 212 return -ENOENT; 213 214 if (reg_num == KVM_REG_RISCV_VECTOR_CSR_REG(vlenb)) { 215 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 216 unsigned long reg_val; 217 218 if (reg_size != sizeof(reg_val)) 219 return -EINVAL; 220 if (copy_from_user(®_val, uaddr, reg_size)) 221 return -EFAULT; 222 if (reg_val != cntx->vector.vlenb) 223 return -EINVAL; 224 225 return 0; 226 } 227 228 rc = kvm_riscv_vcpu_vreg_addr(vcpu, reg_num, reg_size, ®_addr); 229 if (rc) 230 return rc; 231 232 if (copy_from_user(reg_addr, uaddr, reg_size)) 233 return -EFAULT; 234 235 return 0; 236 } 237