1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2023 SiFive 4 * Author: Andy Chiu <andy.chiu@sifive.com> 5 */ 6 #include <linux/export.h> 7 #include <linux/sched/signal.h> 8 #include <linux/types.h> 9 #include <linux/slab.h> 10 #include <linux/sched.h> 11 #include <linux/uaccess.h> 12 #include <linux/prctl.h> 13 14 #include <asm/thread_info.h> 15 #include <asm/processor.h> 16 #include <asm/insn.h> 17 #include <asm/vector.h> 18 #include <asm/csr.h> 19 #include <asm/elf.h> 20 #include <asm/ptrace.h> 21 #include <asm/bug.h> 22 23 static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE); 24 static struct kmem_cache *riscv_v_user_cachep; 25 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE 26 static struct kmem_cache *riscv_v_kernel_cachep; 27 #endif 28 29 unsigned long riscv_v_vsize __read_mostly; 30 EXPORT_SYMBOL_GPL(riscv_v_vsize); 31 32 int riscv_v_setup_vsize(void) 33 { 34 unsigned long this_vsize; 35 36 /* 37 * There are 32 vector registers with vlenb length. 38 * 39 * If the thead,vlenb property was provided by the firmware, use that 40 * instead of probing the CSRs. 41 */ 42 if (thead_vlenb_of) { 43 riscv_v_vsize = thead_vlenb_of * 32; 44 return 0; 45 } 46 47 riscv_v_enable(); 48 this_vsize = csr_read(CSR_VLENB) * 32; 49 riscv_v_disable(); 50 51 if (!riscv_v_vsize) { 52 riscv_v_vsize = this_vsize; 53 return 0; 54 } 55 56 if (riscv_v_vsize != this_vsize) { 57 WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems"); 58 return -EOPNOTSUPP; 59 } 60 61 return 0; 62 } 63 64 void __init riscv_v_setup_ctx_cache(void) 65 { 66 if (!(has_vector() || has_xtheadvector())) 67 return; 68 69 update_regset_vector_info(riscv_v_vsize); 70 71 riscv_v_user_cachep = kmem_cache_create_usercopy("riscv_vector_ctx", 72 riscv_v_vsize, 16, SLAB_PANIC, 73 0, riscv_v_vsize, NULL); 74 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE 75 riscv_v_kernel_cachep = kmem_cache_create("riscv_vector_kctx", 76 riscv_v_vsize, 16, 77 SLAB_PANIC, NULL); 78 #endif 79 } 80 81 bool insn_is_vector(u32 insn_buf) 82 { 83 u32 opcode = insn_buf & __INSN_OPCODE_MASK; 84 u32 width, csr; 85 86 /* 87 * All V-related instructions, including CSR operations are 4-Byte. So, 88 * do not handle if the instruction length is not 4-Byte. 89 */ 90 if (unlikely(GET_INSN_LENGTH(insn_buf) != 4)) 91 return false; 92 93 switch (opcode) { 94 case RVV_OPCODE_VECTOR: 95 return true; 96 case RVV_OPCODE_VL: 97 case RVV_OPCODE_VS: 98 width = RVV_EXTRACT_VL_VS_WIDTH(insn_buf); 99 if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 || 100 width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64) 101 return true; 102 103 break; 104 case RVG_OPCODE_SYSTEM: 105 csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf); 106 if ((csr >= CSR_VSTART && csr <= CSR_VCSR) || 107 (csr >= CSR_VL && csr <= CSR_VLENB)) 108 return true; 109 } 110 111 return false; 112 } 113 114 static int riscv_v_thread_ctx_alloc(struct kmem_cache *cache, 115 struct __riscv_v_ext_state *ctx) 116 { 117 void *datap; 118 119 datap = kmem_cache_zalloc(cache, GFP_KERNEL); 120 if (!datap) 121 return -ENOMEM; 122 123 ctx->datap = datap; 124 memset(ctx, 0, offsetof(struct __riscv_v_ext_state, datap)); 125 ctx->vlenb = riscv_v_vsize / 32; 126 127 return 0; 128 } 129 130 void riscv_v_thread_alloc(struct task_struct *tsk) 131 { 132 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE 133 riscv_v_thread_ctx_alloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate); 134 #endif 135 } 136 137 void riscv_v_thread_free(struct task_struct *tsk) 138 { 139 if (tsk->thread.vstate.datap) 140 kmem_cache_free(riscv_v_user_cachep, tsk->thread.vstate.datap); 141 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE 142 if (tsk->thread.kernel_vstate.datap) 143 kmem_cache_free(riscv_v_kernel_cachep, tsk->thread.kernel_vstate.datap); 144 #endif 145 } 146 147 #define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK) 148 #define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2) 149 #define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) 150 #define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT)) 151 static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk) 152 { 153 return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl); 154 } 155 156 static inline int riscv_v_ctrl_get_next(struct task_struct *tsk) 157 { 158 return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl); 159 } 160 161 static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk) 162 { 163 return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl); 164 } 165 166 static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt, 167 bool inherit) 168 { 169 unsigned long ctrl; 170 171 ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK; 172 ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt); 173 if (inherit) 174 ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT; 175 tsk->thread.vstate_ctrl &= ~PR_RISCV_V_VSTATE_CTRL_MASK; 176 tsk->thread.vstate_ctrl |= ctrl; 177 } 178 179 bool riscv_v_vstate_ctrl_user_allowed(void) 180 { 181 return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON; 182 } 183 EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed); 184 185 bool riscv_v_first_use_handler(struct pt_regs *regs) 186 { 187 u32 __user *epc = (u32 __user *)regs->epc; 188 u32 insn = (u32)regs->badaddr; 189 190 if (!(has_vector() || has_xtheadvector())) 191 return false; 192 193 /* Do not handle if V is not supported, or disabled */ 194 if (!riscv_v_vstate_ctrl_user_allowed()) 195 return false; 196 197 /* If V has been enabled then it is not the first-use trap */ 198 if (riscv_v_vstate_query(regs)) 199 return false; 200 201 /* Get the instruction */ 202 if (!insn) { 203 if (__get_user(insn, epc)) 204 return false; 205 } 206 207 /* Filter out non-V instructions */ 208 if (!insn_is_vector(insn)) 209 return false; 210 211 /* Sanity check. datap should be null by the time of the first-use trap */ 212 WARN_ON(current->thread.vstate.datap); 213 214 /* 215 * Now we sure that this is a V instruction. And it executes in the 216 * context where VS has been off. So, try to allocate the user's V 217 * context and resume execution. 218 */ 219 if (riscv_v_thread_ctx_alloc(riscv_v_user_cachep, ¤t->thread.vstate)) { 220 force_sig(SIGBUS); 221 return true; 222 } 223 224 riscv_v_vstate_on(regs); 225 riscv_v_vstate_set_restore(current, regs); 226 227 return true; 228 } 229 230 void riscv_v_vstate_ctrl_init(struct task_struct *tsk) 231 { 232 bool inherit; 233 int cur, next; 234 235 if (!(has_vector() || has_xtheadvector())) 236 return; 237 238 next = riscv_v_ctrl_get_next(tsk); 239 if (!next) { 240 if (READ_ONCE(riscv_v_implicit_uacc)) 241 cur = PR_RISCV_V_VSTATE_CTRL_ON; 242 else 243 cur = PR_RISCV_V_VSTATE_CTRL_OFF; 244 } else { 245 cur = next; 246 } 247 /* Clear next mask if inherit-bit is not set */ 248 inherit = riscv_v_ctrl_test_inherit(tsk); 249 if (!inherit) 250 next = PR_RISCV_V_VSTATE_CTRL_DEFAULT; 251 252 riscv_v_ctrl_set(tsk, cur, next, inherit); 253 } 254 255 long riscv_v_vstate_ctrl_get_current(void) 256 { 257 if (!(has_vector() || has_xtheadvector())) 258 return -EINVAL; 259 260 return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK; 261 } 262 263 long riscv_v_vstate_ctrl_set_current(unsigned long arg) 264 { 265 bool inherit; 266 int cur, next; 267 268 if (!(has_vector() || has_xtheadvector())) 269 return -EINVAL; 270 271 if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK) 272 return -EINVAL; 273 274 cur = VSTATE_CTRL_GET_CUR(arg); 275 switch (cur) { 276 case PR_RISCV_V_VSTATE_CTRL_OFF: 277 /* Do not allow user to turn off V if current is not off */ 278 if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF) 279 return -EPERM; 280 281 break; 282 case PR_RISCV_V_VSTATE_CTRL_ON: 283 break; 284 case PR_RISCV_V_VSTATE_CTRL_DEFAULT: 285 cur = riscv_v_ctrl_get_cur(current); 286 break; 287 default: 288 return -EINVAL; 289 } 290 291 next = VSTATE_CTRL_GET_NEXT(arg); 292 inherit = VSTATE_CTRL_GET_INHERIT(arg); 293 switch (next) { 294 case PR_RISCV_V_VSTATE_CTRL_DEFAULT: 295 case PR_RISCV_V_VSTATE_CTRL_OFF: 296 case PR_RISCV_V_VSTATE_CTRL_ON: 297 riscv_v_ctrl_set(current, cur, next, inherit); 298 return 0; 299 } 300 301 return -EINVAL; 302 } 303 304 #ifdef CONFIG_SYSCTL 305 306 static const struct ctl_table riscv_v_default_vstate_table[] = { 307 { 308 .procname = "riscv_v_default_allow", 309 .data = &riscv_v_implicit_uacc, 310 .maxlen = sizeof(riscv_v_implicit_uacc), 311 .mode = 0644, 312 .proc_handler = proc_dobool, 313 }, 314 }; 315 316 static int __init riscv_v_sysctl_init(void) 317 { 318 if (has_vector() || has_xtheadvector()) 319 if (!register_sysctl("abi", riscv_v_default_vstate_table)) 320 return -EINVAL; 321 return 0; 322 } 323 324 #else /* ! CONFIG_SYSCTL */ 325 static int __init riscv_v_sysctl_init(void) { return 0; } 326 #endif /* ! CONFIG_SYSCTL */ 327 328 static int __init riscv_v_init(void) 329 { 330 return riscv_v_sysctl_init(); 331 } 332 core_initcall(riscv_v_init); 333