1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2023 SiFive 4 * Author: Andy Chiu <andy.chiu@sifive.com> 5 */ 6 #include <linux/export.h> 7 #include <linux/sched/signal.h> 8 #include <linux/types.h> 9 #include <linux/slab.h> 10 #include <linux/sched.h> 11 #include <linux/uaccess.h> 12 #include <linux/prctl.h> 13 14 #include <asm/thread_info.h> 15 #include <asm/processor.h> 16 #include <asm/insn.h> 17 #include <asm/vector.h> 18 #include <asm/csr.h> 19 #include <asm/elf.h> 20 #include <asm/ptrace.h> 21 #include <asm/bug.h> 22 23 static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE); 24 static struct kmem_cache *riscv_v_user_cachep; 25 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE 26 static struct kmem_cache *riscv_v_kernel_cachep; 27 #endif 28 29 unsigned long riscv_v_vsize __read_mostly; 30 EXPORT_SYMBOL_GPL(riscv_v_vsize); 31 32 int riscv_v_setup_vsize(void) 33 { 34 unsigned long this_vsize; 35 36 /* There are 32 vector registers with vlenb length. */ 37 riscv_v_enable(); 38 this_vsize = csr_read(CSR_VLENB) * 32; 39 riscv_v_disable(); 40 41 if (!riscv_v_vsize) { 42 riscv_v_vsize = this_vsize; 43 return 0; 44 } 45 46 if (riscv_v_vsize != this_vsize) { 47 WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems"); 48 return -EOPNOTSUPP; 49 } 50 51 return 0; 52 } 53 54 void __init riscv_v_setup_ctx_cache(void) 55 { 56 if (!has_vector()) 57 return; 58 59 riscv_v_user_cachep = kmem_cache_create_usercopy("riscv_vector_ctx", 60 riscv_v_vsize, 16, SLAB_PANIC, 61 0, riscv_v_vsize, NULL); 62 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE 63 riscv_v_kernel_cachep = kmem_cache_create("riscv_vector_kctx", 64 riscv_v_vsize, 16, 65 SLAB_PANIC, NULL); 66 #endif 67 } 68 69 static bool insn_is_vector(u32 insn_buf) 70 { 71 u32 opcode = insn_buf & __INSN_OPCODE_MASK; 72 u32 width, csr; 73 74 /* 75 * All V-related instructions, including CSR operations are 4-Byte. So, 76 * do not handle if the instruction length is not 4-Byte. 77 */ 78 if (unlikely(GET_INSN_LENGTH(insn_buf) != 4)) 79 return false; 80 81 switch (opcode) { 82 case RVV_OPCODE_VECTOR: 83 return true; 84 case RVV_OPCODE_VL: 85 case RVV_OPCODE_VS: 86 width = RVV_EXRACT_VL_VS_WIDTH(insn_buf); 87 if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 || 88 width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64) 89 return true; 90 91 break; 92 case RVG_OPCODE_SYSTEM: 93 csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf); 94 if ((csr >= CSR_VSTART && csr <= CSR_VCSR) || 95 (csr >= CSR_VL && csr <= CSR_VLENB)) 96 return true; 97 } 98 99 return false; 100 } 101 102 static int riscv_v_thread_zalloc(struct kmem_cache *cache, 103 struct __riscv_v_ext_state *ctx) 104 { 105 void *datap; 106 107 datap = kmem_cache_zalloc(cache, GFP_KERNEL); 108 if (!datap) 109 return -ENOMEM; 110 111 ctx->datap = datap; 112 memset(ctx, 0, offsetof(struct __riscv_v_ext_state, datap)); 113 return 0; 114 } 115 116 void riscv_v_thread_alloc(struct task_struct *tsk) 117 { 118 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE 119 riscv_v_thread_zalloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate); 120 #endif 121 } 122 123 void riscv_v_thread_free(struct task_struct *tsk) 124 { 125 if (tsk->thread.vstate.datap) 126 kmem_cache_free(riscv_v_user_cachep, tsk->thread.vstate.datap); 127 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE 128 if (tsk->thread.kernel_vstate.datap) 129 kmem_cache_free(riscv_v_kernel_cachep, tsk->thread.kernel_vstate.datap); 130 #endif 131 } 132 133 #define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK) 134 #define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2) 135 #define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) 136 #define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT)) 137 static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk) 138 { 139 return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl); 140 } 141 142 static inline int riscv_v_ctrl_get_next(struct task_struct *tsk) 143 { 144 return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl); 145 } 146 147 static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk) 148 { 149 return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl); 150 } 151 152 static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt, 153 bool inherit) 154 { 155 unsigned long ctrl; 156 157 ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK; 158 ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt); 159 if (inherit) 160 ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT; 161 tsk->thread.vstate_ctrl &= ~PR_RISCV_V_VSTATE_CTRL_MASK; 162 tsk->thread.vstate_ctrl |= ctrl; 163 } 164 165 bool riscv_v_vstate_ctrl_user_allowed(void) 166 { 167 return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON; 168 } 169 EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed); 170 171 bool riscv_v_first_use_handler(struct pt_regs *regs) 172 { 173 u32 __user *epc = (u32 __user *)regs->epc; 174 u32 insn = (u32)regs->badaddr; 175 176 if (!has_vector()) 177 return false; 178 179 /* Do not handle if V is not supported, or disabled */ 180 if (!riscv_v_vstate_ctrl_user_allowed()) 181 return false; 182 183 /* If V has been enabled then it is not the first-use trap */ 184 if (riscv_v_vstate_query(regs)) 185 return false; 186 187 /* Get the instruction */ 188 if (!insn) { 189 if (__get_user(insn, epc)) 190 return false; 191 } 192 193 /* Filter out non-V instructions */ 194 if (!insn_is_vector(insn)) 195 return false; 196 197 /* Sanity check. datap should be null by the time of the first-use trap */ 198 WARN_ON(current->thread.vstate.datap); 199 200 /* 201 * Now we sure that this is a V instruction. And it executes in the 202 * context where VS has been off. So, try to allocate the user's V 203 * context and resume execution. 204 */ 205 if (riscv_v_thread_zalloc(riscv_v_user_cachep, ¤t->thread.vstate)) { 206 force_sig(SIGBUS); 207 return true; 208 } 209 riscv_v_vstate_on(regs); 210 riscv_v_vstate_set_restore(current, regs); 211 return true; 212 } 213 214 void riscv_v_vstate_ctrl_init(struct task_struct *tsk) 215 { 216 bool inherit; 217 int cur, next; 218 219 if (!has_vector()) 220 return; 221 222 next = riscv_v_ctrl_get_next(tsk); 223 if (!next) { 224 if (READ_ONCE(riscv_v_implicit_uacc)) 225 cur = PR_RISCV_V_VSTATE_CTRL_ON; 226 else 227 cur = PR_RISCV_V_VSTATE_CTRL_OFF; 228 } else { 229 cur = next; 230 } 231 /* Clear next mask if inherit-bit is not set */ 232 inherit = riscv_v_ctrl_test_inherit(tsk); 233 if (!inherit) 234 next = PR_RISCV_V_VSTATE_CTRL_DEFAULT; 235 236 riscv_v_ctrl_set(tsk, cur, next, inherit); 237 } 238 239 long riscv_v_vstate_ctrl_get_current(void) 240 { 241 if (!has_vector()) 242 return -EINVAL; 243 244 return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK; 245 } 246 247 long riscv_v_vstate_ctrl_set_current(unsigned long arg) 248 { 249 bool inherit; 250 int cur, next; 251 252 if (!has_vector()) 253 return -EINVAL; 254 255 if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK) 256 return -EINVAL; 257 258 cur = VSTATE_CTRL_GET_CUR(arg); 259 switch (cur) { 260 case PR_RISCV_V_VSTATE_CTRL_OFF: 261 /* Do not allow user to turn off V if current is not off */ 262 if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF) 263 return -EPERM; 264 265 break; 266 case PR_RISCV_V_VSTATE_CTRL_ON: 267 break; 268 case PR_RISCV_V_VSTATE_CTRL_DEFAULT: 269 cur = riscv_v_ctrl_get_cur(current); 270 break; 271 default: 272 return -EINVAL; 273 } 274 275 next = VSTATE_CTRL_GET_NEXT(arg); 276 inherit = VSTATE_CTRL_GET_INHERIT(arg); 277 switch (next) { 278 case PR_RISCV_V_VSTATE_CTRL_DEFAULT: 279 case PR_RISCV_V_VSTATE_CTRL_OFF: 280 case PR_RISCV_V_VSTATE_CTRL_ON: 281 riscv_v_ctrl_set(current, cur, next, inherit); 282 return 0; 283 } 284 285 return -EINVAL; 286 } 287 288 #ifdef CONFIG_SYSCTL 289 290 static struct ctl_table riscv_v_default_vstate_table[] = { 291 { 292 .procname = "riscv_v_default_allow", 293 .data = &riscv_v_implicit_uacc, 294 .maxlen = sizeof(riscv_v_implicit_uacc), 295 .mode = 0644, 296 .proc_handler = proc_dobool, 297 }, 298 }; 299 300 static int __init riscv_v_sysctl_init(void) 301 { 302 if (has_vector()) 303 if (!register_sysctl("abi", riscv_v_default_vstate_table)) 304 return -EINVAL; 305 return 0; 306 } 307 308 #else /* ! CONFIG_SYSCTL */ 309 static int __init riscv_v_sysctl_init(void) { return 0; } 310 #endif /* ! CONFIG_SYSCTL */ 311 312 static int riscv_v_init(void) 313 { 314 return riscv_v_sysctl_init(); 315 } 316 core_initcall(riscv_v_init); 317