1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Copyright (C) 2020 SiFive 4 */ 5 6 #ifndef __ASM_RISCV_VECTOR_H 7 #define __ASM_RISCV_VECTOR_H 8 9 #include <linux/types.h> 10 #include <uapi/asm-generic/errno.h> 11 12 #ifdef CONFIG_RISCV_ISA_V 13 14 #include <linux/stringify.h> 15 #include <linux/sched.h> 16 #include <linux/sched/task_stack.h> 17 #include <asm/ptrace.h> 18 #include <asm/cpufeature.h> 19 #include <asm/csr.h> 20 #include <asm/asm.h> 21 22 extern unsigned long riscv_v_vsize; 23 int riscv_v_setup_vsize(void); 24 bool riscv_v_first_use_handler(struct pt_regs *regs); 25 26 static __always_inline bool has_vector(void) 27 { 28 return riscv_has_extension_unlikely(RISCV_ISA_EXT_v); 29 } 30 31 static inline void __riscv_v_vstate_clean(struct pt_regs *regs) 32 { 33 regs->status = (regs->status & ~SR_VS) | SR_VS_CLEAN; 34 } 35 36 static inline void __riscv_v_vstate_dirty(struct pt_regs *regs) 37 { 38 regs->status = (regs->status & ~SR_VS) | SR_VS_DIRTY; 39 } 40 41 static inline void riscv_v_vstate_off(struct pt_regs *regs) 42 { 43 regs->status = (regs->status & ~SR_VS) | SR_VS_OFF; 44 } 45 46 static inline void riscv_v_vstate_on(struct pt_regs *regs) 47 { 48 regs->status = (regs->status & ~SR_VS) | SR_VS_INITIAL; 49 } 50 51 static inline bool riscv_v_vstate_query(struct pt_regs *regs) 52 { 53 return (regs->status & SR_VS) != 0; 54 } 55 56 static __always_inline void riscv_v_enable(void) 57 { 58 csr_set(CSR_SSTATUS, SR_VS); 59 } 60 61 static __always_inline void riscv_v_disable(void) 62 { 63 csr_clear(CSR_SSTATUS, SR_VS); 64 } 65 66 static __always_inline void __vstate_csr_save(struct __riscv_v_ext_state *dest) 67 { 68 asm volatile ( 69 "csrr %0, " __stringify(CSR_VSTART) "\n\t" 70 "csrr %1, " __stringify(CSR_VTYPE) "\n\t" 71 "csrr %2, " __stringify(CSR_VL) "\n\t" 72 "csrr %3, " __stringify(CSR_VCSR) "\n\t" 73 "csrr %4, " __stringify(CSR_VLENB) "\n\t" 74 : "=r" (dest->vstart), "=r" (dest->vtype), "=r" (dest->vl), 75 "=r" (dest->vcsr), "=r" (dest->vlenb) : :); 76 } 77 78 static __always_inline void __vstate_csr_restore(struct __riscv_v_ext_state *src) 79 { 80 asm volatile ( 81 ".option push\n\t" 82 ".option arch, +v\n\t" 83 "vsetvl x0, %2, %1\n\t" 84 ".option pop\n\t" 85 "csrw " __stringify(CSR_VSTART) ", %0\n\t" 86 "csrw " __stringify(CSR_VCSR) ", %3\n\t" 87 : : "r" (src->vstart), "r" (src->vtype), "r" (src->vl), 88 "r" (src->vcsr) :); 89 } 90 91 static inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to, 92 void *datap) 93 { 94 unsigned long vl; 95 96 riscv_v_enable(); 97 __vstate_csr_save(save_to); 98 asm volatile ( 99 ".option push\n\t" 100 ".option arch, +v\n\t" 101 "vsetvli %0, x0, e8, m8, ta, ma\n\t" 102 "vse8.v v0, (%1)\n\t" 103 "add %1, %1, %0\n\t" 104 "vse8.v v8, (%1)\n\t" 105 "add %1, %1, %0\n\t" 106 "vse8.v v16, (%1)\n\t" 107 "add %1, %1, %0\n\t" 108 "vse8.v v24, (%1)\n\t" 109 ".option pop\n\t" 110 : "=&r" (vl) : "r" (datap) : "memory"); 111 riscv_v_disable(); 112 } 113 114 static inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_from, 115 void *datap) 116 { 117 unsigned long vl; 118 119 riscv_v_enable(); 120 asm volatile ( 121 ".option push\n\t" 122 ".option arch, +v\n\t" 123 "vsetvli %0, x0, e8, m8, ta, ma\n\t" 124 "vle8.v v0, (%1)\n\t" 125 "add %1, %1, %0\n\t" 126 "vle8.v v8, (%1)\n\t" 127 "add %1, %1, %0\n\t" 128 "vle8.v v16, (%1)\n\t" 129 "add %1, %1, %0\n\t" 130 "vle8.v v24, (%1)\n\t" 131 ".option pop\n\t" 132 : "=&r" (vl) : "r" (datap) : "memory"); 133 __vstate_csr_restore(restore_from); 134 riscv_v_disable(); 135 } 136 137 static inline void __riscv_v_vstate_discard(void) 138 { 139 unsigned long vl, vtype_inval = 1UL << (BITS_PER_LONG - 1); 140 141 riscv_v_enable(); 142 asm volatile ( 143 ".option push\n\t" 144 ".option arch, +v\n\t" 145 "vsetvli %0, x0, e8, m8, ta, ma\n\t" 146 "vmv.v.i v0, -1\n\t" 147 "vmv.v.i v8, -1\n\t" 148 "vmv.v.i v16, -1\n\t" 149 "vmv.v.i v24, -1\n\t" 150 "vsetvl %0, x0, %1\n\t" 151 ".option pop\n\t" 152 : "=&r" (vl) : "r" (vtype_inval) : "memory"); 153 riscv_v_disable(); 154 } 155 156 static inline void riscv_v_vstate_discard(struct pt_regs *regs) 157 { 158 if ((regs->status & SR_VS) == SR_VS_OFF) 159 return; 160 161 __riscv_v_vstate_discard(); 162 __riscv_v_vstate_dirty(regs); 163 } 164 165 static inline void riscv_v_vstate_save(struct task_struct *task, 166 struct pt_regs *regs) 167 { 168 if ((regs->status & SR_VS) == SR_VS_DIRTY) { 169 struct __riscv_v_ext_state *vstate = &task->thread.vstate; 170 171 __riscv_v_vstate_save(vstate, vstate->datap); 172 __riscv_v_vstate_clean(regs); 173 } 174 } 175 176 static inline void riscv_v_vstate_restore(struct task_struct *task, 177 struct pt_regs *regs) 178 { 179 if ((regs->status & SR_VS) != SR_VS_OFF) { 180 struct __riscv_v_ext_state *vstate = &task->thread.vstate; 181 182 __riscv_v_vstate_restore(vstate, vstate->datap); 183 __riscv_v_vstate_clean(regs); 184 } 185 } 186 187 static inline void __switch_to_vector(struct task_struct *prev, 188 struct task_struct *next) 189 { 190 struct pt_regs *regs; 191 192 regs = task_pt_regs(prev); 193 riscv_v_vstate_save(prev, regs); 194 riscv_v_vstate_restore(next, task_pt_regs(next)); 195 } 196 197 void riscv_v_vstate_ctrl_init(struct task_struct *tsk); 198 bool riscv_v_vstate_ctrl_user_allowed(void); 199 200 #else /* ! CONFIG_RISCV_ISA_V */ 201 202 struct pt_regs; 203 204 static inline int riscv_v_setup_vsize(void) { return -EOPNOTSUPP; } 205 static __always_inline bool has_vector(void) { return false; } 206 static inline bool riscv_v_first_use_handler(struct pt_regs *regs) { return false; } 207 static inline bool riscv_v_vstate_query(struct pt_regs *regs) { return false; } 208 static inline bool riscv_v_vstate_ctrl_user_allowed(void) { return false; } 209 #define riscv_v_vsize (0) 210 #define riscv_v_vstate_discard(regs) do {} while (0) 211 #define riscv_v_vstate_save(task, regs) do {} while (0) 212 #define riscv_v_vstate_restore(task, regs) do {} while (0) 213 #define __switch_to_vector(__prev, __next) do {} while (0) 214 #define riscv_v_vstate_off(regs) do {} while (0) 215 #define riscv_v_vstate_on(regs) do {} while (0) 216 217 #endif /* CONFIG_RISCV_ISA_V */ 218 219 #endif /* ! __ASM_RISCV_VECTOR_H */ 220