1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Copyright (C) 2020 SiFive 4 */ 5 6 #ifndef __ASM_RISCV_VECTOR_H 7 #define __ASM_RISCV_VECTOR_H 8 9 #include <linux/types.h> 10 #include <uapi/asm-generic/errno.h> 11 12 #ifdef CONFIG_RISCV_ISA_V 13 14 #include <linux/stringify.h> 15 #include <linux/sched.h> 16 #include <linux/sched/task_stack.h> 17 #include <asm/ptrace.h> 18 #include <asm/hwcap.h> 19 #include <asm/csr.h> 20 #include <asm/asm.h> 21 22 extern unsigned long riscv_v_vsize; 23 int riscv_v_setup_vsize(void); 24 bool riscv_v_first_use_handler(struct pt_regs *regs); 25 26 static __always_inline bool has_vector(void) 27 { 28 return riscv_has_extension_unlikely(RISCV_ISA_EXT_v); 29 } 30 31 static inline void __riscv_v_vstate_clean(struct pt_regs *regs) 32 { 33 regs->status = (regs->status & ~SR_VS) | SR_VS_CLEAN; 34 } 35 36 static inline void riscv_v_vstate_off(struct pt_regs *regs) 37 { 38 regs->status = (regs->status & ~SR_VS) | SR_VS_OFF; 39 } 40 41 static inline void riscv_v_vstate_on(struct pt_regs *regs) 42 { 43 regs->status = (regs->status & ~SR_VS) | SR_VS_INITIAL; 44 } 45 46 static inline bool riscv_v_vstate_query(struct pt_regs *regs) 47 { 48 return (regs->status & SR_VS) != 0; 49 } 50 51 static __always_inline void riscv_v_enable(void) 52 { 53 csr_set(CSR_SSTATUS, SR_VS); 54 } 55 56 static __always_inline void riscv_v_disable(void) 57 { 58 csr_clear(CSR_SSTATUS, SR_VS); 59 } 60 61 static __always_inline void __vstate_csr_save(struct __riscv_v_ext_state *dest) 62 { 63 asm volatile ( 64 "csrr %0, " __stringify(CSR_VSTART) "\n\t" 65 "csrr %1, " __stringify(CSR_VTYPE) "\n\t" 66 "csrr %2, " __stringify(CSR_VL) "\n\t" 67 "csrr %3, " __stringify(CSR_VCSR) "\n\t" 68 : "=r" (dest->vstart), "=r" (dest->vtype), "=r" (dest->vl), 69 "=r" (dest->vcsr) : :); 70 } 71 72 static __always_inline void __vstate_csr_restore(struct __riscv_v_ext_state *src) 73 { 74 asm volatile ( 75 ".option push\n\t" 76 ".option arch, +v\n\t" 77 "vsetvl x0, %2, %1\n\t" 78 ".option pop\n\t" 79 "csrw " __stringify(CSR_VSTART) ", %0\n\t" 80 "csrw " __stringify(CSR_VCSR) ", %3\n\t" 81 : : "r" (src->vstart), "r" (src->vtype), "r" (src->vl), 82 "r" (src->vcsr) :); 83 } 84 85 static inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to, 86 void *datap) 87 { 88 unsigned long vl; 89 90 riscv_v_enable(); 91 __vstate_csr_save(save_to); 92 asm volatile ( 93 ".option push\n\t" 94 ".option arch, +v\n\t" 95 "vsetvli %0, x0, e8, m8, ta, ma\n\t" 96 "vse8.v v0, (%1)\n\t" 97 "add %1, %1, %0\n\t" 98 "vse8.v v8, (%1)\n\t" 99 "add %1, %1, %0\n\t" 100 "vse8.v v16, (%1)\n\t" 101 "add %1, %1, %0\n\t" 102 "vse8.v v24, (%1)\n\t" 103 ".option pop\n\t" 104 : "=&r" (vl) : "r" (datap) : "memory"); 105 riscv_v_disable(); 106 } 107 108 static inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_from, 109 void *datap) 110 { 111 unsigned long vl; 112 113 riscv_v_enable(); 114 asm volatile ( 115 ".option push\n\t" 116 ".option arch, +v\n\t" 117 "vsetvli %0, x0, e8, m8, ta, ma\n\t" 118 "vle8.v v0, (%1)\n\t" 119 "add %1, %1, %0\n\t" 120 "vle8.v v8, (%1)\n\t" 121 "add %1, %1, %0\n\t" 122 "vle8.v v16, (%1)\n\t" 123 "add %1, %1, %0\n\t" 124 "vle8.v v24, (%1)\n\t" 125 ".option pop\n\t" 126 : "=&r" (vl) : "r" (datap) : "memory"); 127 __vstate_csr_restore(restore_from); 128 riscv_v_disable(); 129 } 130 131 static inline void riscv_v_vstate_save(struct task_struct *task, 132 struct pt_regs *regs) 133 { 134 if ((regs->status & SR_VS) == SR_VS_DIRTY) { 135 struct __riscv_v_ext_state *vstate = &task->thread.vstate; 136 137 __riscv_v_vstate_save(vstate, vstate->datap); 138 __riscv_v_vstate_clean(regs); 139 } 140 } 141 142 static inline void riscv_v_vstate_restore(struct task_struct *task, 143 struct pt_regs *regs) 144 { 145 if ((regs->status & SR_VS) != SR_VS_OFF) { 146 struct __riscv_v_ext_state *vstate = &task->thread.vstate; 147 148 __riscv_v_vstate_restore(vstate, vstate->datap); 149 __riscv_v_vstate_clean(regs); 150 } 151 } 152 153 static inline void __switch_to_vector(struct task_struct *prev, 154 struct task_struct *next) 155 { 156 struct pt_regs *regs; 157 158 regs = task_pt_regs(prev); 159 riscv_v_vstate_save(prev, regs); 160 riscv_v_vstate_restore(next, task_pt_regs(next)); 161 } 162 163 void riscv_v_vstate_ctrl_init(struct task_struct *tsk); 164 bool riscv_v_vstate_ctrl_user_allowed(void); 165 166 #else /* ! CONFIG_RISCV_ISA_V */ 167 168 struct pt_regs; 169 170 static inline int riscv_v_setup_vsize(void) { return -EOPNOTSUPP; } 171 static __always_inline bool has_vector(void) { return false; } 172 static inline bool riscv_v_first_use_handler(struct pt_regs *regs) { return false; } 173 static inline bool riscv_v_vstate_query(struct pt_regs *regs) { return false; } 174 static inline bool riscv_v_vstate_ctrl_user_allowed(void) { return false; } 175 #define riscv_v_vsize (0) 176 #define riscv_v_vstate_save(task, regs) do {} while (0) 177 #define riscv_v_vstate_restore(task, regs) do {} while (0) 178 #define __switch_to_vector(__prev, __next) do {} while (0) 179 #define riscv_v_vstate_off(regs) do {} while (0) 180 #define riscv_v_vstate_on(regs) do {} while (0) 181 182 #endif /* CONFIG_RISCV_ISA_V */ 183 184 #endif /* ! __ASM_RISCV_VECTOR_H */ 185