1 /*-
2 * Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #include <sys/param.h>
27 #include <sys/systm.h>
28 #include <sys/eventhandler.h>
29 #include <sys/kernel.h>
30 #include <sys/proc.h>
31
32 #include <vm/uma.h>
33
34 #include <machine/vector.h>
35 #include <machine/pcb.h>
36 #include <machine/reg.h>
37 #include <machine/md_var.h>
38
39 static MALLOC_DEFINE(M_RVV_CTX, "rvv_ctx", "Contexts for user Vector state");
40
41 static void
vector_enable(void)42 vector_enable(void)
43 {
44 uint64_t reg;
45
46 reg = SSTATUS_VS_CLEAN;
47
48 csr_set(sstatus, reg);
49 }
50
51 static void
vector_disable(void)52 vector_disable(void)
53 {
54 uint64_t mask;
55
56 mask = SSTATUS_VS_MASK;
57
58 csr_clear(sstatus, mask);
59 }
60
61 static void
vector_state_store_common(struct pcb * p,void * datap)62 vector_state_store_common(struct pcb *p, void *datap)
63 {
64 uint64_t vl;
65
66 vector_enable();
67
68 /* Store vector CSRs. */
69 __asm __volatile(
70 "csrr %0, vstart\n\t"
71 "csrr %1, vtype\n\t"
72 "csrr %2, vl\n\t"
73 "csrr %3, vcsr\n\t"
74 : "=r" (p->pcb_vstart), "=r" (p->pcb_vtype),
75 "=r" (p->pcb_vl), "=r" (p->pcb_vcsr) ::);
76
77 if (datap == NULL)
78 goto done;
79
80 /* Store vector registers. */
81 __asm __volatile(
82 ".option push\n\t"
83 ".option arch, +zve32x\n\t"
84 "vsetvli %0, x0, e8, m8, ta, ma\n\t"
85 "vse8.v v0, (%1)\n\t"
86 "add %1, %1, %0\n\t"
87 "vse8.v v8, (%1)\n\t"
88 "add %1, %1, %0\n\t"
89 "vse8.v v16, (%1)\n\t"
90 "add %1, %1, %0\n\t"
91 "vse8.v v24, (%1)\n\t"
92 ".option pop\n\t"
93 : "=&r" (vl) : "r" (datap) : "memory");
94
95 done:
96 vector_disable();
97 }
98
99 void
vector_state_store_savectx(struct pcb * p)100 vector_state_store_savectx(struct pcb *p)
101 {
102
103 if (!has_vector)
104 return;
105
106 /*
107 * savectx() will be called on panic with dumppcb as an argument.
108 * Save the vector CSR registers, but ignore the vector data.
109 */
110
111 vector_state_store_common(p, NULL);
112 }
113
114 void
vector_state_store(struct thread * td)115 vector_state_store(struct thread *td)
116 {
117 struct pcb *p;
118 void *datap;
119
120 p = td->td_pcb;
121
122 KASSERT(p->pcb_vsaved != NULL, ("VS area is NULL"));
123
124 datap = p->pcb_vsaved;
125
126 vector_state_store_common(p, datap);
127 }
128
129 void
vector_state_restore(struct thread * td)130 vector_state_restore(struct thread *td)
131 {
132 struct pcb *p;
133 void *datap;
134 uint64_t vl;
135
136 p = td->td_pcb;
137
138 KASSERT(p->pcb_vsaved != NULL, ("VS area is NULL"));
139
140 datap = p->pcb_vsaved;
141
142 vector_enable();
143
144 /* Restore vector registers. */
145 __asm __volatile(
146 ".option push\n\t"
147 ".option arch, +zve32x\n\t"
148 "vsetvli %0, x0, e8, m8, ta, ma\n\t"
149 "vle8.v v0, (%1)\n\t"
150 "add %1, %1, %0\n\t"
151 "vle8.v v8, (%1)\n\t"
152 "add %1, %1, %0\n\t"
153 "vle8.v v16, (%1)\n\t"
154 "add %1, %1, %0\n\t"
155 "vle8.v v24, (%1)\n\t"
156 ".option pop\n\t"
157 : "=&r" (vl) : "r" (datap) : "memory");
158
159 /* Restore vector CSRs. */
160 __asm __volatile(
161 ".option push\n\t"
162 ".option arch, +zve32x\n\t"
163 "vsetvl x0, %2, %1\n\t"
164 ".option pop\n\t"
165 "csrw vstart, %0\n\t"
166 "csrw vcsr, %3\n\t"
167 :: "r" (p->pcb_vstart), "r" (p->pcb_vtype),
168 "r" (p->pcb_vl), "r" (p->pcb_vcsr));
169
170 vector_disable();
171 }
172
173 int
vector_get_size(void)174 vector_get_size(void)
175 {
176 int len;
177
178 /* RISC-V has 32 vector registers of vlenb size each. */
179
180 vector_enable();
181 len = csr_read(vlenb) * 32;
182 vector_disable();
183
184 return (len);
185 }
186
187 void
vector_state_init(struct thread * td)188 vector_state_init(struct thread *td)
189 {
190 struct pcb *p;
191 int len;
192
193 p = td->td_pcb;
194
195 KASSERT(p->pcb_vsaved == NULL, ("vsaved is already initialized"));
196
197 len = vector_get_size();
198
199 p->pcb_vsaved = malloc(len, M_RVV_CTX, M_WAITOK | M_ZERO);
200 }
201
202 void
vector_copy_thread(struct thread * td1,struct thread * td2)203 vector_copy_thread(struct thread *td1, struct thread *td2)
204 {
205 struct pcb *p1;
206 struct pcb *p2;
207 int len;
208
209 /* Struct pcb already copied, now init the vector save area. */
210
211 p1 = td1->td_pcb;
212 p2 = td2->td_pcb;
213 p2->pcb_vsaved = NULL;
214
215 vector_state_init(td2);
216
217 len = vector_get_size();
218
219 memcpy(p2->pcb_vsaved, p1->pcb_vsaved, len);
220 }
221
222 static void
vector_thread_dtor(void * arg __unused,struct thread * td)223 vector_thread_dtor(void *arg __unused, struct thread *td)
224 {
225 void *datap;
226
227 datap = td->td_pcb->pcb_vsaved;
228
229 free(datap, M_RVV_CTX);
230 }
231
232 static void
vector_init(const void * dummy __unused)233 vector_init(const void *dummy __unused)
234 {
235
236 EVENTHANDLER_REGISTER(thread_dtor, vector_thread_dtor, NULL,
237 EVENTHANDLER_PRI_ANY);
238 }
239
240 SYSINIT(vector, SI_SUB_SMP, SI_ORDER_ANY, vector_init, NULL);
241