1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2024 Arm Ltd
5 * Copyright (c) 2022 The FreeBSD Foundation
6 *
7 * Portions of this software were developed by Andrew Turner under sponsorship
8 * from the FreeBSD Foundation.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 #include <sys/param.h>
33 #include <sys/bus.h>
34 #include <sys/conf.h>
35 #include <sys/event.h>
36 #include <sys/hwt.h>
37 #include <sys/kernel.h>
38 #include <sys/malloc.h>
39 #include <sys/module.h>
40 #include <sys/mutex.h>
41 #include <sys/rman.h>
42 #include <sys/smp.h>
43 #include <sys/systm.h>
44 #include <sys/taskqueue.h>
45
46 #include <machine/bus.h>
47
48 #include <arm64/spe/arm_spe.h>
49 #include <arm64/spe/arm_spe_dev.h>
50
51 MALLOC_DEFINE(M_ARM_SPE, "armspe", "Arm SPE tracing");
52
53 /*
54 * taskqueue(9) used for sleepable routines called from interrupt handlers
55 */
56 TASKQUEUE_FAST_DEFINE_THREAD(arm_spe);
57
58 void arm_spe_send_buffer(void *, int);
59 static void arm_spe_error(void *, int);
60 static int arm_spe_intr(void *);
61 device_attach_t arm_spe_attach;
62
63 static device_method_t arm_spe_methods[] = {
64 /* Device interface */
65 DEVMETHOD(device_attach, arm_spe_attach),
66
67 DEVMETHOD_END,
68 };
69
70 DEFINE_CLASS_0(spe, arm_spe_driver, arm_spe_methods,
71 sizeof(struct arm_spe_softc));
72
73 #define ARM_SPE_KVA_MAX_ALIGN UL(2048)
74
75 int
arm_spe_attach(device_t dev)76 arm_spe_attach(device_t dev)
77 {
78 struct arm_spe_softc *sc;
79 int error, rid;
80
81 sc = device_get_softc(dev);
82 sc->dev = dev;
83
84 sc->pmbidr = READ_SPECIALREG(PMBIDR_EL1_REG);
85 sc->pmsidr = READ_SPECIALREG(PMSIDR_EL1_REG);
86 device_printf(dev, "PMBIDR_EL1: %#lx\n", sc->pmbidr);
87 device_printf(dev, "PMSIDR_EL1: %#lx\n", sc->pmsidr);
88 if ((sc->pmbidr & PMBIDR_P) != 0) {
89 device_printf(dev, "Profiling Buffer is owned by a higher Exception level\n");
90 return (EPERM);
91 }
92
93 sc->kva_align = 1 << ((sc->pmbidr & PMBIDR_Align_MASK) >> PMBIDR_Align_SHIFT);
94 if (sc->kva_align > ARM_SPE_KVA_MAX_ALIGN) {
95 device_printf(dev, "Invalid PMBIDR.Align value of %d\n", sc->kva_align);
96 return (EINVAL);
97 }
98
99 rid = 0;
100 sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
101 RF_ACTIVE);
102 if (sc->sc_irq_res == NULL) {
103 device_printf(dev, "Unable to allocate interrupt\n");
104 return (ENXIO);
105 }
106 error = bus_setup_intr(dev, sc->sc_irq_res,
107 INTR_TYPE_MISC | INTR_MPSAFE, arm_spe_intr, NULL, sc,
108 &sc->sc_irq_cookie);
109 if (error != 0) {
110 device_printf(dev, "Unable to set up interrupt\n");
111 return (error);
112 }
113
114 mtx_init(&sc->sc_lock, "Arm SPE lock", NULL, MTX_SPIN);
115
116 STAILQ_INIT(&sc->pending);
117 sc->npending = 0;
118
119 spe_register(dev);
120
121 return (0);
122 }
123
124 /* Interrupt handler runs on the same core that triggered the exception */
125 static int
arm_spe_intr(void * arg)126 arm_spe_intr(void *arg)
127 {
128 int cpu_id = PCPU_GET(cpuid);
129 struct arm_spe_softc *sc = arg;
130 uint64_t pmbsr;
131 uint64_t base, limit;
132 uint8_t ec;
133 struct arm_spe_info *info = &sc->spe_info[cpu_id];
134 uint8_t i = info->buf_idx;
135 struct arm_spe_buf_info *buf = &info->buf_info[i];
136 struct arm_spe_buf_info *prev_buf = &info->buf_info[!i];
137 device_t dev = sc->dev;
138
139 /* Make sure the profiling data is visible to the CPU */
140 psb_csync();
141 dsb(nsh);
142
143 /* Make sure any HW update of PMBPTR_EL1 is visible to the CPU */
144 isb();
145
146 pmbsr = READ_SPECIALREG(PMBSR_EL1_REG);
147
148 if (!(pmbsr & PMBSR_S))
149 return (FILTER_STRAY);
150
151 /* Event Class */
152 ec = PMBSR_EC_VAL(pmbsr);
153 switch (ec)
154 {
155 case PMBSR_EC_OTHER_BUF_MGMT: /* Other buffer management event */
156 break;
157 case PMBSR_EC_GRAN_PROT_CHK: /* Granule Protection Check fault */
158 device_printf(dev, "PMBSR_EC_GRAN_PROT_CHK\n");
159 break;
160 case PMBSR_EC_STAGE1_DA: /* Stage 1 Data Abort */
161 device_printf(dev, "PMBSR_EC_STAGE1_DA\n");
162 break;
163 case PMBSR_EC_STAGE2_DA: /* Stage 2 Data Abort */
164 device_printf(dev, "PMBSR_EC_STAGE2_DA\n");
165 break;
166 default:
167 /* Unknown EC */
168 device_printf(dev, "unknown PMBSR_EC: %#x\n", ec);
169 arm_spe_disable(NULL);
170 TASK_INIT(&sc->task, 0, (task_fn_t *)arm_spe_error, sc->ctx);
171 taskqueue_enqueue(taskqueue_arm_spe, &sc->task);
172 return (FILTER_HANDLED);
173 }
174
175 switch (ec) {
176 case PMBSR_EC_OTHER_BUF_MGMT:
177 /* Buffer Status Code = buffer filled */
178 if ((pmbsr & PMBSR_MSS_BSC_MASK) == PMBSR_MSS_BSC_BUFFER_FILLED) {
179 dprintf("%s SPE buffer full event (cpu:%d)\n",
180 __func__, cpu_id);
181 break;
182 }
183 case PMBSR_EC_GRAN_PROT_CHK:
184 case PMBSR_EC_STAGE1_DA:
185 case PMBSR_EC_STAGE2_DA:
186 /*
187 * If we have one of these, we've messed up the
188 * programming somehow (e.g. passed invalid memory to
189 * SPE) and can't recover
190 */
191 arm_spe_disable(NULL);
192 TASK_INIT(&sc->task, 0, (task_fn_t *)arm_spe_error, sc->ctx);
193 taskqueue_enqueue(taskqueue_arm_spe, &sc->task);
194 /* PMBPTR_EL1 is fault address if PMBSR_DL is 1 */
195 device_printf(dev, "CPU:%d PMBSR_EL1:%#lx\n", cpu_id, pmbsr);
196 device_printf(dev, "PMBPTR_EL1:%#lx PMBLIMITR_EL1:%#lx\n",
197 READ_SPECIALREG(PMBPTR_EL1_REG),
198 READ_SPECIALREG(PMBLIMITR_EL1_REG));
199 return (FILTER_HANDLED);
200 }
201
202 mtx_lock_spin(&info->lock);
203
204 /*
205 * Data Loss bit - pmbptr might not be pointing to the end of the last
206 * complete record
207 */
208 if ((pmbsr & PMBSR_DL) == PMBSR_DL)
209 buf->partial_rec = 1;
210 buf->pmbptr = READ_SPECIALREG(PMBPTR_EL1_REG);
211 buf->buf_svc = true;
212
213 /* Setup regs ready to start writing to the other half of the buffer */
214 info->buf_idx = !info->buf_idx;
215 base = buf_start_addr(info->buf_idx, info);
216 limit = base + (info->buf_size/2);
217 limit &= PMBLIMITR_LIMIT_MASK;
218 limit |= PMBLIMITR_E;
219 WRITE_SPECIALREG(PMBPTR_EL1_REG, base);
220 WRITE_SPECIALREG(PMBLIMITR_EL1_REG, limit);
221 isb();
222
223 /*
224 * Notify userspace via kqueue that buffer is full and needs copying
225 * out - since kqueue can sleep, don't do this in the interrupt handler,
226 * add to a taskqueue to be scheduled later instead
227 */
228 TASK_INIT(&info->task[i], 0, (task_fn_t *)arm_spe_send_buffer, buf);
229 taskqueue_enqueue(taskqueue_arm_spe, &info->task[i]);
230
231 /*
232 * It's possible userspace hasn't yet notified us they've copied out the
233 * other half of the buffer
234 *
235 * This might be because:
236 * a) Kernel hasn't scheduled the task via taskqueue to notify
237 * userspace to copy out the data
238 * b) Userspace is still copying the buffer or hasn't notified us
239 * back via the HWT_IOC_SVC_BUF ioctl
240 *
241 * Either way we need to avoid overwriting uncopied data in the
242 * buffer, so disable profiling until we receive that SVC_BUF
243 * ioctl
244 *
245 * Using a larger buffer size should help to minimise these events and
246 * loss of profiling data while profiling is disabled
247 */
248 if (prev_buf->buf_svc) {
249 device_printf(sc->dev, "cpu%d: buffer full interrupt, but other"
250 " half of buffer has not been copied out - consider"
251 " increasing buffer size to minimise loss of profiling data\n",
252 cpu_id);
253 WRITE_SPECIALREG(PMSCR_EL1_REG, 0x0);
254 prev_buf->buf_wait = true;
255 }
256
257 mtx_unlock_spin(&info->lock);
258
259 /* Clear Profiling Buffer Status Register */
260 WRITE_SPECIALREG(PMBSR_EL1_REG, 0);
261
262 isb();
263
264 return (FILTER_HANDLED);
265 }
266
267 /* note: Scheduled and run via taskqueue, so can run on any CPU at any time */
268 void
arm_spe_send_buffer(void * arg,int pending __unused)269 arm_spe_send_buffer(void *arg, int pending __unused)
270 {
271 struct arm_spe_buf_info *buf = (struct arm_spe_buf_info *)arg;
272 struct arm_spe_info *info = buf->info;
273 struct arm_spe_queue *queue;
274 struct kevent kev;
275 int ret;
276
277 queue = malloc(sizeof(struct arm_spe_queue), M_ARM_SPE,
278 M_WAITOK | M_ZERO);
279
280 mtx_lock_spin(&info->lock);
281
282 /* Add to queue for userspace to pickup */
283 queue->ident = info->ident;
284 queue->offset = buf->pmbptr - buf_start_addr(buf->buf_idx, info);
285 queue->buf_idx = buf->buf_idx;
286 queue->final_buf = !info->enabled;
287 queue->partial_rec = buf->partial_rec;
288 mtx_unlock_spin(&info->lock);
289
290 mtx_lock_spin(&info->sc->sc_lock);
291 STAILQ_INSERT_TAIL(&info->sc->pending, queue, next);
292 info->sc->npending++;
293 EV_SET(&kev, ARM_SPE_KQ_BUF, EVFILT_USER, 0, NOTE_TRIGGER,
294 info->sc->npending, NULL);
295 mtx_unlock_spin(&info->sc->sc_lock);
296
297 /* Notify userspace */
298 ret = kqfd_register(info->sc->kqueue_fd, &kev, info->sc->hwt_td,
299 M_WAITOK);
300 if (ret) {
301 dprintf("%s kqfd_register ret:%d\n", __func__, ret);
302 arm_spe_error(info->sc->ctx, 0);
303 }
304 }
305
306 static void
arm_spe_error(void * arg,int pending __unused)307 arm_spe_error(void *arg, int pending __unused)
308 {
309 struct hwt_context *ctx = arg;
310 struct kevent kev;
311 int ret;
312
313 smp_rendezvous_cpus(ctx->cpu_map, smp_no_rendezvous_barrier,
314 arm_spe_disable, smp_no_rendezvous_barrier, NULL);
315
316 EV_SET(&kev, ARM_SPE_KQ_SHUTDOWN, EVFILT_USER, 0, NOTE_TRIGGER, 0, NULL);
317 ret = kqfd_register(ctx->kqueue_fd, &kev, ctx->hwt_td, M_WAITOK);
318 if (ret)
319 dprintf("%s kqfd_register ret:%d\n", __func__, ret);
320 }
321
322 MODULE_DEPEND(spe, hwt, 1, 1, 1);
323 MODULE_VERSION(spe, 1);
324