1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2024 Arm Ltd
5 * Copyright (c) 2022 The FreeBSD Foundation
6 *
7 * Portions of this software were developed by Andrew Turner under sponsorship
8 * from the FreeBSD Foundation.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 /*
33 * Arm Statistical Profiling Extension (SPE) backend
34 *
35 * Basic SPE operation
36 *
37 * SPE is enabled and configured on a per-core basis, with each core requiring
38 * separate code to enable and configure. Each core also requires a separate
39 * buffer passed as config where the CPU will write profiling data. When the
40 * profiling buffer is full, an interrupt will be taken on the same CPU.
41 *
42 * Driver Design
43 *
44 * - HWT allocates a large single buffer per core. This buffer is split in half
45 * to create a 2 element circular buffer (aka ping-pong buffer) where the
46 * kernel writes to one half while userspace is copying the other half
47 * - SMP calls are used to enable and configure each core, with SPE initially
48 * configured to write to the first half of the buffer
49 * - When the first half of the buffer is full, a buffer full interrupt will
50 * immediately switch writing to the second half. The kernel adds the details
51 * of the half that needs copying to a FIFO STAILQ and notifies userspace via
52 * kqueue by sending a ARM_SPE_KQ_BUF kevent with how many buffers on the
53 * queue need servicing
54 * - The kernel responds to HWT_IOC_BUFPTR_GET ioctl by sending details of the
55 * first item from the queue
56 * - The buffers pending copying will not be overwritten until an
57 * HWT_IOC_SVC_BUF ioctl is received from userspace confirming the data has
58 * been copied out
59 * - In the case where both halfs of the buffer are full, profiling will be
60 * paused until notification via HWT_IOC_SVC_BUF is received
61 *
62 * Future improvements and limitations
63 *
64 * - Using large buffer sizes should minimise pauses and loss of profiling
65 * data while kernel is waiting for userspace to copy out data. Since it is
66 * generally expected that consuming (copying) this data is faster than
67 * producing it, in practice this has not so far been an issue. If it does
68 * prove to be an issue even with large buffer sizes then additional buffering
69 * i.e. n element circular buffers might be required.
70 *
71 * - kqueue can only notify and queue one kevent of the same type, with
72 * subsequent events overwriting data in the first event. The kevent
73 * ARM_SPE_KQ_BUF can therefore only contain the number of buffers on the
74 * STAILQ, incrementing each time a new buffer is full. In this case kqueue
75 * serves just as a notification to userspace to wake up and query the kernel
76 * with the appropriate ioctl. An alternative might be custom kevents where
77 * the kevent identifier is encoded with something like n+cpu_id or n+tid. In
78 * this case data could be sent directly with kqueue via the kevent data and
79 * fflags elements, avoiding the extra ioctl.
80 *
81 */
82
83 #include <sys/param.h>
84 #include <sys/bus.h>
85 #include <sys/conf.h>
86 #include <sys/hwt.h>
87 #include <sys/kernel.h>
88 #include <sys/lock.h>
89 #include <sys/malloc.h>
90 #include <sys/mman.h>
91 #include <sys/module.h>
92 #include <sys/mutex.h>
93 #include <sys/proc.h>
94 #include <sys/queue.h>
95 #include <sys/rman.h>
96 #include <sys/rwlock.h>
97 #include <sys/smp.h>
98 #include <sys/sysctl.h>
99 #include <sys/systm.h>
100 #include <sys/taskqueue.h>
101
102 #include <machine/bus.h>
103
104 #include <arm64/spe/arm_spe_dev.h>
105
106 #include <dev/hwt/hwt_vm.h>
107 #include <dev/hwt/hwt_backend.h>
108 #include <dev/hwt/hwt_config.h>
109 #include <dev/hwt/hwt_context.h>
110 #include <dev/hwt/hwt_cpu.h>
111 #include <dev/hwt/hwt_thread.h>
112
113 MALLOC_DECLARE(M_ARM_SPE);
114
115 extern u_int mp_maxid;
116 extern struct taskqueue *taskqueue_arm_spe;
117
118 int spe_backend_disable_smp(struct hwt_context *ctx);
119
120 static device_t spe_dev;
121 static struct hwt_backend_ops spe_ops;
122 static struct hwt_backend backend = {
123 .ops = &spe_ops,
124 .name = "spe",
125 .kva_req = 1,
126 };
127
128 /* Pointers to current info structure per CPU. This points to either a per-CPU
129 * structure (for CPU mode) or a per-thread structure (for thread mode).
130 */
131 static struct arm_spe_info **spe_info;
132
133 static struct arm_spe_info *spe_info_cpu;
134
135 static void
spe_backend_init_cpu(struct hwt_context * ctx)136 spe_backend_init_cpu(struct hwt_context *ctx)
137 {
138 struct arm_spe_info *info;
139 struct arm_spe_softc *sc = device_get_softc(spe_dev);
140 char lock_name[32];
141 char *tmp = "Arm SPE lock/cpu/";
142 int cpu_id;
143
144 spe_info_cpu = malloc(sizeof(struct arm_spe_info) * mp_ncpus,
145 M_ARM_SPE, M_WAITOK | M_ZERO);
146
147
148 CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {
149 info = &spe_info_cpu[cpu_id];
150 info->sc = sc;
151 info->ident = cpu_id;
152 info->buf_info[0].info = info;
153 info->buf_info[0].buf_idx = 0;
154 info->buf_info[1].info = info;
155 info->buf_info[1].buf_idx = 1;
156 snprintf(lock_name, sizeof(lock_name), "%s%d", tmp, cpu_id);
157 mtx_init(&info->lock, lock_name, NULL, MTX_SPIN);
158
159 spe_info[cpu_id] = info;
160 }
161 }
162
163 static int
spe_backend_init(struct hwt_context * ctx)164 spe_backend_init(struct hwt_context *ctx)
165 {
166 struct arm_spe_softc *sc = device_get_softc(spe_dev);
167 int error = 0;
168
169 /*
170 * HWT currently specifies buffer size must be a multiple of PAGE_SIZE,
171 * i.e. minimum 4KB + the maximum PMBIDR.Align is 2KB
172 * This should never happen but it's good to sense check
173 */
174 if (ctx->bufsize % sc->kva_align != 0)
175 return (EINVAL);
176
177 /*
178 * Since we're splitting the buffer in half + PMBLIMITR needs to be page
179 * aligned, minimum buffer size needs to be 2x PAGE_SIZE
180 */
181 if (ctx->bufsize < (2 * PAGE_SIZE))
182 return (EINVAL);
183
184 sc->ctx = ctx;
185 sc->kqueue_fd = ctx->kqueue_fd;
186 sc->hwt_td = ctx->hwt_td;
187
188 spe_info = malloc(sizeof(struct arm_spe_info *) * mp_ncpus,
189 M_ARM_SPE, M_WAITOK | M_ZERO);
190 sc->spe_info = spe_info;
191
192 if (ctx->mode == HWT_MODE_CPU)
193 spe_backend_init_cpu(ctx);
194
195 return (error);
196 }
197
198 #ifdef ARM_SPE_DEBUG
hex_dump(uint8_t * buf,size_t len)199 static void hex_dump(uint8_t *buf, size_t len)
200 {
201 size_t i;
202
203 printf("--------------------------------------------------------------\n");
204 for (i = 0; i < len; ++i) {
205 if (i % 8 == 0) {
206 printf(" ");
207 }
208 if (i % 16 == 0) {
209 if (i != 0) {
210 printf("\r\n");
211 }
212 printf("\t");
213 }
214 printf("%02X ", buf[i]);
215 }
216 printf("\r\n");
217 }
218 #endif
219
220 static int
spe_backend_deinit(struct hwt_context * ctx)221 spe_backend_deinit(struct hwt_context *ctx)
222 {
223 #ifdef ARM_SPE_DEBUG
224 struct arm_spe_info *info;
225 struct hwt_thread *thr;
226 int cpu_id;
227
228 if (ctx->mode == HWT_MODE_CPU) {
229 CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {
230 info = &spe_info_cpu[cpu_id];
231 printf("CPU %u:\n", cpu_id);
232 hex_dump((void *)info->kvaddr, 128);
233 hex_dump((void *)(info->kvaddr + (info->buf_size/2)), 128);
234 }
235 } else {
236 TAILQ_FOREACH(thr, &ctx->threads, next) {
237 info = (struct arm_spe_info *)thr->private;
238 printf("TID %u:\n", thr->thread_id);
239 hex_dump((void *)info->kvaddr, 128);
240 hex_dump((void *)(info->kvaddr + (info->buf_size/2)), 128);
241 }
242 }
243 #endif
244
245 spe_backend_disable_smp(ctx);
246
247 if (ctx->mode == HWT_MODE_CPU)
248 free(spe_info_cpu, M_ARM_SPE);
249
250 free(spe_info, M_ARM_SPE);
251
252 return (0);
253 }
254
255 static uint64_t
arm_spe_min_interval(struct arm_spe_softc * sc)256 arm_spe_min_interval(struct arm_spe_softc *sc)
257 {
258 /* IMPLEMENTATION DEFINED */
259 switch (PMSIDR_Interval_VAL(sc->pmsidr))
260 {
261 case PMSIDR_Interval_256:
262 return (256);
263 case PMSIDR_Interval_512:
264 return (512);
265 case PMSIDR_Interval_768:
266 return (768);
267 case PMSIDR_Interval_1024:
268 return (1024);
269 case PMSIDR_Interval_1536:
270 return (1536);
271 case PMSIDR_Interval_2048:
272 return (2048);
273 case PMSIDR_Interval_3072:
274 return (3072);
275 case PMSIDR_Interval_4096:
276 return (4096);
277 default:
278 return (4096);
279 }
280 }
281
282 static inline void
arm_spe_set_interval(struct arm_spe_info * info,uint64_t interval)283 arm_spe_set_interval(struct arm_spe_info *info, uint64_t interval)
284 {
285 uint64_t min_interval = arm_spe_min_interval(info->sc);
286
287 interval = MAX(interval, min_interval);
288 interval = MIN(interval, 1 << 24); /* max 24 bits */
289
290 dprintf("%s %lu\n", __func__, interval);
291
292 info->pmsirr &= ~(PMSIRR_INTERVAL_MASK);
293 info->pmsirr |= (interval << PMSIRR_INTERVAL_SHIFT);
294 }
295
296 static int
spe_backend_configure(struct hwt_context * ctx,int cpu_id,int thread_id)297 spe_backend_configure(struct hwt_context *ctx, int cpu_id, int thread_id)
298 {
299 struct arm_spe_info *info = NULL;
300 struct arm_spe_config *cfg;
301 struct hwt_thread *thr = NULL;
302 int err = 0;
303
304 if (ctx->mode == HWT_MODE_CPU)
305 info = &spe_info_cpu[cpu_id];
306 else {
307 TAILQ_FOREACH(thr, &ctx->threads, next) {
308 if (thr->thread_id != thread_id)
309 continue;
310 info = (struct arm_spe_info *)thr->private;
311 break;
312 }
313 if (info == NULL)
314 return (ENOENT);
315 }
316
317 mtx_lock_spin(&info->lock);
318 if (ctx->mode == HWT_MODE_CPU)
319 info->ident = cpu_id;
320 else
321 info->ident = thread_id;
322 /* Set defaults */
323 info->pmsfcr = 0;
324 info->pmsevfr = 0xFFFFFFFFFFFFFFFFUL;
325 info->pmslatfr = 0;
326 info->pmsirr =
327 (arm_spe_min_interval(info->sc) << PMSIRR_INTERVAL_SHIFT)
328 | PMSIRR_RND;
329 info->pmsicr = 0;
330 info->pmscr = PMSCR_TS | PMSCR_PA | PMSCR_CX | PMSCR_E1SPE | PMSCR_E0SPE;
331
332 if (ctx->config != NULL &&
333 ctx->config_size == sizeof(struct arm_spe_config) &&
334 ctx->config_version == 1) {
335 cfg = (struct arm_spe_config *)ctx->config;
336 if (cfg->interval)
337 arm_spe_set_interval(info, cfg->interval);
338 if (cfg->level == ARM_SPE_KERNEL_ONLY)
339 info->pmscr &= ~(PMSCR_E0SPE); /* turn off user */
340 if (cfg->level == ARM_SPE_USER_ONLY)
341 info->pmscr &= ~(PMSCR_E1SPE); /* turn off kern */
342 if (cfg->ctx_field)
343 info->ctx_field = cfg->ctx_field;
344 } else
345 err = (EINVAL);
346
347 if (ctx->mode == HWT_MODE_THREAD) {
348 info->kvaddr = thr->vm->kvaddr;
349 info->buf_size = ctx->bufsize;
350 }
351
352 spe_info[cpu_id] = info;
353 mtx_unlock_spin(&info->lock);
354
355 return (err);
356 }
357
358
359 static void
arm_spe_enable(void * arg __unused)360 arm_spe_enable(void *arg __unused)
361 {
362 struct arm_spe_info *info = spe_info[PCPU_GET(cpuid)];
363 struct arm_spe_buf_info *buf = &info->buf_info[info->buf_idx];
364 struct hwt_context *ctx = info->sc->ctx;
365 uint64_t base, limit;
366
367 dprintf("%s on cpu:%d\n", __func__, PCPU_GET(cpuid));
368
369 mtx_lock_spin(&info->lock);
370
371 if (info->stopped) {
372 mtx_unlock_spin(&info->lock);
373 return;
374 }
375
376 if (info->ctx_field == ARM_SPE_CTX_CPU_ID)
377 WRITE_SPECIALREG(CONTEXTIDR_EL1_REG, PCPU_GET(cpuid));
378
379 WRITE_SPECIALREG(PMSFCR_EL1_REG, info->pmsfcr);
380 WRITE_SPECIALREG(PMSEVFR_EL1_REG, info->pmsevfr);
381 WRITE_SPECIALREG(PMSLATFR_EL1_REG, info->pmslatfr);
382
383 /* Set the sampling interval */
384 WRITE_SPECIALREG(PMSIRR_EL1_REG, info->pmsirr);
385 isb();
386
387 /* Write 0 here before enabling sampling */
388 WRITE_SPECIALREG(PMSICR_EL1_REG, info->pmsicr);
389 isb();
390
391 base = buf_start_addr(info->buf_idx, info);
392 limit = base + (info->buf_size/2);
393 /* Enable the buffer */
394 limit &= PMBLIMITR_LIMIT_MASK; /* Zero lower 12 bits */
395 limit |= PMBLIMITR_E;
396 /* Set the base and limit. Restore base pointer if sampling has previously
397 * been enabled for this thread.
398 */
399 if (buf->pmbptr == 0) {
400 WRITE_SPECIALREG(PMBPTR_EL1_REG, base);
401 } else {
402 WRITE_SPECIALREG(PMBPTR_EL1_REG, buf->pmbptr);
403 }
404 WRITE_SPECIALREG(PMBLIMITR_EL1_REG, limit);
405 isb();
406
407 /* Enable sampling */
408 WRITE_SPECIALREG(PMSCR_EL1_REG, info->pmscr);
409 isb();
410
411 info->enabled = true;
412
413 if (ctx->mode == HWT_MODE_THREAD)
414 CPU_SET(PCPU_GET(cpuid), &ctx->cpu_map);
415
416 mtx_unlock_spin(&info->lock);
417 }
418
419 static int
spe_backend_enable_smp(struct hwt_context * ctx)420 spe_backend_enable_smp(struct hwt_context *ctx)
421 {
422 struct arm_spe_info *info;
423 struct hwt_vm *vm;
424 int cpu_id;
425
426 KASSERT(ctx->mode == HWT_MODE_CPU, ("%s: should only be called for CPU mode", __func__));
427
428 HWT_CTX_LOCK(ctx);
429 CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {
430 vm = hwt_cpu_get(ctx, cpu_id)->vm;
431 KASSERT(spe_info[cpu_id] == &spe_info_cpu[cpu_id], ("%s: spe_info mismatch for cpu_id=%u", __func__, cpu_id));
432 info = &spe_info_cpu[cpu_id];
433
434 mtx_lock_spin(&info->lock);
435 info->kvaddr = vm->kvaddr;
436 info->buf_size = ctx->bufsize;
437 mtx_unlock_spin(&info->lock);
438 }
439 HWT_CTX_UNLOCK(ctx);
440
441 cpu_id = CPU_FFS(&ctx->cpu_map) - 1;
442 KASSERT(spe_info[cpu_id] == &spe_info_cpu[cpu_id], ("%s: spe_info mismatch for cpu_id=%u", __func__, cpu_id));
443 info = spe_info[cpu_id];
444 if (info->ctx_field == ARM_SPE_CTX_PID)
445 arm64_pid_in_contextidr = true;
446 else
447 arm64_pid_in_contextidr = false;
448
449 smp_rendezvous_cpus(ctx->cpu_map, smp_no_rendezvous_barrier,
450 arm_spe_enable, smp_no_rendezvous_barrier, NULL);
451
452 return (0);
453 }
454
455 static void
arm_spe_disable_nolock(void)456 arm_spe_disable_nolock(void)
457 {
458 struct arm_spe_info *info = spe_info[PCPU_GET(cpuid)];
459 struct arm_spe_buf_info *buf = &info->buf_info[info->buf_idx];
460 struct hwt_context *ctx = info->sc->ctx;
461
462 if (!info->enabled)
463 return;
464
465 dprintf("%s on cpu:%d\n", __func__, PCPU_GET(cpuid));
466
467 /* Disable profiling */
468 WRITE_SPECIALREG(PMSCR_EL1_REG, 0x0);
469 isb();
470
471 /* Drain any remaining tracing data */
472 psb_csync();
473 dsb(nsh);
474
475 /* Disable the profiling buffer */
476 WRITE_SPECIALREG(PMBLIMITR_EL1_REG, 0);
477 isb();
478
479 /* Clear interrupt status reg */
480 WRITE_SPECIALREG(PMBSR_EL1_REG, 0x0);
481
482 /* Clear PID/CPU_ID from context ID reg */
483 WRITE_SPECIALREG(CONTEXTIDR_EL1_REG, 0);
484
485 buf->pmbptr = READ_SPECIALREG(PMBPTR_EL1_REG);
486 info->enabled = false;
487
488 if (ctx->mode == HWT_MODE_THREAD)
489 CPU_CLR(PCPU_GET(cpuid), &ctx->cpu_map);
490 }
491
492 void
arm_spe_disable(void * arg __unused)493 arm_spe_disable(void *arg __unused)
494 {
495 struct arm_spe_info *info = spe_info[PCPU_GET(cpuid)];
496
497 mtx_lock_spin(&info->lock);
498 arm_spe_disable_nolock();
499 mtx_unlock_spin(&info->lock);
500 }
501
502 int
spe_backend_disable_smp(struct hwt_context * ctx)503 spe_backend_disable_smp(struct hwt_context *ctx)
504 {
505 struct kevent kev;
506 struct arm_spe_info *info;
507 struct arm_spe_buf_info *buf;
508 int cpu_id;
509 int ret;
510
511 if (!CPU_EMPTY(&ctx->cpu_map)) {
512 /* Disable and send out remaining data in bufs */
513 smp_rendezvous_cpus(ctx->cpu_map, smp_no_rendezvous_barrier,
514 arm_spe_disable, smp_no_rendezvous_barrier, NULL);
515
516 CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) {
517 info = spe_info[cpu_id];
518 buf = &info->buf_info[info->buf_idx];
519 arm_spe_send_buffer(buf, 0);
520 }
521 }
522
523 arm64_pid_in_contextidr = false;
524
525 /*
526 * Tracing on all CPUs has been disabled, and we've sent write ptr
527 * offsets for all bufs - let userspace know it can shutdown
528 */
529 EV_SET(&kev, ARM_SPE_KQ_SHUTDOWN, EVFILT_USER, 0, NOTE_TRIGGER, 0, NULL);
530 ret = kqfd_register(ctx->kqueue_fd, &kev, ctx->hwt_td, M_WAITOK);
531 if (ret)
532 dprintf("%s kqfd_register ret:%d\n", __func__, ret);
533
534 return (0);
535 }
536
537 static void
spe_backend_enable(struct hwt_context * ctx,int cpu_id)538 spe_backend_enable(struct hwt_context *ctx, int cpu_id)
539 {
540 struct arm_spe_info *info;
541
542 if (ctx->mode == HWT_MODE_CPU)
543 return;
544 KASSERT(curcpu == cpu_id, ("%s: attempting to enable SPE on another cpu", __func__));
545
546 info = spe_info[cpu_id];
547
548 KASSERT(info != NULL, ("%s: info=NULL", __func__));
549
550 if (info->ctx_field == ARM_SPE_CTX_PID)
551 arm64_pid_in_contextidr = true;
552 else
553 arm64_pid_in_contextidr = false;
554
555 arm_spe_enable(NULL);
556 }
557
558 static void
spe_backend_disable(struct hwt_context * ctx,int cpu_id)559 spe_backend_disable(struct hwt_context *ctx, int cpu_id)
560 {
561 struct arm_spe_info *info = spe_info[PCPU_GET(cpuid)];
562
563 if (ctx->mode == HWT_MODE_CPU)
564 return;
565
566 KASSERT(curcpu == cpu_id, ("%s: attempting to disable SPE on another cpu", __func__));
567
568 mtx_lock_spin(&info->lock);
569
570 if (!info->stopped)
571 arm_spe_disable_nolock();
572
573 mtx_unlock_spin(&info->lock);
574 }
575
576 static void
arm_spe_flush(void * arg,int pending __unused)577 arm_spe_flush(void *arg, int pending __unused)
578 {
579 struct arm_spe_info *info = arg;
580 struct arm_spe_buf_info *buf = &info->buf_info[info->buf_idx];
581
582 arm_spe_send_buffer(buf, 0);
583 }
584
585 static void
spe_backend_stop(struct hwt_context * ctx)586 spe_backend_stop(struct hwt_context *ctx)
587 {
588 struct arm_spe_info *info;
589 struct hwt_thread *thr;
590
591 HWT_CTX_LOCK(ctx);
592
593 if (ctx->mode == HWT_MODE_THREAD) {
594 ctx->state = CTX_STATE_STOPPED;
595
596 TAILQ_FOREACH(thr, &ctx->threads, next) {
597 info = (struct arm_spe_info *)thr->private;
598
599 mtx_lock_spin(&info->lock);
600
601 info->stopped = true;
602
603 if (!info->enabled) {
604 /* Not currently tracing. Enqueue buffer for sending */
605 TASK_INIT(&info->flush_task, 0, (task_fn_t *)arm_spe_flush, info);
606 taskqueue_enqueue(taskqueue_arm_spe, &info->flush_task);
607 }
608 /* Otherwise tracing currently active. As this thread has been
609 * marked as stopped, buffer will be sent on next disable
610 */
611
612 mtx_unlock_spin(&info->lock);
613 }
614
615 }
616
617 HWT_CTX_UNLOCK(ctx);
618
619 taskqueue_drain_all(taskqueue_arm_spe);
620
621 spe_backend_disable_smp(ctx);
622 }
623
624 static void
arm_spe_reenable(void * arg __unused)625 arm_spe_reenable(void *arg __unused)
626 {
627 struct arm_spe_info *info = spe_info[PCPU_GET(cpuid)];
628
629 WRITE_SPECIALREG(PMSCR_EL1_REG, info->pmscr);
630 isb();
631 }
632
633 static int
spe_backend_svc_buf(struct hwt_context * ctx,void * data,size_t data_size,int data_version)634 spe_backend_svc_buf(struct hwt_context *ctx, void *data, size_t data_size,
635 int data_version)
636 {
637 struct arm_spe_info *info = NULL;
638 struct arm_spe_buf_info *buf;
639 struct arm_spe_svc_buf *s;
640 struct hwt_thread *thr;
641 int err = 0;
642 cpuset_t cpu_set;
643
644 if (data_size != sizeof(struct arm_spe_svc_buf))
645 return (E2BIG);
646
647 if (data_version != 1)
648 return (EINVAL);
649
650 s = (struct arm_spe_svc_buf *)data;
651 if (s->buf_idx > 1)
652 return (ENODEV);
653
654 if (ctx->mode == HWT_MODE_CPU) {
655 if (s->ident >= mp_ncpus)
656 return (EINVAL);
657
658 info = spe_info[s->ident];
659 } else {
660 TAILQ_FOREACH(thr, &ctx->threads, next) {
661 if (thr->thread_id != s->ident)
662 continue;
663 info = (struct arm_spe_info *)thr->private;
664 break;
665 }
666
667 if (info == NULL)
668 return (ENOENT);
669 }
670
671 mtx_lock_spin(&info->lock);
672
673 buf = &info->buf_info[s->buf_idx];
674
675 if (!info->enabled && ctx->mode == HWT_MODE_CPU) {
676 err = ENXIO;
677 goto end;
678 }
679
680 /* Clear the flag the signals buffer needs servicing */
681 buf->buf_svc = false;
682
683 /* Re-enable profiling if we've been waiting for this notification */
684 if (buf->buf_wait && !info->stopped) {
685 CPU_SETOF(s->ident, &cpu_set);
686
687 mtx_unlock_spin(&info->lock);
688 smp_rendezvous_cpus(cpu_set, smp_no_rendezvous_barrier,
689 arm_spe_reenable, smp_no_rendezvous_barrier, NULL);
690 mtx_lock_spin(&info->lock);
691
692 buf->buf_wait = false;
693 }
694
695 end:
696 mtx_unlock_spin(&info->lock);
697 return (err);
698 }
699
700 static int
spe_backend_read(struct hwt_vm * vm,int * ident,vm_offset_t * offset,uint64_t * data)701 spe_backend_read(struct hwt_vm *vm, int *ident, vm_offset_t *offset,
702 uint64_t *data)
703 {
704 struct arm_spe_queue *q;
705 struct arm_spe_softc *sc = device_get_softc(spe_dev);
706 int error = 0;
707
708 mtx_lock_spin(&sc->sc_lock);
709
710 /* Return the first pending buffer that needs servicing */
711 q = STAILQ_FIRST(&sc->pending);
712 if (q == NULL) {
713 error = ENOENT;
714 goto error;
715 }
716 *ident = q->ident;
717 *offset = q->offset;
718 *data = (q->buf_idx << KQ_BUF_POS_SHIFT) |
719 (q->partial_rec << KQ_PARTREC_SHIFT) |
720 (q->final_buf << KQ_FINAL_BUF_SHIFT);
721
722 STAILQ_REMOVE_HEAD(&sc->pending, next);
723 sc->npending--;
724
725 error:
726 mtx_unlock_spin(&sc->sc_lock);
727 if (error)
728 return (error);
729
730 free(q, M_ARM_SPE);
731 return (0);
732 }
733
734 static int
spe_backend_thread_alloc(struct hwt_thread * thr)735 spe_backend_thread_alloc(struct hwt_thread *thr)
736 {
737 struct arm_spe_softc *sc = device_get_softc(spe_dev);
738 char lock_name[32];
739 struct arm_spe_info *info;
740
741 info = malloc(sizeof(*info), M_ARM_SPE, M_WAITOK | M_ZERO);
742
743 info->sc = sc;
744 info->buf_info[0].info = info;
745 info->buf_info[0].buf_idx = 0;
746 info->buf_info[1].info = info;
747 info->buf_info[1].buf_idx = 1;
748 snprintf(lock_name, sizeof(lock_name), "Arm SPE lock/thr/%d", thr->thread_id);
749 mtx_init(&info->lock, lock_name, NULL, MTX_SPIN);
750
751 thr->private = info;
752
753 return (0);
754 }
755
756 static void
spe_backend_thread_free(struct hwt_thread * thr)757 spe_backend_thread_free(struct hwt_thread *thr)
758 {
759 struct arm_spe_info *info;
760
761 info = (struct arm_spe_info *)thr->private;
762
763 free(info, M_ARM_SPE);
764 }
765
766 static struct hwt_backend_ops spe_ops = {
767 .hwt_backend_init = spe_backend_init,
768 .hwt_backend_deinit = spe_backend_deinit,
769
770 .hwt_backend_configure = spe_backend_configure,
771 .hwt_backend_svc_buf = spe_backend_svc_buf,
772 .hwt_backend_stop = spe_backend_stop,
773
774 .hwt_backend_enable = spe_backend_enable,
775 .hwt_backend_disable = spe_backend_disable,
776
777 .hwt_backend_enable_smp = spe_backend_enable_smp,
778 .hwt_backend_disable_smp = spe_backend_disable_smp,
779
780 .hwt_backend_read = spe_backend_read,
781
782 .hwt_backend_thread_alloc = spe_backend_thread_alloc,
783 .hwt_backend_thread_free = spe_backend_thread_free,
784 };
785
786 int
spe_register(device_t dev)787 spe_register(device_t dev)
788 {
789 spe_dev = dev;
790
791 return (hwt_backend_register(&backend));
792 }
793