1*df114daeSRuslan Bukin /*-
2*df114daeSRuslan Bukin * SPDX-License-Identifier: BSD-2-Clause
3*df114daeSRuslan Bukin *
4*df114daeSRuslan Bukin * Copyright (c) 2023-2025 Ruslan Bukin <br@bsdpad.com>
5*df114daeSRuslan Bukin *
6*df114daeSRuslan Bukin * This work was supported by Innovate UK project 105694, "Digital Security
7*df114daeSRuslan Bukin * by Design (DSbD) Technology Platform Prototype".
8*df114daeSRuslan Bukin *
9*df114daeSRuslan Bukin * Redistribution and use in source and binary forms, with or without
10*df114daeSRuslan Bukin * modification, are permitted provided that the following conditions
11*df114daeSRuslan Bukin * are met:
12*df114daeSRuslan Bukin * 1. Redistributions of source code must retain the above copyright
13*df114daeSRuslan Bukin * notice, this list of conditions and the following disclaimer.
14*df114daeSRuslan Bukin * 2. Redistributions in binary form must reproduce the above copyright
15*df114daeSRuslan Bukin * notice, this list of conditions and the following disclaimer in the
16*df114daeSRuslan Bukin * documentation and/or other materials provided with the distribution.
17*df114daeSRuslan Bukin *
18*df114daeSRuslan Bukin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19*df114daeSRuslan Bukin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20*df114daeSRuslan Bukin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21*df114daeSRuslan Bukin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22*df114daeSRuslan Bukin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23*df114daeSRuslan Bukin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24*df114daeSRuslan Bukin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25*df114daeSRuslan Bukin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26*df114daeSRuslan Bukin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27*df114daeSRuslan Bukin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28*df114daeSRuslan Bukin * SUCH DAMAGE.
29*df114daeSRuslan Bukin */
30*df114daeSRuslan Bukin
31*df114daeSRuslan Bukin #include <sys/param.h>
32*df114daeSRuslan Bukin #include <sys/kernel.h>
33*df114daeSRuslan Bukin #include <sys/malloc.h>
34*df114daeSRuslan Bukin #include <sys/lock.h>
35*df114daeSRuslan Bukin #include <sys/mutex.h>
36*df114daeSRuslan Bukin #include <sys/refcount.h>
37*df114daeSRuslan Bukin #include <sys/rwlock.h>
38*df114daeSRuslan Bukin #include <sys/hwt.h>
39*df114daeSRuslan Bukin
40*df114daeSRuslan Bukin #include <vm/vm.h>
41*df114daeSRuslan Bukin #include <vm/vm_param.h>
42*df114daeSRuslan Bukin #include <vm/vm_kern.h>
43*df114daeSRuslan Bukin #include <vm/vm_page.h>
44*df114daeSRuslan Bukin #include <vm/vm_object.h>
45*df114daeSRuslan Bukin #include <vm/vm_pager.h>
46*df114daeSRuslan Bukin #include <vm/vm_pageout.h>
47*df114daeSRuslan Bukin #include <vm/vm_phys.h>
48*df114daeSRuslan Bukin
49*df114daeSRuslan Bukin #include <dev/hwt/hwt_hook.h>
50*df114daeSRuslan Bukin #include <dev/hwt/hwt_context.h>
51*df114daeSRuslan Bukin #include <dev/hwt/hwt_contexthash.h>
52*df114daeSRuslan Bukin #include <dev/hwt/hwt_config.h>
53*df114daeSRuslan Bukin #include <dev/hwt/hwt_thread.h>
54*df114daeSRuslan Bukin #include <dev/hwt/hwt_owner.h>
55*df114daeSRuslan Bukin #include <dev/hwt/hwt_ownerhash.h>
56*df114daeSRuslan Bukin #include <dev/hwt/hwt_backend.h>
57*df114daeSRuslan Bukin #include <dev/hwt/hwt_vm.h>
58*df114daeSRuslan Bukin #include <dev/hwt/hwt_record.h>
59*df114daeSRuslan Bukin
60*df114daeSRuslan Bukin #define HWT_THREAD_DEBUG
61*df114daeSRuslan Bukin #undef HWT_THREAD_DEBUG
62*df114daeSRuslan Bukin
63*df114daeSRuslan Bukin #ifdef HWT_THREAD_DEBUG
64*df114daeSRuslan Bukin #define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__)
65*df114daeSRuslan Bukin #else
66*df114daeSRuslan Bukin #define dprintf(fmt, ...)
67*df114daeSRuslan Bukin #endif
68*df114daeSRuslan Bukin
69*df114daeSRuslan Bukin static MALLOC_DEFINE(M_HWT_THREAD, "hwt_thread", "Hardware Trace");
70*df114daeSRuslan Bukin
71*df114daeSRuslan Bukin struct hwt_thread *
hwt_thread_first(struct hwt_context * ctx)72*df114daeSRuslan Bukin hwt_thread_first(struct hwt_context *ctx)
73*df114daeSRuslan Bukin {
74*df114daeSRuslan Bukin struct hwt_thread *thr;
75*df114daeSRuslan Bukin
76*df114daeSRuslan Bukin HWT_CTX_ASSERT_LOCKED(ctx);
77*df114daeSRuslan Bukin
78*df114daeSRuslan Bukin thr = TAILQ_FIRST(&ctx->threads);
79*df114daeSRuslan Bukin
80*df114daeSRuslan Bukin KASSERT(thr != NULL, ("thr is NULL"));
81*df114daeSRuslan Bukin
82*df114daeSRuslan Bukin return (thr);
83*df114daeSRuslan Bukin }
84*df114daeSRuslan Bukin
85*df114daeSRuslan Bukin /*
86*df114daeSRuslan Bukin * To use by hwt_switch_in/out() only.
87*df114daeSRuslan Bukin */
88*df114daeSRuslan Bukin struct hwt_thread *
hwt_thread_lookup(struct hwt_context * ctx,struct thread * td)89*df114daeSRuslan Bukin hwt_thread_lookup(struct hwt_context *ctx, struct thread *td)
90*df114daeSRuslan Bukin {
91*df114daeSRuslan Bukin struct hwt_thread *thr;
92*df114daeSRuslan Bukin
93*df114daeSRuslan Bukin /* Caller of this func holds ctx refcnt right here. */
94*df114daeSRuslan Bukin
95*df114daeSRuslan Bukin HWT_CTX_LOCK(ctx);
96*df114daeSRuslan Bukin TAILQ_FOREACH(thr, &ctx->threads, next) {
97*df114daeSRuslan Bukin if (thr->td == td) {
98*df114daeSRuslan Bukin HWT_CTX_UNLOCK(ctx);
99*df114daeSRuslan Bukin return (thr);
100*df114daeSRuslan Bukin }
101*df114daeSRuslan Bukin }
102*df114daeSRuslan Bukin HWT_CTX_UNLOCK(ctx);
103*df114daeSRuslan Bukin
104*df114daeSRuslan Bukin /*
105*df114daeSRuslan Bukin * We are here because the hook on thread creation failed to allocate
106*df114daeSRuslan Bukin * a thread.
107*df114daeSRuslan Bukin */
108*df114daeSRuslan Bukin
109*df114daeSRuslan Bukin return (NULL);
110*df114daeSRuslan Bukin }
111*df114daeSRuslan Bukin
112*df114daeSRuslan Bukin int
hwt_thread_alloc(struct hwt_thread ** thr0,char * path,size_t bufsize,int kva_req)113*df114daeSRuslan Bukin hwt_thread_alloc(struct hwt_thread **thr0, char *path, size_t bufsize,
114*df114daeSRuslan Bukin int kva_req)
115*df114daeSRuslan Bukin {
116*df114daeSRuslan Bukin struct hwt_thread *thr;
117*df114daeSRuslan Bukin struct hwt_vm *vm;
118*df114daeSRuslan Bukin int error;
119*df114daeSRuslan Bukin
120*df114daeSRuslan Bukin error = hwt_vm_alloc(bufsize, kva_req, path, &vm);
121*df114daeSRuslan Bukin if (error)
122*df114daeSRuslan Bukin return (error);
123*df114daeSRuslan Bukin
124*df114daeSRuslan Bukin thr = malloc(sizeof(struct hwt_thread), M_HWT_THREAD,
125*df114daeSRuslan Bukin M_WAITOK | M_ZERO);
126*df114daeSRuslan Bukin thr->vm = vm;
127*df114daeSRuslan Bukin
128*df114daeSRuslan Bukin mtx_init(&thr->mtx, "thr", NULL, MTX_DEF);
129*df114daeSRuslan Bukin
130*df114daeSRuslan Bukin refcount_init(&thr->refcnt, 1);
131*df114daeSRuslan Bukin
132*df114daeSRuslan Bukin vm->thr = thr;
133*df114daeSRuslan Bukin
134*df114daeSRuslan Bukin *thr0 = thr;
135*df114daeSRuslan Bukin
136*df114daeSRuslan Bukin return (0);
137*df114daeSRuslan Bukin }
138*df114daeSRuslan Bukin
139*df114daeSRuslan Bukin void
hwt_thread_free(struct hwt_thread * thr)140*df114daeSRuslan Bukin hwt_thread_free(struct hwt_thread *thr)
141*df114daeSRuslan Bukin {
142*df114daeSRuslan Bukin
143*df114daeSRuslan Bukin hwt_vm_free(thr->vm);
144*df114daeSRuslan Bukin /* Free private backend data, if any. */
145*df114daeSRuslan Bukin if (thr->private != NULL)
146*df114daeSRuslan Bukin hwt_backend_thread_free(thr);
147*df114daeSRuslan Bukin free(thr, M_HWT_THREAD);
148*df114daeSRuslan Bukin }
149*df114daeSRuslan Bukin
150*df114daeSRuslan Bukin /*
151*df114daeSRuslan Bukin * Inserts a new thread and a thread creation record into the
152*df114daeSRuslan Bukin * context notifies userspace about the newly created thread.
153*df114daeSRuslan Bukin */
154*df114daeSRuslan Bukin void
hwt_thread_insert(struct hwt_context * ctx,struct hwt_thread * thr,struct hwt_record_entry * entry)155*df114daeSRuslan Bukin hwt_thread_insert(struct hwt_context *ctx, struct hwt_thread *thr,
156*df114daeSRuslan Bukin struct hwt_record_entry *entry)
157*df114daeSRuslan Bukin {
158*df114daeSRuslan Bukin
159*df114daeSRuslan Bukin HWT_CTX_ASSERT_LOCKED(ctx);
160*df114daeSRuslan Bukin TAILQ_INSERT_TAIL(&ctx->threads, thr, next);
161*df114daeSRuslan Bukin TAILQ_INSERT_TAIL(&ctx->records, entry, next);
162*df114daeSRuslan Bukin }
163