1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2003-2008 Joseph Koshy
5 * Copyright (c) 2007 The FreeBSD Foundation
6 * All rights reserved.
7 *
8 * Portions of this software were developed by A. Joseph Koshy under
9 * sponsorship from the FreeBSD Foundation and Google, Inc.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 #include <sys/cdefs.h>
34 #include "opt_hwpmc_hooks.h"
35
36 #include <sys/param.h>
37 #include <sys/ctype.h>
38 #include <sys/domainset.h>
39 #include <sys/param.h>
40 #include <sys/malloc.h>
41 #include <sys/kernel.h>
42 #include <sys/lock.h>
43 #include <sys/mutex.h>
44 #include <sys/pmc.h>
45 #include <sys/pmckern.h>
46 #include <sys/smp.h>
47 #include <sys/sysctl.h>
48 #include <sys/systm.h>
49
50 #include <vm/vm.h>
51 #include <vm/vm_extern.h>
52 #include <vm/vm_kern.h>
53
54 #ifdef HWPMC_HOOKS
55 FEATURE(hwpmc_hooks, "Kernel support for HW PMC");
56 #define PMC_KERNEL_VERSION PMC_VERSION
57 #else
58 #define PMC_KERNEL_VERSION 0
59 #endif
60
61 MALLOC_DECLARE(M_PMCHOOKS);
62 MALLOC_DEFINE(M_PMCHOOKS, "pmchooks", "Memory space for PMC hooks");
63
64 /* memory pool */
65 MALLOC_DEFINE(M_PMC, "pmc", "Memory space for the PMC module");
66
67 const int pmc_kernel_version = PMC_KERNEL_VERSION;
68
69 /* Hook variable. */
70 int __read_mostly (*pmc_hook)(struct thread *td, int function, void *arg) = NULL;
71
72 /* Interrupt handler */
73 int __read_mostly (*pmc_intr)(struct trapframe *tf) = NULL;
74
75 /* HWT hooks */
76 void __read_mostly (*hwt_hook)(struct thread *td, int func, void *arg) = NULL;
77 int __read_mostly (*hwt_intr)(struct trapframe *tf) = NULL;
78
79 DPCPU_DEFINE(uint8_t, pmc_sampled);
80
81 /*
82 * A global count of SS mode PMCs. When non-zero, this means that
83 * we have processes that are sampling the system as a whole.
84 */
85 volatile int pmc_ss_count;
86
87 /*
88 * Since PMC(4) may not be loaded in the current kernel, the
89 * convention followed is that a non-NULL value of 'pmc_hook' implies
90 * the presence of this kernel module.
91 *
92 * This requires us to protect 'pmc_hook' with a
93 * shared (sx) lock -- thus making the process of calling into PMC(4)
94 * somewhat more expensive than a simple 'if' check and indirect call.
95 */
96 struct sx pmc_sx;
97 SX_SYSINIT(pmcsx, &pmc_sx, "pmc-sx");
98
99 /*
100 * PMC Soft per cpu trapframe.
101 */
102 struct trapframe pmc_tf[MAXCPU];
103
104 /*
105 * Per domain list of buffer headers
106 */
107 __read_mostly struct pmc_domain_buffer_header *pmc_dom_hdrs[MAXMEMDOM];
108
109 /*
110 * PMC Soft use a global table to store registered events.
111 */
112
113 SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
114 "HWPMC parameters");
115
116 static int pmc_softevents = 16;
117 SYSCTL_INT(_kern_hwpmc, OID_AUTO, softevents, CTLFLAG_RDTUN,
118 &pmc_softevents, 0, "maximum number of soft events");
119
120 int pmc_softs_count;
121 struct pmc_soft **pmc_softs;
122
123 struct mtx pmc_softs_mtx;
124 MTX_SYSINIT(pmc_soft_mtx, &pmc_softs_mtx, "pmc-softs", MTX_SPIN);
125
126 /*
127 * Helper functions.
128 */
129
130 /*
131 * A note on the CPU numbering scheme used by the hwpmc(4) driver.
132 *
133 * CPUs are denoted using numbers in the range 0..[pmc_cpu_max()-1].
134 * CPUs could be numbered "sparsely" in this range; the predicate
135 * `pmc_cpu_is_present()' is used to test whether a given CPU is
136 * physically present.
137 *
138 * Further, a CPU that is physically present may be administratively
139 * disabled or otherwise unavailable for use by hwpmc(4). The
140 * `pmc_cpu_is_active()' predicate tests for CPU usability. An
141 * "active" CPU participates in thread scheduling and can field
142 * interrupts raised by PMC hardware.
143 *
144 * On systems with hyperthreaded CPUs, multiple logical CPUs may share
145 * PMC hardware resources. For such processors one logical CPU is
146 * denoted as the primary owner of the in-CPU PMC resources. The
147 * pmc_cpu_is_primary() predicate is used to distinguish this primary
148 * CPU from the others.
149 */
150
151 int
pmc_cpu_is_active(int cpu)152 pmc_cpu_is_active(int cpu)
153 {
154 #ifdef SMP
155 return (pmc_cpu_is_present(cpu));
156 #else
157 return (1);
158 #endif
159 }
160
161 /* Deprecated. */
162 int
pmc_cpu_is_disabled(int cpu)163 pmc_cpu_is_disabled(int cpu)
164 {
165 return (!pmc_cpu_is_active(cpu));
166 }
167
168 int
pmc_cpu_is_present(int cpu)169 pmc_cpu_is_present(int cpu)
170 {
171 #ifdef SMP
172 return (!CPU_ABSENT(cpu));
173 #else
174 return (1);
175 #endif
176 }
177
178 int
pmc_cpu_is_primary(int cpu)179 pmc_cpu_is_primary(int cpu)
180 {
181 #ifdef SMP
182 return (!CPU_ISSET(cpu, &logical_cpus_mask));
183 #else
184 return (1);
185 #endif
186 }
187
188 /*
189 * Return the maximum CPU number supported by the system. The return
190 * value is used for scaling internal data structures and for runtime
191 * checks.
192 */
193 unsigned int
pmc_cpu_max(void)194 pmc_cpu_max(void)
195 {
196 #ifdef SMP
197 return (mp_maxid+1);
198 #else
199 return (1);
200 #endif
201 }
202
203 #ifdef INVARIANTS
204
205 /*
206 * Return the count of CPUs in the `active' state in the system.
207 */
208 int
pmc_cpu_max_active(void)209 pmc_cpu_max_active(void)
210 {
211 #ifdef SMP
212 /*
213 * When support for CPU hot-plugging is added to the kernel,
214 * this function would change to return the current number
215 * of "active" CPUs.
216 */
217 return (mp_ncpus);
218 #else
219 return (1);
220 #endif
221 }
222
223 #endif
224
225 /*
226 * Cleanup event name:
227 * - remove duplicate '_'
228 * - all uppercase
229 */
230 static void
pmc_soft_namecleanup(char * name)231 pmc_soft_namecleanup(char *name)
232 {
233 char *p, *q;
234
235 p = q = name;
236
237 for ( ; *p == '_' ; p++)
238 ;
239 for ( ; *p ; p++) {
240 if (*p == '_' && (*(p + 1) == '_' || *(p + 1) == '\0'))
241 continue;
242 else
243 *q++ = toupper(*p);
244 }
245 *q = '\0';
246 }
247
248 void
pmc_soft_ev_register(struct pmc_soft * ps)249 pmc_soft_ev_register(struct pmc_soft *ps)
250 {
251 static int warned = 0;
252 int n;
253
254 ps->ps_running = 0;
255 ps->ps_ev.pm_ev_code = 0; /* invalid */
256 pmc_soft_namecleanup(ps->ps_ev.pm_ev_name);
257
258 mtx_lock_spin(&pmc_softs_mtx);
259
260 if (pmc_softs_count >= pmc_softevents) {
261 /*
262 * XXX Reusing events can enter a race condition where
263 * new allocated event will be used as an old one.
264 */
265 for (n = 0; n < pmc_softevents; n++)
266 if (pmc_softs[n] == NULL)
267 break;
268 if (n == pmc_softevents) {
269 mtx_unlock_spin(&pmc_softs_mtx);
270 if (!warned) {
271 printf("hwpmc: too many soft events, "
272 "increase kern.hwpmc.softevents tunable\n");
273 warned = 1;
274 }
275 return;
276 }
277
278 ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + n;
279 pmc_softs[n] = ps;
280 } else {
281 ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + pmc_softs_count;
282 pmc_softs[pmc_softs_count++] = ps;
283 }
284
285 mtx_unlock_spin(&pmc_softs_mtx);
286 }
287
288 void
pmc_soft_ev_deregister(struct pmc_soft * ps)289 pmc_soft_ev_deregister(struct pmc_soft *ps)
290 {
291
292 KASSERT(ps != NULL, ("pmc_soft_deregister: called with NULL"));
293
294 mtx_lock_spin(&pmc_softs_mtx);
295
296 if (ps->ps_ev.pm_ev_code != 0 &&
297 (ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST) < pmc_softevents) {
298 KASSERT((int)ps->ps_ev.pm_ev_code >= PMC_EV_SOFT_FIRST &&
299 (int)ps->ps_ev.pm_ev_code <= PMC_EV_SOFT_LAST,
300 ("pmc_soft_deregister: invalid event value"));
301 pmc_softs[ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST] = NULL;
302 }
303
304 mtx_unlock_spin(&pmc_softs_mtx);
305 }
306
307 struct pmc_soft *
pmc_soft_ev_acquire(enum pmc_event ev)308 pmc_soft_ev_acquire(enum pmc_event ev)
309 {
310 struct pmc_soft *ps;
311
312 if (ev == 0 || (ev - PMC_EV_SOFT_FIRST) >= pmc_softevents)
313 return NULL;
314
315 KASSERT((int)ev >= PMC_EV_SOFT_FIRST &&
316 (int)ev <= PMC_EV_SOFT_LAST,
317 ("event out of range"));
318
319 mtx_lock_spin(&pmc_softs_mtx);
320
321 ps = pmc_softs[ev - PMC_EV_SOFT_FIRST];
322 if (ps == NULL)
323 mtx_unlock_spin(&pmc_softs_mtx);
324
325 return ps;
326 }
327
328 void
pmc_soft_ev_release(struct pmc_soft * ps)329 pmc_soft_ev_release(struct pmc_soft *ps)
330 {
331
332 mtx_unlock_spin(&pmc_softs_mtx);
333 }
334
335 /*
336 * Initialise hwpmc.
337 */
338 static void
init_hwpmc(void * dummy __unused)339 init_hwpmc(void *dummy __unused)
340 {
341 int domain, cpu;
342
343 if (pmc_softevents <= 0 ||
344 pmc_softevents > PMC_EV_DYN_COUNT) {
345 (void) printf("hwpmc: tunable \"softevents\"=%d out of "
346 "range.\n", pmc_softevents);
347 pmc_softevents = PMC_EV_DYN_COUNT;
348 }
349 pmc_softs = malloc(pmc_softevents * sizeof(*pmc_softs), M_PMCHOOKS,
350 M_WAITOK | M_ZERO);
351
352 for (domain = 0; domain < vm_ndomains; domain++) {
353 pmc_dom_hdrs[domain] = malloc_domainset(
354 sizeof(struct pmc_domain_buffer_header), M_PMC,
355 DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
356 mtx_init(&pmc_dom_hdrs[domain]->pdbh_mtx, "pmc_bufferlist_mtx", "pmc-leaf", MTX_SPIN);
357 TAILQ_INIT(&pmc_dom_hdrs[domain]->pdbh_head);
358 }
359 CPU_FOREACH(cpu) {
360 domain = pcpu_find(cpu)->pc_domain;
361 KASSERT(pmc_dom_hdrs[domain] != NULL, ("no mem allocated for domain: %d", domain));
362 pmc_dom_hdrs[domain]->pdbh_ncpus++;
363 }
364
365 }
366
367 SYSINIT(hwpmc, SI_SUB_KDTRACE, SI_ORDER_FIRST, init_hwpmc, NULL);
368