1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2003-2008 Joseph Koshy
5 * Copyright (c) 2007 The FreeBSD Foundation
6 * All rights reserved.
7 *
8 * Portions of this software were developed by A. Joseph Koshy under
9 * sponsorship from the FreeBSD Foundation and Google, Inc.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 #include <sys/cdefs.h>
34 #include "opt_hwpmc_hooks.h"
35
36 #include <sys/param.h>
37 #include <sys/ctype.h>
38 #include <sys/domainset.h>
39 #include <sys/param.h>
40 #include <sys/malloc.h>
41 #include <sys/kernel.h>
42 #include <sys/lock.h>
43 #include <sys/mutex.h>
44 #include <sys/pmc.h>
45 #include <sys/pmckern.h>
46 #include <sys/smp.h>
47 #include <sys/sysctl.h>
48 #include <sys/systm.h>
49
50 #include <vm/vm.h>
51 #include <vm/vm_extern.h>
52 #include <vm/vm_kern.h>
53
54 #ifdef HWPMC_HOOKS
55 FEATURE(hwpmc_hooks, "Kernel support for HW PMC");
56 #define PMC_KERNEL_VERSION PMC_VERSION
57 #else
58 #define PMC_KERNEL_VERSION 0
59 #endif
60
61 MALLOC_DECLARE(M_PMCHOOKS);
62 MALLOC_DEFINE(M_PMCHOOKS, "pmchooks", "Memory space for PMC hooks");
63
64 /* memory pool */
65 MALLOC_DEFINE(M_PMC, "pmc", "Memory space for the PMC module");
66
67 const int pmc_kernel_version = PMC_KERNEL_VERSION;
68
69 /* Hook variable. */
70 int __read_mostly (*pmc_hook)(struct thread *td, int function, void *arg) = NULL;
71
72 /* Interrupt handler */
73 int __read_mostly (*pmc_intr)(struct trapframe *tf) = NULL;
74
75 /* HWT hooks */
76 void __read_mostly (*hwt_hook)(struct thread *td, int func, void *arg) = NULL;
77 int __read_mostly (*hwt_intr)(struct trapframe *tf) = NULL;
78
79 DPCPU_DEFINE(uint8_t, pmc_sampled);
80
81 /*
82 * A global count of SS mode PMCs. When non-zero, this means that
83 * we have processes that are sampling the system as a whole.
84 */
85 volatile int pmc_ss_count;
86
87 /*
88 * Since PMC(4) may not be loaded in the current kernel, the
89 * convention followed is that a non-NULL value of 'pmc_hook' implies
90 * the presence of this kernel module.
91 *
92 * This requires us to protect 'pmc_hook' with a
93 * shared (sx) lock -- thus making the process of calling into PMC(4)
94 * somewhat more expensive than a simple 'if' check and indirect call.
95 */
96 struct sx pmc_sx;
97 SX_SYSINIT(pmcsx, &pmc_sx, "pmc-sx");
98
99 /*
100 * PMC Soft per cpu trapframe.
101 */
102 struct trapframe pmc_tf[MAXCPU];
103
104 /*
105 * Per domain list of buffer headers
106 */
107 __read_mostly struct pmc_domain_buffer_header *pmc_dom_hdrs[MAXMEMDOM];
108
109 /*
110 * PMC Soft use a global table to store registered events.
111 */
112
113 SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
114 "HWPMC parameters");
115
116 static int pmc_softevents = 16;
117 SYSCTL_INT(_kern_hwpmc, OID_AUTO, softevents, CTLFLAG_RDTUN,
118 &pmc_softevents, 0, "maximum number of soft events");
119
120 int pmc_softs_count;
121 struct pmc_soft **pmc_softs;
122
123 struct mtx pmc_softs_mtx;
124 MTX_SYSINIT(pmc_soft_mtx, &pmc_softs_mtx, "pmc-softs", MTX_SPIN);
125
126 /*
127 * Helper functions.
128 */
129
130 /*
131 * A note on the CPU numbering scheme used by the hwpmc(4) driver.
132 *
133 * CPUs are denoted using numbers in the range 0..[pmc_cpu_max()-1].
134 * CPUs could be numbered "sparsely" in this range; the predicate
135 * `pmc_cpu_is_present()' is used to test whether a given CPU is
136 * physically present.
137 *
138 * Further, a CPU that is physically present may be administratively
139 * disabled or otherwise unavailable for use by hwpmc(4). The
140 * `pmc_cpu_is_active()' predicate tests for CPU usability. An
141 * "active" CPU participates in thread scheduling and can field
142 * interrupts raised by PMC hardware.
143 *
144 * On systems with hyperthreaded CPUs, multiple logical CPUs may share
145 * PMC hardware resources. For such processors one logical CPU is
146 * denoted as the primary owner of the in-CPU PMC resources. The
147 * pmc_cpu_is_primary() predicate is used to distinguish this primary
148 * CPU from the others.
149 */
150
151 int
pmc_cpu_is_active(int cpu)152 pmc_cpu_is_active(int cpu)
153 {
154 #ifdef SMP
155 return (pmc_cpu_is_present(cpu) &&
156 !CPU_ISSET(cpu, &hlt_cpus_mask));
157 #else
158 return (1);
159 #endif
160 }
161
162 /* Deprecated. */
163 int
pmc_cpu_is_disabled(int cpu)164 pmc_cpu_is_disabled(int cpu)
165 {
166 return (!pmc_cpu_is_active(cpu));
167 }
168
169 int
pmc_cpu_is_present(int cpu)170 pmc_cpu_is_present(int cpu)
171 {
172 #ifdef SMP
173 return (!CPU_ABSENT(cpu));
174 #else
175 return (1);
176 #endif
177 }
178
179 int
pmc_cpu_is_primary(int cpu)180 pmc_cpu_is_primary(int cpu)
181 {
182 #ifdef SMP
183 return (!CPU_ISSET(cpu, &logical_cpus_mask));
184 #else
185 return (1);
186 #endif
187 }
188
189 /*
190 * Return the maximum CPU number supported by the system. The return
191 * value is used for scaling internal data structures and for runtime
192 * checks.
193 */
194 unsigned int
pmc_cpu_max(void)195 pmc_cpu_max(void)
196 {
197 #ifdef SMP
198 return (mp_maxid+1);
199 #else
200 return (1);
201 #endif
202 }
203
204 #ifdef INVARIANTS
205
206 /*
207 * Return the count of CPUs in the `active' state in the system.
208 */
209 int
pmc_cpu_max_active(void)210 pmc_cpu_max_active(void)
211 {
212 #ifdef SMP
213 /*
214 * When support for CPU hot-plugging is added to the kernel,
215 * this function would change to return the current number
216 * of "active" CPUs.
217 */
218 return (mp_ncpus);
219 #else
220 return (1);
221 #endif
222 }
223
224 #endif
225
226 /*
227 * Cleanup event name:
228 * - remove duplicate '_'
229 * - all uppercase
230 */
231 static void
pmc_soft_namecleanup(char * name)232 pmc_soft_namecleanup(char *name)
233 {
234 char *p, *q;
235
236 p = q = name;
237
238 for ( ; *p == '_' ; p++)
239 ;
240 for ( ; *p ; p++) {
241 if (*p == '_' && (*(p + 1) == '_' || *(p + 1) == '\0'))
242 continue;
243 else
244 *q++ = toupper(*p);
245 }
246 *q = '\0';
247 }
248
249 void
pmc_soft_ev_register(struct pmc_soft * ps)250 pmc_soft_ev_register(struct pmc_soft *ps)
251 {
252 static int warned = 0;
253 int n;
254
255 ps->ps_running = 0;
256 ps->ps_ev.pm_ev_code = 0; /* invalid */
257 pmc_soft_namecleanup(ps->ps_ev.pm_ev_name);
258
259 mtx_lock_spin(&pmc_softs_mtx);
260
261 if (pmc_softs_count >= pmc_softevents) {
262 /*
263 * XXX Reusing events can enter a race condition where
264 * new allocated event will be used as an old one.
265 */
266 for (n = 0; n < pmc_softevents; n++)
267 if (pmc_softs[n] == NULL)
268 break;
269 if (n == pmc_softevents) {
270 mtx_unlock_spin(&pmc_softs_mtx);
271 if (!warned) {
272 printf("hwpmc: too many soft events, "
273 "increase kern.hwpmc.softevents tunable\n");
274 warned = 1;
275 }
276 return;
277 }
278
279 ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + n;
280 pmc_softs[n] = ps;
281 } else {
282 ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + pmc_softs_count;
283 pmc_softs[pmc_softs_count++] = ps;
284 }
285
286 mtx_unlock_spin(&pmc_softs_mtx);
287 }
288
289 void
pmc_soft_ev_deregister(struct pmc_soft * ps)290 pmc_soft_ev_deregister(struct pmc_soft *ps)
291 {
292
293 KASSERT(ps != NULL, ("pmc_soft_deregister: called with NULL"));
294
295 mtx_lock_spin(&pmc_softs_mtx);
296
297 if (ps->ps_ev.pm_ev_code != 0 &&
298 (ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST) < pmc_softevents) {
299 KASSERT((int)ps->ps_ev.pm_ev_code >= PMC_EV_SOFT_FIRST &&
300 (int)ps->ps_ev.pm_ev_code <= PMC_EV_SOFT_LAST,
301 ("pmc_soft_deregister: invalid event value"));
302 pmc_softs[ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST] = NULL;
303 }
304
305 mtx_unlock_spin(&pmc_softs_mtx);
306 }
307
308 struct pmc_soft *
pmc_soft_ev_acquire(enum pmc_event ev)309 pmc_soft_ev_acquire(enum pmc_event ev)
310 {
311 struct pmc_soft *ps;
312
313 if (ev == 0 || (ev - PMC_EV_SOFT_FIRST) >= pmc_softevents)
314 return NULL;
315
316 KASSERT((int)ev >= PMC_EV_SOFT_FIRST &&
317 (int)ev <= PMC_EV_SOFT_LAST,
318 ("event out of range"));
319
320 mtx_lock_spin(&pmc_softs_mtx);
321
322 ps = pmc_softs[ev - PMC_EV_SOFT_FIRST];
323 if (ps == NULL)
324 mtx_unlock_spin(&pmc_softs_mtx);
325
326 return ps;
327 }
328
329 void
pmc_soft_ev_release(struct pmc_soft * ps)330 pmc_soft_ev_release(struct pmc_soft *ps)
331 {
332
333 mtx_unlock_spin(&pmc_softs_mtx);
334 }
335
336 /*
337 * Initialise hwpmc.
338 */
339 static void
init_hwpmc(void * dummy __unused)340 init_hwpmc(void *dummy __unused)
341 {
342 int domain, cpu;
343
344 if (pmc_softevents <= 0 ||
345 pmc_softevents > PMC_EV_DYN_COUNT) {
346 (void) printf("hwpmc: tunable \"softevents\"=%d out of "
347 "range.\n", pmc_softevents);
348 pmc_softevents = PMC_EV_DYN_COUNT;
349 }
350 pmc_softs = malloc(pmc_softevents * sizeof(*pmc_softs), M_PMCHOOKS,
351 M_WAITOK | M_ZERO);
352
353 for (domain = 0; domain < vm_ndomains; domain++) {
354 pmc_dom_hdrs[domain] = malloc_domainset(
355 sizeof(struct pmc_domain_buffer_header), M_PMC,
356 DOMAINSET_PREF(domain), M_WAITOK | M_ZERO);
357 mtx_init(&pmc_dom_hdrs[domain]->pdbh_mtx, "pmc_bufferlist_mtx", "pmc-leaf", MTX_SPIN);
358 TAILQ_INIT(&pmc_dom_hdrs[domain]->pdbh_head);
359 }
360 CPU_FOREACH(cpu) {
361 domain = pcpu_find(cpu)->pc_domain;
362 KASSERT(pmc_dom_hdrs[domain] != NULL, ("no mem allocated for domain: %d", domain));
363 pmc_dom_hdrs[domain]->pdbh_ncpus++;
364 }
365
366 }
367
368 SYSINIT(hwpmc, SI_SUB_KDTRACE, SI_ORDER_FIRST, init_hwpmc, NULL);
369