xref: /freebsd/sys/kern/kern_pmc.c (revision e4f6a1bfa31a2299612bbf9dae402bb38d38b1df)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2003-2008 Joseph Koshy
5  * Copyright (c) 2007 The FreeBSD Foundation
6  * All rights reserved.
7  *
8  * Portions of this software were developed by A. Joseph Koshy under
9  * sponsorship from the FreeBSD Foundation and Google, Inc.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30  * SUCH DAMAGE.
31  */
32 
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
35 
36 #include "opt_hwpmc_hooks.h"
37 
38 #include <sys/types.h>
39 #include <sys/ctype.h>
40 #include <sys/param.h>
41 #include <sys/malloc.h>
42 #include <sys/kernel.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/pmc.h>
46 #include <sys/pmckern.h>
47 #include <sys/smp.h>
48 #include <sys/sysctl.h>
49 #include <sys/systm.h>
50 
51 #ifdef	HWPMC_HOOKS
52 FEATURE(hwpmc_hooks, "Kernel support for HW PMC");
53 #define	PMC_KERNEL_VERSION	PMC_VERSION
54 #else
55 #define	PMC_KERNEL_VERSION	0
56 #endif
57 
58 MALLOC_DECLARE(M_PMCHOOKS);
59 MALLOC_DEFINE(M_PMCHOOKS, "pmchooks", "Memory space for PMC hooks");
60 
61 const int pmc_kernel_version = PMC_KERNEL_VERSION;
62 
63 /* Hook variable. */
64 int __read_mostly (*pmc_hook)(struct thread *td, int function, void *arg) = NULL;
65 
66 /* Interrupt handler */
67 int __read_mostly (*pmc_intr)(int cpu, struct trapframe *tf) = NULL;
68 
69 /* Bitmask of CPUs requiring servicing at hardclock time */
70 volatile cpuset_t pmc_cpumask;
71 
72 /*
73  * A global count of SS mode PMCs.  When non-zero, this means that
74  * we have processes that are sampling the system as a whole.
75  */
76 volatile int pmc_ss_count;
77 
78 /*
79  * Since PMC(4) may not be loaded in the current kernel, the
80  * convention followed is that a non-NULL value of 'pmc_hook' implies
81  * the presence of this kernel module.
82  *
83  * This requires us to protect 'pmc_hook' with a
84  * shared (sx) lock -- thus making the process of calling into PMC(4)
85  * somewhat more expensive than a simple 'if' check and indirect call.
86  */
87 struct sx pmc_sx;
88 
89 /*
90  * PMC Soft per cpu trapframe.
91  */
92 struct trapframe pmc_tf[MAXCPU];
93 
94 /*
95  * PMC Soft use a global table to store registered events.
96  */
97 
98 SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters");
99 
100 static int pmc_softevents = 16;
101 SYSCTL_INT(_kern_hwpmc, OID_AUTO, softevents, CTLFLAG_RDTUN,
102     &pmc_softevents, 0, "maximum number of soft events");
103 
104 struct mtx pmc_softs_mtx;
105 int pmc_softs_count;
106 struct pmc_soft **pmc_softs;
107 
108 MTX_SYSINIT(pmc_soft_mtx, &pmc_softs_mtx, "pmc-softs", MTX_SPIN);
109 
110 static void
111 pmc_init_sx(void)
112 {
113 	sx_init_flags(&pmc_sx, "pmc-sx", SX_NOWITNESS);
114 }
115 
116 SYSINIT(pmcsx, SI_SUB_LOCK, SI_ORDER_MIDDLE, pmc_init_sx, NULL);
117 
118 /*
119  * Helper functions.
120  */
121 
122 /*
123  * A note on the CPU numbering scheme used by the hwpmc(4) driver.
124  *
125  * CPUs are denoted using numbers in the range 0..[pmc_cpu_max()-1].
126  * CPUs could be numbered "sparsely" in this range; the predicate
127  * `pmc_cpu_is_present()' is used to test whether a given CPU is
128  * physically present.
129  *
130  * Further, a CPU that is physically present may be administratively
131  * disabled or otherwise unavailable for use by hwpmc(4).  The
132  * `pmc_cpu_is_active()' predicate tests for CPU usability.  An
133  * "active" CPU participates in thread scheduling and can field
134  * interrupts raised by PMC hardware.
135  *
136  * On systems with hyperthreaded CPUs, multiple logical CPUs may share
137  * PMC hardware resources.  For such processors one logical CPU is
138  * denoted as the primary owner of the in-CPU PMC resources. The
139  * pmc_cpu_is_primary() predicate is used to distinguish this primary
140  * CPU from the others.
141  */
142 
143 int
144 pmc_cpu_is_active(int cpu)
145 {
146 #ifdef	SMP
147 	return (pmc_cpu_is_present(cpu) &&
148 	    !CPU_ISSET(cpu, &hlt_cpus_mask));
149 #else
150 	return (1);
151 #endif
152 }
153 
154 /* Deprecated. */
155 int
156 pmc_cpu_is_disabled(int cpu)
157 {
158 	return (!pmc_cpu_is_active(cpu));
159 }
160 
161 int
162 pmc_cpu_is_present(int cpu)
163 {
164 #ifdef	SMP
165 	return (!CPU_ABSENT(cpu));
166 #else
167 	return (1);
168 #endif
169 }
170 
171 int
172 pmc_cpu_is_primary(int cpu)
173 {
174 #ifdef	SMP
175 	return (!CPU_ISSET(cpu, &logical_cpus_mask));
176 #else
177 	return (1);
178 #endif
179 }
180 
181 
182 /*
183  * Return the maximum CPU number supported by the system.  The return
184  * value is used for scaling internal data structures and for runtime
185  * checks.
186  */
187 unsigned int
188 pmc_cpu_max(void)
189 {
190 #ifdef	SMP
191 	return (mp_maxid+1);
192 #else
193 	return (1);
194 #endif
195 }
196 
197 #ifdef	INVARIANTS
198 
199 /*
200  * Return the count of CPUs in the `active' state in the system.
201  */
202 int
203 pmc_cpu_max_active(void)
204 {
205 #ifdef	SMP
206 	/*
207 	 * When support for CPU hot-plugging is added to the kernel,
208 	 * this function would change to return the current number
209 	 * of "active" CPUs.
210 	 */
211 	return (mp_ncpus);
212 #else
213 	return (1);
214 #endif
215 }
216 
217 #endif
218 
219 /*
220  * Cleanup event name:
221  * - remove duplicate '_'
222  * - all uppercase
223  */
224 static void
225 pmc_soft_namecleanup(char *name)
226 {
227 	char *p, *q;
228 
229 	p = q = name;
230 
231 	for ( ; *p == '_' ; p++)
232 		;
233 	for ( ; *p ; p++) {
234 		if (*p == '_' && (*(p + 1) == '_' || *(p + 1) == '\0'))
235 			continue;
236 		else
237 			*q++ = toupper(*p);
238 	}
239 	*q = '\0';
240 }
241 
242 void
243 pmc_soft_ev_register(struct pmc_soft *ps)
244 {
245 	static int warned = 0;
246 	int n;
247 
248 	ps->ps_running  = 0;
249 	ps->ps_ev.pm_ev_code = 0; /* invalid */
250 	pmc_soft_namecleanup(ps->ps_ev.pm_ev_name);
251 
252 	mtx_lock_spin(&pmc_softs_mtx);
253 
254 	if (pmc_softs_count >= pmc_softevents) {
255 		/*
256 		 * XXX Reusing events can enter a race condition where
257 		 * new allocated event will be used as an old one.
258 		 */
259 		for (n = 0; n < pmc_softevents; n++)
260 			if (pmc_softs[n] == NULL)
261 				break;
262 		if (n == pmc_softevents) {
263 			mtx_unlock_spin(&pmc_softs_mtx);
264 			if (!warned) {
265 				printf("hwpmc: too many soft events, "
266 				    "increase kern.hwpmc.softevents tunable\n");
267 				warned = 1;
268 			}
269 			return;
270 		}
271 
272 		ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + n;
273 		pmc_softs[n] = ps;
274 	} else {
275 		ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + pmc_softs_count;
276 		pmc_softs[pmc_softs_count++] = ps;
277 	}
278 
279 	mtx_unlock_spin(&pmc_softs_mtx);
280 }
281 
282 void
283 pmc_soft_ev_deregister(struct pmc_soft *ps)
284 {
285 
286 	KASSERT(ps != NULL, ("pmc_soft_deregister: called with NULL"));
287 
288 	mtx_lock_spin(&pmc_softs_mtx);
289 
290 	if (ps->ps_ev.pm_ev_code != 0 &&
291 	    (ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST) < pmc_softevents) {
292 		KASSERT((int)ps->ps_ev.pm_ev_code >= PMC_EV_SOFT_FIRST &&
293 		    (int)ps->ps_ev.pm_ev_code <= PMC_EV_SOFT_LAST,
294 		    ("pmc_soft_deregister: invalid event value"));
295 		pmc_softs[ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST] = NULL;
296 	}
297 
298 	mtx_unlock_spin(&pmc_softs_mtx);
299 }
300 
301 struct pmc_soft *
302 pmc_soft_ev_acquire(enum pmc_event ev)
303 {
304 	struct pmc_soft *ps;
305 
306 	if (ev == 0 || (ev - PMC_EV_SOFT_FIRST) >= pmc_softevents)
307 		return NULL;
308 
309 	KASSERT((int)ev >= PMC_EV_SOFT_FIRST &&
310 	    (int)ev <= PMC_EV_SOFT_LAST,
311 	    ("event out of range"));
312 
313 	mtx_lock_spin(&pmc_softs_mtx);
314 
315 	ps = pmc_softs[ev - PMC_EV_SOFT_FIRST];
316 	if (ps == NULL)
317 		mtx_unlock_spin(&pmc_softs_mtx);
318 
319 	return ps;
320 }
321 
322 void
323 pmc_soft_ev_release(struct pmc_soft *ps)
324 {
325 
326 	mtx_unlock_spin(&pmc_softs_mtx);
327 }
328 
329 /*
330  *  Initialise hwpmc.
331  */
332 static void
333 init_hwpmc(void *dummy __unused)
334 {
335 	if (pmc_softevents <= 0 ||
336 	    pmc_softevents > PMC_EV_DYN_COUNT) {
337 		(void) printf("hwpmc: tunable \"softevents\"=%d out of "
338 		    "range.\n", pmc_softevents);
339 		pmc_softevents = PMC_EV_DYN_COUNT;
340 	}
341 	pmc_softs = malloc(pmc_softevents * sizeof(struct pmc_soft *), M_PMCHOOKS, M_NOWAIT|M_ZERO);
342 	KASSERT(pmc_softs != NULL, ("cannot allocate soft events table"));
343 }
344 
345 SYSINIT(hwpmc, SI_SUB_KDTRACE, SI_ORDER_FIRST, init_hwpmc, NULL);
346 
347