xref: /freebsd/sys/kern/subr_lock.c (revision 0c66dc6758f234077f7b55898dcdc81c25a4a2ac)
183a81bcbSJohn Baldwin /*-
283a81bcbSJohn Baldwin  * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org>
383a81bcbSJohn Baldwin  * All rights reserved.
483a81bcbSJohn Baldwin  *
583a81bcbSJohn Baldwin  * Redistribution and use in source and binary forms, with or without
683a81bcbSJohn Baldwin  * modification, are permitted provided that the following conditions
783a81bcbSJohn Baldwin  * are met:
883a81bcbSJohn Baldwin  * 1. Redistributions of source code must retain the above copyright
983a81bcbSJohn Baldwin  *    notice, this list of conditions and the following disclaimer.
1083a81bcbSJohn Baldwin  * 2. Redistributions in binary form must reproduce the above copyright
1183a81bcbSJohn Baldwin  *    notice, this list of conditions and the following disclaimer in the
1283a81bcbSJohn Baldwin  *    documentation and/or other materials provided with the distribution.
1383a81bcbSJohn Baldwin  * 3. Neither the name of the author nor the names of any co-contributors
1483a81bcbSJohn Baldwin  *    may be used to endorse or promote products derived from this software
1583a81bcbSJohn Baldwin  *    without specific prior written permission.
1683a81bcbSJohn Baldwin  *
1783a81bcbSJohn Baldwin  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
1883a81bcbSJohn Baldwin  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
1983a81bcbSJohn Baldwin  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
2083a81bcbSJohn Baldwin  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
2183a81bcbSJohn Baldwin  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
2283a81bcbSJohn Baldwin  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
2383a81bcbSJohn Baldwin  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
2483a81bcbSJohn Baldwin  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
2583a81bcbSJohn Baldwin  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
2683a81bcbSJohn Baldwin  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
2783a81bcbSJohn Baldwin  * SUCH DAMAGE.
2883a81bcbSJohn Baldwin  */
2983a81bcbSJohn Baldwin 
3083a81bcbSJohn Baldwin /*
3183a81bcbSJohn Baldwin  * This module holds the global variables and functions used to maintain
3283a81bcbSJohn Baldwin  * lock_object structures.
3383a81bcbSJohn Baldwin  */
3483a81bcbSJohn Baldwin 
3583a81bcbSJohn Baldwin #include <sys/cdefs.h>
3683a81bcbSJohn Baldwin __FBSDID("$FreeBSD$");
3783a81bcbSJohn Baldwin 
386ef970a9SJohn Baldwin #include "opt_ddb.h"
397c0435b9SKip Macy #include "opt_mprof.h"
406ef970a9SJohn Baldwin 
4183a81bcbSJohn Baldwin #include <sys/param.h>
4283a81bcbSJohn Baldwin #include <sys/systm.h>
43eea4f254SJeff Roberson #include <sys/kernel.h>
4483a81bcbSJohn Baldwin #include <sys/ktr.h>
456ef970a9SJohn Baldwin #include <sys/linker_set.h>
4683a81bcbSJohn Baldwin #include <sys/lock.h>
477c0435b9SKip Macy #include <sys/lock_profile.h>
48eea4f254SJeff Roberson #include <sys/malloc.h>
49eea4f254SJeff Roberson #include <sys/pcpu.h>
50eea4f254SJeff Roberson #include <sys/proc.h>
51eea4f254SJeff Roberson #include <sys/sbuf.h>
52eea4f254SJeff Roberson #include <sys/smp.h>
53eea4f254SJeff Roberson #include <sys/sysctl.h>
5483a81bcbSJohn Baldwin 
5583a81bcbSJohn Baldwin #ifdef DDB
5683a81bcbSJohn Baldwin #include <ddb/ddb.h>
5783a81bcbSJohn Baldwin #endif
5883a81bcbSJohn Baldwin 
59eea4f254SJeff Roberson #include <machine/cpufunc.h>
60eea4f254SJeff Roberson 
6183a81bcbSJohn Baldwin CTASSERT(LOCK_CLASS_MAX == 15);
6283a81bcbSJohn Baldwin 
6383a81bcbSJohn Baldwin struct lock_class *lock_classes[LOCK_CLASS_MAX + 1] = {
6483a81bcbSJohn Baldwin 	&lock_class_mtx_spin,
6583a81bcbSJohn Baldwin 	&lock_class_mtx_sleep,
6683a81bcbSJohn Baldwin 	&lock_class_sx,
67f53d15feSStephan Uphoff 	&lock_class_rm,
683f08bd8bSJohn Baldwin 	&lock_class_rw,
6961bd5e21SKip Macy 	&lock_class_lockmgr,
7083a81bcbSJohn Baldwin };
7183a81bcbSJohn Baldwin 
7283a81bcbSJohn Baldwin void
7383a81bcbSJohn Baldwin lock_init(struct lock_object *lock, struct lock_class *class, const char *name,
7483a81bcbSJohn Baldwin     const char *type, int flags)
7583a81bcbSJohn Baldwin {
7683a81bcbSJohn Baldwin 	int i;
7783a81bcbSJohn Baldwin 
7883a81bcbSJohn Baldwin 	/* Check for double-init and zero object. */
7983a81bcbSJohn Baldwin 	KASSERT(!lock_initalized(lock), ("lock \"%s\" %p already initialized",
8083a81bcbSJohn Baldwin 	    name, lock));
8183a81bcbSJohn Baldwin 
8283a81bcbSJohn Baldwin 	/* Look up lock class to find its index. */
8383a81bcbSJohn Baldwin 	for (i = 0; i < LOCK_CLASS_MAX; i++)
8483a81bcbSJohn Baldwin 		if (lock_classes[i] == class) {
8583a81bcbSJohn Baldwin 			lock->lo_flags = i << LO_CLASSSHIFT;
8683a81bcbSJohn Baldwin 			break;
8783a81bcbSJohn Baldwin 		}
8883a81bcbSJohn Baldwin 	KASSERT(i < LOCK_CLASS_MAX, ("unknown lock class %p", class));
8983a81bcbSJohn Baldwin 
9083a81bcbSJohn Baldwin 	/* Initialize the lock object. */
9183a81bcbSJohn Baldwin 	lock->lo_name = name;
9244a96b46SKip Macy 	lock->lo_type = type != NULL ? type : name;
9383a81bcbSJohn Baldwin 	lock->lo_flags |= flags | LO_INITIALIZED;
9483a81bcbSJohn Baldwin 	LOCK_LOG_INIT(lock, 0);
9583a81bcbSJohn Baldwin 	WITNESS_INIT(lock);
9683a81bcbSJohn Baldwin }
9783a81bcbSJohn Baldwin 
9883a81bcbSJohn Baldwin void
9983a81bcbSJohn Baldwin lock_destroy(struct lock_object *lock)
10083a81bcbSJohn Baldwin {
10183a81bcbSJohn Baldwin 
10283a81bcbSJohn Baldwin 	KASSERT(lock_initalized(lock), ("lock %p is not initialized", lock));
10383a81bcbSJohn Baldwin 	WITNESS_DESTROY(lock);
10483a81bcbSJohn Baldwin 	LOCK_LOG_DESTROY(lock, 0);
10583a81bcbSJohn Baldwin 	lock->lo_flags &= ~LO_INITIALIZED;
10683a81bcbSJohn Baldwin }
10783a81bcbSJohn Baldwin 
10883a81bcbSJohn Baldwin #ifdef DDB
10983a81bcbSJohn Baldwin DB_SHOW_COMMAND(lock, db_show_lock)
11083a81bcbSJohn Baldwin {
11183a81bcbSJohn Baldwin 	struct lock_object *lock;
11283a81bcbSJohn Baldwin 	struct lock_class *class;
11383a81bcbSJohn Baldwin 
11483a81bcbSJohn Baldwin 	if (!have_addr)
11583a81bcbSJohn Baldwin 		return;
11683a81bcbSJohn Baldwin 	lock = (struct lock_object *)addr;
11783a81bcbSJohn Baldwin 	if (LO_CLASSINDEX(lock) > LOCK_CLASS_MAX) {
11883a81bcbSJohn Baldwin 		db_printf("Unknown lock class: %d\n", LO_CLASSINDEX(lock));
11983a81bcbSJohn Baldwin 		return;
12083a81bcbSJohn Baldwin 	}
12183a81bcbSJohn Baldwin 	class = LOCK_CLASS(lock);
12283a81bcbSJohn Baldwin 	db_printf(" class: %s\n", class->lc_name);
12383a81bcbSJohn Baldwin 	db_printf(" name: %s\n", lock->lo_name);
12483a81bcbSJohn Baldwin 	if (lock->lo_type && lock->lo_type != lock->lo_name)
12583a81bcbSJohn Baldwin 		db_printf(" type: %s\n", lock->lo_type);
12683a81bcbSJohn Baldwin 	class->lc_ddb_show(lock);
12783a81bcbSJohn Baldwin }
12883a81bcbSJohn Baldwin #endif
1297c0435b9SKip Macy 
1307c0435b9SKip Macy #ifdef LOCK_PROFILING
131eea4f254SJeff Roberson 
132eea4f254SJeff Roberson /*
133eea4f254SJeff Roberson  * One object per-thread for each lock the thread owns.  Tracks individual
134eea4f254SJeff Roberson  * lock instances.
135eea4f254SJeff Roberson  */
136eea4f254SJeff Roberson struct lock_profile_object {
137eea4f254SJeff Roberson 	LIST_ENTRY(lock_profile_object) lpo_link;
138eea4f254SJeff Roberson 	struct lock_object *lpo_obj;
139eea4f254SJeff Roberson 	const char	*lpo_file;
140eea4f254SJeff Roberson 	int		lpo_line;
141eea4f254SJeff Roberson 	uint16_t	lpo_ref;
142eea4f254SJeff Roberson 	uint16_t	lpo_cnt;
143eea4f254SJeff Roberson 	u_int64_t	lpo_acqtime;
144eea4f254SJeff Roberson 	u_int64_t	lpo_waittime;
145eea4f254SJeff Roberson 	u_int		lpo_contest_locking;
146eea4f254SJeff Roberson };
147eea4f254SJeff Roberson 
148eea4f254SJeff Roberson /*
149eea4f254SJeff Roberson  * One lock_prof for each (file, line, lock object) triple.
150eea4f254SJeff Roberson  */
151eea4f254SJeff Roberson struct lock_prof {
152eea4f254SJeff Roberson 	SLIST_ENTRY(lock_prof) link;
1530c66dc67SJeff Roberson 	struct lock_class *class;
154eea4f254SJeff Roberson 	const char	*file;
155eea4f254SJeff Roberson 	const char	*name;
156eea4f254SJeff Roberson 	int		line;
157eea4f254SJeff Roberson 	int		ticks;
158eea4f254SJeff Roberson 	uintmax_t	cnt_max;
159eea4f254SJeff Roberson 	uintmax_t	cnt_tot;
160eea4f254SJeff Roberson 	uintmax_t	cnt_wait;
161eea4f254SJeff Roberson 	uintmax_t	cnt_cur;
162eea4f254SJeff Roberson 	uintmax_t	cnt_contest_locking;
163eea4f254SJeff Roberson };
164eea4f254SJeff Roberson 
165eea4f254SJeff Roberson SLIST_HEAD(lphead, lock_prof);
166eea4f254SJeff Roberson 
167eea4f254SJeff Roberson #define	LPROF_HASH_SIZE		4096
168eea4f254SJeff Roberson #define	LPROF_HASH_MASK		(LPROF_HASH_SIZE - 1)
169eea4f254SJeff Roberson #define	LPROF_CACHE_SIZE	4096
170eea4f254SJeff Roberson 
171eea4f254SJeff Roberson /*
172eea4f254SJeff Roberson  * Array of objects and profs for each type of object for each cpu.  Spinlocks
173eea4f254SJeff Roberson  * are handled seperately because a thread may be preempted and acquire a
174eea4f254SJeff Roberson  * spinlock while in the lock profiling code of a non-spinlock.  In this way
175eea4f254SJeff Roberson  * we only need a critical section to protect the per-cpu lists.
176eea4f254SJeff Roberson  */
177eea4f254SJeff Roberson struct lock_prof_type {
178eea4f254SJeff Roberson 	struct lphead		lpt_lpalloc;
179eea4f254SJeff Roberson 	struct lpohead		lpt_lpoalloc;
180eea4f254SJeff Roberson 	struct lphead		lpt_hash[LPROF_HASH_SIZE];
181eea4f254SJeff Roberson 	struct lock_prof	lpt_prof[LPROF_CACHE_SIZE];
182eea4f254SJeff Roberson 	struct lock_profile_object lpt_objs[LPROF_CACHE_SIZE];
183eea4f254SJeff Roberson };
184eea4f254SJeff Roberson 
185eea4f254SJeff Roberson struct lock_prof_cpu {
186eea4f254SJeff Roberson 	struct lock_prof_type	lpc_types[2]; /* One for spin one for other. */
187eea4f254SJeff Roberson };
188eea4f254SJeff Roberson 
189eea4f254SJeff Roberson struct lock_prof_cpu *lp_cpu[MAXCPU];
190eea4f254SJeff Roberson 
191eea4f254SJeff Roberson int lock_prof_enable = 0;
192eea4f254SJeff Roberson 
193eea4f254SJeff Roberson /* SWAG: sbuf size = avg stat. line size * number of locks */
194eea4f254SJeff Roberson #define LPROF_SBUF_SIZE		256 * 400
195eea4f254SJeff Roberson 
196eea4f254SJeff Roberson static int lock_prof_rejected;
197eea4f254SJeff Roberson static int lock_prof_skipspin;
198eea4f254SJeff Roberson static int lock_prof_skipcount;
199eea4f254SJeff Roberson 
200eea4f254SJeff Roberson #ifndef USE_CPU_NANOSECONDS
201eea4f254SJeff Roberson u_int64_t
202eea4f254SJeff Roberson nanoseconds(void)
2037c0435b9SKip Macy {
204eea4f254SJeff Roberson 	struct bintime bt;
205eea4f254SJeff Roberson 	u_int64_t ns;
2067c0435b9SKip Macy 
207eea4f254SJeff Roberson 	binuptime(&bt);
208eea4f254SJeff Roberson 	/* From bintime2timespec */
209eea4f254SJeff Roberson 	ns = bt.sec * (u_int64_t)1000000000;
210eea4f254SJeff Roberson 	ns += ((uint64_t)1000000000 * (uint32_t)(bt.frac >> 32)) >> 32;
211eea4f254SJeff Roberson 	return (ns);
212eea4f254SJeff Roberson }
213eea4f254SJeff Roberson #endif
214fe68a916SKip Macy 
215eea4f254SJeff Roberson static void
216eea4f254SJeff Roberson lock_prof_init_type(struct lock_prof_type *type)
217eea4f254SJeff Roberson {
218eea4f254SJeff Roberson 	int i;
219fe68a916SKip Macy 
220eea4f254SJeff Roberson 	SLIST_INIT(&type->lpt_lpalloc);
221eea4f254SJeff Roberson 	LIST_INIT(&type->lpt_lpoalloc);
222eea4f254SJeff Roberson 	for (i = 0; i < LPROF_CACHE_SIZE; i++) {
223eea4f254SJeff Roberson 		SLIST_INSERT_HEAD(&type->lpt_lpalloc, &type->lpt_prof[i],
224eea4f254SJeff Roberson 		    link);
225eea4f254SJeff Roberson 		LIST_INSERT_HEAD(&type->lpt_lpoalloc, &type->lpt_objs[i],
226eea4f254SJeff Roberson 		    lpo_link);
227eea4f254SJeff Roberson 	}
228eea4f254SJeff Roberson }
229eea4f254SJeff Roberson 
230eea4f254SJeff Roberson static void
231eea4f254SJeff Roberson lock_prof_init(void *arg)
232eea4f254SJeff Roberson {
233eea4f254SJeff Roberson 	int cpu;
234eea4f254SJeff Roberson 
235eea4f254SJeff Roberson 	for (cpu = 0; cpu <= mp_maxid; cpu++) {
236eea4f254SJeff Roberson 		lp_cpu[cpu] = malloc(sizeof(*lp_cpu[cpu]), M_DEVBUF,
237eea4f254SJeff Roberson 		    M_WAITOK | M_ZERO);
238eea4f254SJeff Roberson 		lock_prof_init_type(&lp_cpu[cpu]->lpc_types[0]);
239eea4f254SJeff Roberson 		lock_prof_init_type(&lp_cpu[cpu]->lpc_types[1]);
240eea4f254SJeff Roberson 	}
241eea4f254SJeff Roberson }
242eea4f254SJeff Roberson SYSINIT(lockprof, SI_SUB_SMP, SI_ORDER_ANY, lock_prof_init, NULL);
243eea4f254SJeff Roberson 
244eea4f254SJeff Roberson static void
245eea4f254SJeff Roberson lock_prof_reset(void)
246eea4f254SJeff Roberson {
247eea4f254SJeff Roberson 	struct lock_prof_cpu *lpc;
248eea4f254SJeff Roberson 	int enabled, i, cpu;
249eea4f254SJeff Roberson 
250eea4f254SJeff Roberson 	enabled = lock_prof_enable;
251eea4f254SJeff Roberson 	lock_prof_enable = 0;
2520c66dc67SJeff Roberson 	pause("lpreset", hz / 10);
253eea4f254SJeff Roberson 	for (cpu = 0; cpu <= mp_maxid; cpu++) {
254eea4f254SJeff Roberson 		lpc = lp_cpu[cpu];
255eea4f254SJeff Roberson 		for (i = 0; i < LPROF_CACHE_SIZE; i++) {
256eea4f254SJeff Roberson 			LIST_REMOVE(&lpc->lpc_types[0].lpt_objs[i], lpo_link);
257eea4f254SJeff Roberson 			LIST_REMOVE(&lpc->lpc_types[1].lpt_objs[i], lpo_link);
258eea4f254SJeff Roberson 		}
259eea4f254SJeff Roberson 		bzero(lpc, sizeof(*lpc));
260eea4f254SJeff Roberson 		lock_prof_init_type(&lpc->lpc_types[0]);
261eea4f254SJeff Roberson 		lock_prof_init_type(&lpc->lpc_types[1]);
262eea4f254SJeff Roberson 	}
263eea4f254SJeff Roberson 	lock_prof_enable = enabled;
264eea4f254SJeff Roberson }
265eea4f254SJeff Roberson 
266eea4f254SJeff Roberson static void
267eea4f254SJeff Roberson lock_prof_output(struct lock_prof *lp, struct sbuf *sb)
268eea4f254SJeff Roberson {
269eea4f254SJeff Roberson 	const char *p;
270eea4f254SJeff Roberson 
271eea4f254SJeff Roberson 	for (p = lp->file; p != NULL && strncmp(p, "../", 3) == 0; p += 3);
272eea4f254SJeff Roberson 	sbuf_printf(sb,
273eea4f254SJeff Roberson 	    "%6ju %12ju %12ju %11ju %5ju %5ju %12ju %12ju %s:%d (%s:%s)\n",
274eea4f254SJeff Roberson 	    lp->cnt_max / 1000, lp->cnt_tot / 1000,
275eea4f254SJeff Roberson 	    lp->cnt_wait / 1000, lp->cnt_cur,
276eea4f254SJeff Roberson 	    lp->cnt_cur == 0 ? (uintmax_t)0 :
277eea4f254SJeff Roberson 	    lp->cnt_tot / (lp->cnt_cur * 1000),
278eea4f254SJeff Roberson 	    lp->cnt_cur == 0 ? (uintmax_t)0 :
279eea4f254SJeff Roberson 	    lp->cnt_wait / (lp->cnt_cur * 1000),
280eea4f254SJeff Roberson 	    (uintmax_t)0, lp->cnt_contest_locking,
2810c66dc67SJeff Roberson 	    p, lp->line, lp->class->lc_name, lp->name);
282eea4f254SJeff Roberson }
283eea4f254SJeff Roberson 
284eea4f254SJeff Roberson static void
285eea4f254SJeff Roberson lock_prof_sum(struct lock_prof *match, struct lock_prof *dst, int hash,
286eea4f254SJeff Roberson     int spin, int t)
287eea4f254SJeff Roberson {
288eea4f254SJeff Roberson 	struct lock_prof_type *type;
289eea4f254SJeff Roberson 	struct lock_prof *l;
290eea4f254SJeff Roberson 	int cpu;
291eea4f254SJeff Roberson 
292eea4f254SJeff Roberson 	dst->file = match->file;
293eea4f254SJeff Roberson 	dst->line = match->line;
2940c66dc67SJeff Roberson 	dst->class = match->class;
295eea4f254SJeff Roberson 	dst->name = match->name;
296eea4f254SJeff Roberson 
297eea4f254SJeff Roberson 	for (cpu = 0; cpu <= mp_maxid; cpu++) {
298eea4f254SJeff Roberson 		if (lp_cpu[cpu] == NULL)
299eea4f254SJeff Roberson 			continue;
300eea4f254SJeff Roberson 		type = &lp_cpu[cpu]->lpc_types[spin];
301eea4f254SJeff Roberson 		SLIST_FOREACH(l, &type->lpt_hash[hash], link) {
302eea4f254SJeff Roberson 			if (l->ticks == t)
303eea4f254SJeff Roberson 				continue;
304eea4f254SJeff Roberson 			if (l->file != match->file || l->line != match->line ||
3050c66dc67SJeff Roberson 			    l->name != match->name)
306eea4f254SJeff Roberson 				continue;
307eea4f254SJeff Roberson 			l->ticks = t;
308eea4f254SJeff Roberson 			if (l->cnt_max > dst->cnt_max)
309eea4f254SJeff Roberson 				dst->cnt_max = l->cnt_max;
310eea4f254SJeff Roberson 			dst->cnt_tot += l->cnt_tot;
311eea4f254SJeff Roberson 			dst->cnt_wait += l->cnt_wait;
312eea4f254SJeff Roberson 			dst->cnt_cur += l->cnt_cur;
313eea4f254SJeff Roberson 			dst->cnt_contest_locking += l->cnt_contest_locking;
314eea4f254SJeff Roberson 		}
315eea4f254SJeff Roberson 	}
316eea4f254SJeff Roberson 
317eea4f254SJeff Roberson }
318eea4f254SJeff Roberson 
319eea4f254SJeff Roberson static void
320eea4f254SJeff Roberson lock_prof_type_stats(struct lock_prof_type *type, struct sbuf *sb, int spin,
321eea4f254SJeff Roberson     int t)
322eea4f254SJeff Roberson {
323eea4f254SJeff Roberson 	struct lock_prof *l;
324eea4f254SJeff Roberson 	int i;
325eea4f254SJeff Roberson 
326eea4f254SJeff Roberson 	for (i = 0; i < LPROF_HASH_SIZE; ++i) {
327eea4f254SJeff Roberson 		SLIST_FOREACH(l, &type->lpt_hash[i], link) {
328eea4f254SJeff Roberson 			struct lock_prof lp = {};
329eea4f254SJeff Roberson 
330eea4f254SJeff Roberson 			if (l->ticks == t)
331eea4f254SJeff Roberson 				continue;
332eea4f254SJeff Roberson 			lock_prof_sum(l, &lp, i, spin, t);
333eea4f254SJeff Roberson 			lock_prof_output(&lp, sb);
334eea4f254SJeff Roberson 			if (sbuf_overflowed(sb))
335eea4f254SJeff Roberson 				return;
336eea4f254SJeff Roberson 		}
337eea4f254SJeff Roberson 	}
338eea4f254SJeff Roberson }
339eea4f254SJeff Roberson 
340eea4f254SJeff Roberson static int
341eea4f254SJeff Roberson dump_lock_prof_stats(SYSCTL_HANDLER_ARGS)
342eea4f254SJeff Roberson {
343eea4f254SJeff Roberson 	static int multiplier = 1;
344eea4f254SJeff Roberson 	struct sbuf *sb;
345eea4f254SJeff Roberson 	int error, cpu, t;
3460c66dc67SJeff Roberson 	int enabled;
347eea4f254SJeff Roberson 
348eea4f254SJeff Roberson retry_sbufops:
349eea4f254SJeff Roberson 	sb = sbuf_new(NULL, NULL, LPROF_SBUF_SIZE * multiplier, SBUF_FIXEDLEN);
350eea4f254SJeff Roberson 	sbuf_printf(sb, "\n%6s %12s %12s %11s %5s %5s %12s %12s %s\n",
351eea4f254SJeff Roberson 	    "max", "total", "wait_total", "count", "avg", "wait_avg", "cnt_hold", "cnt_lock", "name");
3520c66dc67SJeff Roberson 	enabled = lock_prof_enable;
3530c66dc67SJeff Roberson 	lock_prof_enable = 0;
3540c66dc67SJeff Roberson 	pause("lpreset", hz / 10);
355eea4f254SJeff Roberson 	t = ticks;
356eea4f254SJeff Roberson 	for (cpu = 0; cpu <= mp_maxid; cpu++) {
357eea4f254SJeff Roberson 		if (lp_cpu[cpu] == NULL)
358eea4f254SJeff Roberson 			continue;
359eea4f254SJeff Roberson 		lock_prof_type_stats(&lp_cpu[cpu]->lpc_types[0], sb, 0, t);
360eea4f254SJeff Roberson 		lock_prof_type_stats(&lp_cpu[cpu]->lpc_types[1], sb, 1, t);
361eea4f254SJeff Roberson 		if (sbuf_overflowed(sb)) {
362eea4f254SJeff Roberson 			sbuf_delete(sb);
363eea4f254SJeff Roberson 			multiplier++;
364eea4f254SJeff Roberson 			goto retry_sbufops;
365eea4f254SJeff Roberson 		}
366eea4f254SJeff Roberson 	}
3670c66dc67SJeff Roberson 	lock_prof_enable = enabled;
368eea4f254SJeff Roberson 
369eea4f254SJeff Roberson 	sbuf_finish(sb);
370eea4f254SJeff Roberson 	error = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1);
371eea4f254SJeff Roberson 	sbuf_delete(sb);
372eea4f254SJeff Roberson 	return (error);
373eea4f254SJeff Roberson }
374eea4f254SJeff Roberson 
375eea4f254SJeff Roberson static int
376eea4f254SJeff Roberson enable_lock_prof(SYSCTL_HANDLER_ARGS)
377eea4f254SJeff Roberson {
378eea4f254SJeff Roberson 	int error, v;
379eea4f254SJeff Roberson 
380eea4f254SJeff Roberson 	v = lock_prof_enable;
381eea4f254SJeff Roberson 	error = sysctl_handle_int(oidp, &v, v, req);
382eea4f254SJeff Roberson 	if (error)
383eea4f254SJeff Roberson 		return (error);
384eea4f254SJeff Roberson 	if (req->newptr == NULL)
385eea4f254SJeff Roberson 		return (error);
386eea4f254SJeff Roberson 	if (v == lock_prof_enable)
387eea4f254SJeff Roberson 		return (0);
388eea4f254SJeff Roberson 	if (v == 1)
389eea4f254SJeff Roberson 		lock_prof_reset();
390eea4f254SJeff Roberson 	lock_prof_enable = !!v;
391eea4f254SJeff Roberson 
392eea4f254SJeff Roberson 	return (0);
393eea4f254SJeff Roberson }
394eea4f254SJeff Roberson 
395eea4f254SJeff Roberson static int
396eea4f254SJeff Roberson reset_lock_prof_stats(SYSCTL_HANDLER_ARGS)
397eea4f254SJeff Roberson {
398eea4f254SJeff Roberson 	int error, v;
399eea4f254SJeff Roberson 
400eea4f254SJeff Roberson 	v = 0;
401eea4f254SJeff Roberson 	error = sysctl_handle_int(oidp, &v, 0, req);
402eea4f254SJeff Roberson 	if (error)
403eea4f254SJeff Roberson 		return (error);
404eea4f254SJeff Roberson 	if (req->newptr == NULL)
405eea4f254SJeff Roberson 		return (error);
406eea4f254SJeff Roberson 	if (v == 0)
407eea4f254SJeff Roberson 		return (0);
408eea4f254SJeff Roberson 	lock_prof_reset();
409eea4f254SJeff Roberson 
410eea4f254SJeff Roberson 	return (0);
411eea4f254SJeff Roberson }
412eea4f254SJeff Roberson 
413eea4f254SJeff Roberson static struct lock_prof *
414eea4f254SJeff Roberson lock_profile_lookup(struct lock_object *lo, int spin, const char *file,
415eea4f254SJeff Roberson     int line)
416eea4f254SJeff Roberson {
417eea4f254SJeff Roberson 	const char *unknown = "(unknown)";
418eea4f254SJeff Roberson 	struct lock_prof_type *type;
419eea4f254SJeff Roberson 	struct lock_prof *lp;
420eea4f254SJeff Roberson 	struct lphead *head;
421eea4f254SJeff Roberson 	const char *p;
422eea4f254SJeff Roberson 	u_int hash;
423eea4f254SJeff Roberson 
424eea4f254SJeff Roberson 	p = file;
425eea4f254SJeff Roberson 	if (p == NULL || *p == '\0')
426eea4f254SJeff Roberson 		p = unknown;
427eea4f254SJeff Roberson 	hash = (uintptr_t)lo->lo_name * 31 + (uintptr_t)p * 31 + line;
428eea4f254SJeff Roberson 	hash &= LPROF_HASH_MASK;
429eea4f254SJeff Roberson 	type = &lp_cpu[PCPU_GET(cpuid)]->lpc_types[spin];
430eea4f254SJeff Roberson 	head = &type->lpt_hash[hash];
431eea4f254SJeff Roberson 	SLIST_FOREACH(lp, head, link) {
432eea4f254SJeff Roberson 		if (lp->line == line && lp->file == p &&
433eea4f254SJeff Roberson 		    lp->name == lo->lo_name)
434eea4f254SJeff Roberson 			return (lp);
435eea4f254SJeff Roberson 
436eea4f254SJeff Roberson 	}
437eea4f254SJeff Roberson 	lp = SLIST_FIRST(&type->lpt_lpalloc);
438eea4f254SJeff Roberson 	if (lp == NULL) {
439eea4f254SJeff Roberson 		lock_prof_rejected++;
440eea4f254SJeff Roberson 		return (lp);
441eea4f254SJeff Roberson 	}
442eea4f254SJeff Roberson 	SLIST_REMOVE_HEAD(&type->lpt_lpalloc, link);
443eea4f254SJeff Roberson 	lp->file = p;
444eea4f254SJeff Roberson 	lp->line = line;
4450c66dc67SJeff Roberson 	lp->class = LOCK_CLASS(lo);
446eea4f254SJeff Roberson 	lp->name = lo->lo_name;
447eea4f254SJeff Roberson 	SLIST_INSERT_HEAD(&type->lpt_hash[hash], lp, link);
448eea4f254SJeff Roberson 	return (lp);
449eea4f254SJeff Roberson }
450eea4f254SJeff Roberson 
451eea4f254SJeff Roberson static struct lock_profile_object *
452eea4f254SJeff Roberson lock_profile_object_lookup(struct lock_object *lo, int spin, const char *file,
453eea4f254SJeff Roberson     int line)
454eea4f254SJeff Roberson {
455eea4f254SJeff Roberson 	struct lock_profile_object *l;
456eea4f254SJeff Roberson 	struct lock_prof_type *type;
457eea4f254SJeff Roberson 	struct lpohead *head;
458eea4f254SJeff Roberson 
459eea4f254SJeff Roberson 	head = &curthread->td_lprof[spin];
460eea4f254SJeff Roberson 	LIST_FOREACH(l, head, lpo_link)
461eea4f254SJeff Roberson 		if (l->lpo_obj == lo && l->lpo_file == file &&
462eea4f254SJeff Roberson 		    l->lpo_line == line)
463eea4f254SJeff Roberson 			return (l);
464eea4f254SJeff Roberson 	critical_enter();
465eea4f254SJeff Roberson 	type = &lp_cpu[PCPU_GET(cpuid)]->lpc_types[spin];
466eea4f254SJeff Roberson 	l = LIST_FIRST(&type->lpt_lpoalloc);
467eea4f254SJeff Roberson 	if (l == NULL) {
468eea4f254SJeff Roberson 		lock_prof_rejected++;
469eea4f254SJeff Roberson 		critical_exit();
470eea4f254SJeff Roberson 		return (NULL);
471eea4f254SJeff Roberson 	}
472eea4f254SJeff Roberson 	LIST_REMOVE(l, lpo_link);
473eea4f254SJeff Roberson 	critical_exit();
474eea4f254SJeff Roberson 	l->lpo_obj = lo;
475eea4f254SJeff Roberson 	l->lpo_file = file;
476eea4f254SJeff Roberson 	l->lpo_line = line;
477eea4f254SJeff Roberson 	l->lpo_cnt = 0;
478eea4f254SJeff Roberson 	LIST_INSERT_HEAD(head, l, lpo_link);
479eea4f254SJeff Roberson 
480eea4f254SJeff Roberson 	return (l);
481eea4f254SJeff Roberson }
482eea4f254SJeff Roberson 
483eea4f254SJeff Roberson void
484eea4f254SJeff Roberson lock_profile_obtain_lock_success(struct lock_object *lo, int contested,
485eea4f254SJeff Roberson     uint64_t waittime, const char *file, int line)
486eea4f254SJeff Roberson {
487eea4f254SJeff Roberson 	static int lock_prof_count;
488eea4f254SJeff Roberson 	struct lock_profile_object *l;
489eea4f254SJeff Roberson 	int spin;
490eea4f254SJeff Roberson 
491eea4f254SJeff Roberson 	/* don't reset the timer when/if recursing */
492eea4f254SJeff Roberson 	if (!lock_prof_enable || (lo->lo_flags & LO_NOPROFILE))
493eea4f254SJeff Roberson 		return;
494eea4f254SJeff Roberson 	if (lock_prof_skipcount &&
495eea4f254SJeff Roberson 	    (++lock_prof_count % lock_prof_skipcount) == 0)
496eea4f254SJeff Roberson 		return;
497eea4f254SJeff Roberson 	spin = LOCK_CLASS(lo) == &lock_class_mtx_spin;
498eea4f254SJeff Roberson 	if (spin && lock_prof_skipspin == 1)
499eea4f254SJeff Roberson 		return;
500eea4f254SJeff Roberson 	l = lock_profile_object_lookup(lo, spin, file, line);
501eea4f254SJeff Roberson 	if (l == NULL)
502eea4f254SJeff Roberson 		return;
503eea4f254SJeff Roberson 	l->lpo_cnt++;
504eea4f254SJeff Roberson 	if (++l->lpo_ref > 1)
505eea4f254SJeff Roberson 		return;
506eea4f254SJeff Roberson 	l->lpo_contest_locking = contested;
5077c0435b9SKip Macy 	l->lpo_acqtime = nanoseconds();
508aa077979SKip Macy 	if (waittime && (l->lpo_acqtime > waittime))
5097c0435b9SKip Macy 		l->lpo_waittime = l->lpo_acqtime - waittime;
510aa077979SKip Macy 	else
511aa077979SKip Macy 		l->lpo_waittime = 0;
5127c0435b9SKip Macy }
5137c0435b9SKip Macy 
514eea4f254SJeff Roberson void
515eea4f254SJeff Roberson lock_profile_release_lock(struct lock_object *lo)
5167c0435b9SKip Macy {
517eea4f254SJeff Roberson 	struct lock_profile_object *l;
518eea4f254SJeff Roberson 	struct lock_prof_type *type;
519eea4f254SJeff Roberson 	struct lock_prof *lp;
520eea4f254SJeff Roberson 	u_int64_t holdtime;
521eea4f254SJeff Roberson 	struct lpohead *head;
522eea4f254SJeff Roberson 	int spin;
5237c0435b9SKip Macy 
524eea4f254SJeff Roberson 	if (!lock_prof_enable || (lo->lo_flags & LO_NOPROFILE))
5257c0435b9SKip Macy 		return;
526eea4f254SJeff Roberson 	spin = LOCK_CLASS(lo) == &lock_class_mtx_spin;
527eea4f254SJeff Roberson 	head = &curthread->td_lprof[spin];
528eea4f254SJeff Roberson 	critical_enter();
529eea4f254SJeff Roberson 	LIST_FOREACH(l, head, lpo_link)
530eea4f254SJeff Roberson 		if (l->lpo_obj == lo)
5317c0435b9SKip Macy 			break;
532eea4f254SJeff Roberson 	if (l == NULL)
533eea4f254SJeff Roberson 		goto out;
534eea4f254SJeff Roberson 	if (--l->lpo_ref > 0)
535eea4f254SJeff Roberson 		goto out;
536eea4f254SJeff Roberson 	lp = lock_profile_lookup(lo, spin, l->lpo_file, l->lpo_line);
537eea4f254SJeff Roberson 	if (lp == NULL)
538eea4f254SJeff Roberson 		goto release;
539eea4f254SJeff Roberson 	holdtime = nanoseconds() - l->lpo_acqtime;
540eea4f254SJeff Roberson 	if (holdtime < 0)
541eea4f254SJeff Roberson 		goto release;
5427c0435b9SKip Macy 	/*
54383b72e3eSKip Macy 	 * Record if the lock has been held longer now than ever
5447c0435b9SKip Macy 	 * before.
5457c0435b9SKip Macy 	 */
546eea4f254SJeff Roberson 	if (holdtime > lp->cnt_max)
547eea4f254SJeff Roberson 		lp->cnt_max = holdtime;
548eea4f254SJeff Roberson 	lp->cnt_tot += holdtime;
549eea4f254SJeff Roberson 	lp->cnt_wait += l->lpo_waittime;
550eea4f254SJeff Roberson 	lp->cnt_contest_locking += l->lpo_contest_locking;
551eea4f254SJeff Roberson 	lp->cnt_cur += l->lpo_cnt;
552eea4f254SJeff Roberson release:
553eea4f254SJeff Roberson 	LIST_REMOVE(l, lpo_link);
554eea4f254SJeff Roberson 	type = &lp_cpu[PCPU_GET(cpuid)]->lpc_types[spin];
555eea4f254SJeff Roberson 	LIST_INSERT_HEAD(&type->lpt_lpoalloc, l, lpo_link);
556eea4f254SJeff Roberson out:
557eea4f254SJeff Roberson 	critical_exit();
558eea4f254SJeff Roberson }
5597c0435b9SKip Macy 
560eea4f254SJeff Roberson SYSCTL_NODE(_debug, OID_AUTO, lock, CTLFLAG_RD, NULL, "lock debugging");
561eea4f254SJeff Roberson SYSCTL_NODE(_debug_lock, OID_AUTO, prof, CTLFLAG_RD, NULL, "lock profiling");
562eea4f254SJeff Roberson SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipspin, CTLFLAG_RW,
563eea4f254SJeff Roberson     &lock_prof_skipspin, 0, "Skip profiling on spinlocks.");
564eea4f254SJeff Roberson SYSCTL_INT(_debug_lock_prof, OID_AUTO, skipcount, CTLFLAG_RW,
565eea4f254SJeff Roberson     &lock_prof_skipcount, 0, "Sample approximately every N lock acquisitions.");
566eea4f254SJeff Roberson SYSCTL_INT(_debug_lock_prof, OID_AUTO, rejected, CTLFLAG_RD,
567eea4f254SJeff Roberson     &lock_prof_rejected, 0, "Number of rejected profiling records");
568eea4f254SJeff Roberson SYSCTL_PROC(_debug_lock_prof, OID_AUTO, stats, CTLTYPE_STRING | CTLFLAG_RD,
569eea4f254SJeff Roberson     NULL, 0, dump_lock_prof_stats, "A", "Lock profiling statistics");
570eea4f254SJeff Roberson SYSCTL_PROC(_debug_lock_prof, OID_AUTO, reset, CTLTYPE_INT | CTLFLAG_RW,
571eea4f254SJeff Roberson     NULL, 0, reset_lock_prof_stats, "I", "Reset lock profiling statistics");
572eea4f254SJeff Roberson SYSCTL_PROC(_debug_lock_prof, OID_AUTO, enable, CTLTYPE_INT | CTLFLAG_RW,
573eea4f254SJeff Roberson     NULL, 0, enable_lock_prof, "I", "Enable lock profiling");
574eea4f254SJeff Roberson 
5757c0435b9SKip Macy #endif
576