1 /*- 2 * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. Neither the name of the author nor the names of any co-contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /* 31 * This module holds the global variables and functions used to maintain 32 * lock_object structures. 33 */ 34 35 #include <sys/cdefs.h> 36 __FBSDID("$FreeBSD$"); 37 38 #include "opt_ddb.h" 39 #include "opt_mprof.h" 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/ktr.h> 44 #include <sys/linker_set.h> 45 #include <sys/lock.h> 46 #include <sys/sbuf.h> 47 #include <sys/sysctl.h> 48 #include <sys/lock_profile.h> 49 50 #ifdef DDB 51 #include <ddb/ddb.h> 52 #endif 53 54 CTASSERT(LOCK_CLASS_MAX == 15); 55 56 struct lock_class *lock_classes[LOCK_CLASS_MAX + 1] = { 57 &lock_class_mtx_spin, 58 &lock_class_mtx_sleep, 59 &lock_class_sx, 60 &lock_class_rm, 61 &lock_class_rw, 62 &lock_class_lockmgr, 63 }; 64 65 #ifdef LOCK_PROFILING 66 #include <machine/cpufunc.h> 67 68 SYSCTL_NODE(_debug, OID_AUTO, lock, CTLFLAG_RD, NULL, "lock debugging"); 69 SYSCTL_NODE(_debug_lock, OID_AUTO, prof, CTLFLAG_RD, NULL, "lock profiling"); 70 int lock_prof_enable = 0; 71 SYSCTL_INT(_debug_lock_prof, OID_AUTO, enable, CTLFLAG_RW, 72 &lock_prof_enable, 0, "Enable lock profiling"); 73 74 /* 75 * lprof_buf is a static pool of profiling records to avoid possible 76 * reentrance of the memory allocation functions. 77 * 78 * Note: NUM_LPROF_BUFFERS must be smaller than LPROF_HASH_SIZE. 79 */ 80 struct lock_prof lprof_buf[LPROF_HASH_SIZE]; 81 static int allocated_lprof_buf; 82 struct mtx lprof_locks[LPROF_LOCK_SIZE]; 83 84 85 /* SWAG: sbuf size = avg stat. line size * number of locks */ 86 #define LPROF_SBUF_SIZE 256 * 400 87 88 static int lock_prof_acquisitions; 89 SYSCTL_INT(_debug_lock_prof, OID_AUTO, acquisitions, CTLFLAG_RD, 90 &lock_prof_acquisitions, 0, "Number of lock acquistions recorded"); 91 static int lock_prof_records; 92 SYSCTL_INT(_debug_lock_prof, OID_AUTO, records, CTLFLAG_RD, 93 &lock_prof_records, 0, "Number of profiling records"); 94 static int lock_prof_maxrecords = LPROF_HASH_SIZE; 95 SYSCTL_INT(_debug_lock_prof, OID_AUTO, maxrecords, CTLFLAG_RD, 96 &lock_prof_maxrecords, 0, "Maximum number of profiling records"); 97 static int lock_prof_rejected; 98 SYSCTL_INT(_debug_lock_prof, OID_AUTO, rejected, CTLFLAG_RD, 99 &lock_prof_rejected, 0, "Number of rejected profiling records"); 100 static int lock_prof_hashsize = LPROF_HASH_SIZE; 101 SYSCTL_INT(_debug_lock_prof, OID_AUTO, hashsize, CTLFLAG_RD, 102 &lock_prof_hashsize, 0, "Hash size"); 103 static int lock_prof_collisions = 0; 104 SYSCTL_INT(_debug_lock_prof, OID_AUTO, collisions, CTLFLAG_RD, 105 &lock_prof_collisions, 0, "Number of hash collisions"); 106 107 #ifndef USE_CPU_NANOSECONDS 108 u_int64_t 109 nanoseconds(void) 110 { 111 struct timespec tv; 112 113 nanotime(&tv); 114 return (tv.tv_sec * (u_int64_t)1000000000 + tv.tv_nsec); 115 } 116 #endif 117 118 static int 119 dump_lock_prof_stats(SYSCTL_HANDLER_ARGS) 120 { 121 struct sbuf *sb; 122 int error, i; 123 static int multiplier = 1; 124 const char *p; 125 126 if (allocated_lprof_buf == 0) 127 return (SYSCTL_OUT(req, "No locking recorded", 128 sizeof("No locking recorded"))); 129 130 retry_sbufops: 131 sb = sbuf_new(NULL, NULL, LPROF_SBUF_SIZE * multiplier, SBUF_FIXEDLEN); 132 sbuf_printf(sb, "\n%6s %12s %12s %11s %5s %5s %12s %12s %s\n", 133 "max", "total", "wait_total", "count", "avg", "wait_avg", "cnt_hold", "cnt_lock", "name"); 134 for (i = 0; i < LPROF_HASH_SIZE; ++i) { 135 if (lprof_buf[i].name == NULL) 136 continue; 137 for (p = lprof_buf[i].file; 138 p != NULL && strncmp(p, "../", 3) == 0; p += 3) 139 /* nothing */ ; 140 sbuf_printf(sb, "%6ju %12ju %12ju %11ju %5ju %5ju %12ju %12ju %s:%d (%s:%s)\n", 141 lprof_buf[i].cnt_max / 1000, 142 lprof_buf[i].cnt_tot / 1000, 143 lprof_buf[i].cnt_wait / 1000, 144 lprof_buf[i].cnt_cur, 145 lprof_buf[i].cnt_cur == 0 ? (uintmax_t)0 : 146 lprof_buf[i].cnt_tot / (lprof_buf[i].cnt_cur * 1000), 147 lprof_buf[i].cnt_cur == 0 ? (uintmax_t)0 : 148 lprof_buf[i].cnt_wait / (lprof_buf[i].cnt_cur * 1000), 149 lprof_buf[i].cnt_contest_holding, 150 lprof_buf[i].cnt_contest_locking, 151 p, lprof_buf[i].line, 152 lprof_buf[i].type, 153 lprof_buf[i].name); 154 if (sbuf_overflowed(sb)) { 155 sbuf_delete(sb); 156 multiplier++; 157 goto retry_sbufops; 158 } 159 } 160 161 sbuf_finish(sb); 162 error = SYSCTL_OUT(req, sbuf_data(sb), sbuf_len(sb) + 1); 163 sbuf_delete(sb); 164 return (error); 165 } 166 static int 167 reset_lock_prof_stats(SYSCTL_HANDLER_ARGS) 168 { 169 int error, v; 170 171 if (allocated_lprof_buf == 0) 172 return (0); 173 174 v = 0; 175 error = sysctl_handle_int(oidp, &v, 0, req); 176 if (error) 177 return (error); 178 if (req->newptr == NULL) 179 return (error); 180 if (v == 0) 181 return (0); 182 183 bzero(lprof_buf, LPROF_HASH_SIZE*sizeof(*lprof_buf)); 184 allocated_lprof_buf = 0; 185 return (0); 186 } 187 188 SYSCTL_PROC(_debug_lock_prof, OID_AUTO, stats, CTLTYPE_STRING | CTLFLAG_RD, 189 NULL, 0, dump_lock_prof_stats, "A", "Lock profiling statistics"); 190 191 SYSCTL_PROC(_debug_lock_prof, OID_AUTO, reset, CTLTYPE_INT | CTLFLAG_RW, 192 NULL, 0, reset_lock_prof_stats, "I", "Reset lock profiling statistics"); 193 #endif 194 195 void 196 lock_init(struct lock_object *lock, struct lock_class *class, const char *name, 197 const char *type, int flags) 198 { 199 int i; 200 201 /* Check for double-init and zero object. */ 202 KASSERT(!lock_initalized(lock), ("lock \"%s\" %p already initialized", 203 name, lock)); 204 205 /* Look up lock class to find its index. */ 206 for (i = 0; i < LOCK_CLASS_MAX; i++) 207 if (lock_classes[i] == class) { 208 lock->lo_flags = i << LO_CLASSSHIFT; 209 break; 210 } 211 KASSERT(i < LOCK_CLASS_MAX, ("unknown lock class %p", class)); 212 213 /* Initialize the lock object. */ 214 lock->lo_name = name; 215 lock->lo_type = type != NULL ? type : name; 216 lock->lo_flags |= flags | LO_INITIALIZED; 217 LOCK_LOG_INIT(lock, 0); 218 WITNESS_INIT(lock); 219 lock_profile_object_init(lock, class, name); 220 } 221 222 void 223 lock_destroy(struct lock_object *lock) 224 { 225 226 KASSERT(lock_initalized(lock), ("lock %p is not initialized", lock)); 227 lock_profile_object_destroy(lock); 228 WITNESS_DESTROY(lock); 229 LOCK_LOG_DESTROY(lock, 0); 230 lock->lo_flags &= ~LO_INITIALIZED; 231 } 232 233 #ifdef DDB 234 DB_SHOW_COMMAND(lock, db_show_lock) 235 { 236 struct lock_object *lock; 237 struct lock_class *class; 238 239 if (!have_addr) 240 return; 241 lock = (struct lock_object *)addr; 242 if (LO_CLASSINDEX(lock) > LOCK_CLASS_MAX) { 243 db_printf("Unknown lock class: %d\n", LO_CLASSINDEX(lock)); 244 return; 245 } 246 class = LOCK_CLASS(lock); 247 db_printf(" class: %s\n", class->lc_name); 248 db_printf(" name: %s\n", lock->lo_name); 249 if (lock->lo_type && lock->lo_type != lock->lo_name) 250 db_printf(" type: %s\n", lock->lo_type); 251 class->lc_ddb_show(lock); 252 } 253 #endif 254 255 #ifdef LOCK_PROFILING 256 void _lock_profile_obtain_lock_success(struct lock_object *lo, int contested, uint64_t waittime, const char *file, int line) 257 { 258 struct lock_profile_object *l = &lo->lo_profile_obj; 259 260 lo->lo_profile_obj.lpo_contest_holding = 0; 261 262 if (contested) 263 lo->lo_profile_obj.lpo_contest_locking++; 264 265 l->lpo_filename = file; 266 l->lpo_lineno = line; 267 l->lpo_acqtime = nanoseconds(); 268 if (waittime && (l->lpo_acqtime > waittime)) 269 l->lpo_waittime = l->lpo_acqtime - waittime; 270 else 271 l->lpo_waittime = 0; 272 } 273 274 void _lock_profile_release_lock(struct lock_object *lo) 275 { 276 struct lock_profile_object *l = &lo->lo_profile_obj; 277 278 if (l->lpo_acqtime) { 279 const char *unknown = "(unknown)"; 280 u_int64_t acqtime, now, waittime; 281 struct lock_prof *mpp; 282 u_int hash; 283 const char *p = l->lpo_filename; 284 int collision = 0; 285 286 now = nanoseconds(); 287 acqtime = l->lpo_acqtime; 288 waittime = l->lpo_waittime; 289 if (now <= acqtime) 290 return; 291 if (p == NULL || *p == '\0') 292 p = unknown; 293 hash = (l->lpo_namehash * 31 * 31 + (uintptr_t)p * 31 + l->lpo_lineno) & LPROF_HASH_MASK; 294 mpp = &lprof_buf[hash]; 295 while (mpp->name != NULL) { 296 if (mpp->line == l->lpo_lineno && 297 mpp->file == p && 298 mpp->namehash == l->lpo_namehash) 299 break; 300 /* If the lprof_hash entry is allocated to someone 301 * else, try the next one 302 */ 303 collision = 1; 304 hash = (hash + 1) & LPROF_HASH_MASK; 305 mpp = &lprof_buf[hash]; 306 } 307 if (mpp->name == NULL) { 308 int buf; 309 310 buf = atomic_fetchadd_int(&allocated_lprof_buf, 1); 311 /* Just exit if we cannot get a trace buffer */ 312 if (buf >= LPROF_HASH_SIZE) { 313 ++lock_prof_rejected; 314 return; 315 } 316 mpp->file = p; 317 mpp->line = l->lpo_lineno; 318 mpp->namehash = l->lpo_namehash; 319 mpp->type = l->lpo_type; 320 mpp->name = lo->lo_name; 321 322 if (collision) 323 ++lock_prof_collisions; 324 325 /* 326 * We might have raced someone else but who cares, 327 * they'll try again next time 328 */ 329 ++lock_prof_records; 330 } 331 LPROF_LOCK(hash); 332 /* 333 * Record if the lock has been held longer now than ever 334 * before. 335 */ 336 if (now - acqtime > mpp->cnt_max) 337 mpp->cnt_max = now - acqtime; 338 mpp->cnt_tot += now - acqtime; 339 mpp->cnt_wait += waittime; 340 mpp->cnt_cur++; 341 /* 342 * There's a small race, really we should cmpxchg 343 * 0 with the current value, but that would bill 344 * the contention to the wrong lock instance if 345 * it followed this also. 346 */ 347 mpp->cnt_contest_holding += l->lpo_contest_holding; 348 mpp->cnt_contest_locking += l->lpo_contest_locking; 349 LPROF_UNLOCK(hash); 350 351 } 352 l->lpo_acqtime = 0; 353 l->lpo_waittime = 0; 354 l->lpo_contest_locking = 0; 355 l->lpo_contest_holding = 0; 356 } 357 #endif 358