1 /* 2 * Copyright (C) 2001 Jason Evans <jasone@freebsd.org>. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice(s), this list of conditions and the following disclaimer as 9 * the first lines of this file unmodified other than the possible 10 * addition of one or more copyright notices. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice(s), this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY 16 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY 19 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 22 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 25 * DAMAGE. 26 */ 27 28 /* 29 * Shared/exclusive locks. This implementation assures deterministic lock 30 * granting behavior, so that slocks and xlocks are interleaved. 31 * 32 * Priority propagation will not generally raise the priority of lock holders, 33 * so should not be relied upon in combination with sx locks. 34 */ 35 36 #include <sys/cdefs.h> 37 __FBSDID("$FreeBSD$"); 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/ktr.h> 42 #include <sys/condvar.h> 43 #include <sys/lock.h> 44 #include <sys/mutex.h> 45 #include <sys/sx.h> 46 47 struct lock_class lock_class_sx = { 48 "sx", 49 LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE 50 }; 51 52 #ifndef INVARIANTS 53 #define _sx_assert(sx, what, file, line) 54 #endif 55 56 void 57 sx_sysinit(void *arg) 58 { 59 struct sx_args *sargs = arg; 60 61 sx_init(sargs->sa_sx, sargs->sa_desc); 62 } 63 64 void 65 sx_init(struct sx *sx, const char *description) 66 { 67 struct lock_object *lock; 68 69 lock = &sx->sx_object; 70 KASSERT((lock->lo_flags & LO_INITIALIZED) == 0, 71 ("sx lock %s %p already initialized", description, sx)); 72 bzero(sx, sizeof(*sx)); 73 lock->lo_class = &lock_class_sx; 74 lock->lo_type = lock->lo_name = description; 75 lock->lo_flags = LO_WITNESS | LO_RECURSABLE | LO_SLEEPABLE | 76 LO_UPGRADABLE; 77 sx->sx_lock = mtx_pool_find(mtxpool_lockbuilder, sx); 78 sx->sx_cnt = 0; 79 cv_init(&sx->sx_shrd_cv, description); 80 sx->sx_shrd_wcnt = 0; 81 cv_init(&sx->sx_excl_cv, description); 82 sx->sx_excl_wcnt = 0; 83 sx->sx_xholder = NULL; 84 85 LOCK_LOG_INIT(lock, 0); 86 87 WITNESS_INIT(lock); 88 } 89 90 void 91 sx_destroy(struct sx *sx) 92 { 93 94 LOCK_LOG_DESTROY(&sx->sx_object, 0); 95 96 KASSERT((sx->sx_cnt == 0 && sx->sx_shrd_wcnt == 0 && sx->sx_excl_wcnt == 97 0), ("%s (%s): holders or waiters\n", __func__, 98 sx->sx_object.lo_name)); 99 100 sx->sx_lock = NULL; 101 cv_destroy(&sx->sx_shrd_cv); 102 cv_destroy(&sx->sx_excl_cv); 103 104 WITNESS_DESTROY(&sx->sx_object); 105 } 106 107 void 108 _sx_slock(struct sx *sx, const char *file, int line) 109 { 110 111 mtx_lock(sx->sx_lock); 112 KASSERT(sx->sx_xholder != curthread, 113 ("%s (%s): slock while xlock is held @ %s:%d\n", __func__, 114 sx->sx_object.lo_name, file, line)); 115 116 /* 117 * Loop in case we lose the race for lock acquisition. 118 */ 119 while (sx->sx_cnt < 0) { 120 sx->sx_shrd_wcnt++; 121 cv_wait(&sx->sx_shrd_cv, sx->sx_lock); 122 sx->sx_shrd_wcnt--; 123 } 124 125 /* Acquire a shared lock. */ 126 sx->sx_cnt++; 127 128 LOCK_LOG_LOCK("SLOCK", &sx->sx_object, 0, 0, file, line); 129 WITNESS_LOCK(&sx->sx_object, 0, file, line); 130 131 mtx_unlock(sx->sx_lock); 132 } 133 134 int 135 _sx_try_slock(struct sx *sx, const char *file, int line) 136 { 137 138 mtx_lock(sx->sx_lock); 139 if (sx->sx_cnt >= 0) { 140 sx->sx_cnt++; 141 LOCK_LOG_TRY("SLOCK", &sx->sx_object, 0, 1, file, line); 142 WITNESS_LOCK(&sx->sx_object, LOP_TRYLOCK, file, line); 143 mtx_unlock(sx->sx_lock); 144 return (1); 145 } else { 146 LOCK_LOG_TRY("SLOCK", &sx->sx_object, 0, 0, file, line); 147 mtx_unlock(sx->sx_lock); 148 return (0); 149 } 150 } 151 152 void 153 _sx_xlock(struct sx *sx, const char *file, int line) 154 { 155 156 mtx_lock(sx->sx_lock); 157 158 /* 159 * With sx locks, we're absolutely not permitted to recurse on 160 * xlocks, as it is fatal (deadlock). Normally, recursion is handled 161 * by WITNESS, but as it is not semantically correct to hold the 162 * xlock while in here, we consider it API abuse and put it under 163 * INVARIANTS. 164 */ 165 KASSERT(sx->sx_xholder != curthread, 166 ("%s (%s): xlock already held @ %s:%d", __func__, 167 sx->sx_object.lo_name, file, line)); 168 169 /* Loop in case we lose the race for lock acquisition. */ 170 while (sx->sx_cnt != 0) { 171 sx->sx_excl_wcnt++; 172 cv_wait(&sx->sx_excl_cv, sx->sx_lock); 173 sx->sx_excl_wcnt--; 174 } 175 176 MPASS(sx->sx_cnt == 0); 177 178 /* Acquire an exclusive lock. */ 179 sx->sx_cnt--; 180 sx->sx_xholder = curthread; 181 182 LOCK_LOG_LOCK("XLOCK", &sx->sx_object, 0, 0, file, line); 183 WITNESS_LOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line); 184 185 mtx_unlock(sx->sx_lock); 186 } 187 188 int 189 _sx_try_xlock(struct sx *sx, const char *file, int line) 190 { 191 192 mtx_lock(sx->sx_lock); 193 if (sx->sx_cnt == 0) { 194 sx->sx_cnt--; 195 sx->sx_xholder = curthread; 196 LOCK_LOG_TRY("XLOCK", &sx->sx_object, 0, 1, file, line); 197 WITNESS_LOCK(&sx->sx_object, LOP_EXCLUSIVE | LOP_TRYLOCK, file, 198 line); 199 mtx_unlock(sx->sx_lock); 200 return (1); 201 } else { 202 LOCK_LOG_TRY("XLOCK", &sx->sx_object, 0, 0, file, line); 203 mtx_unlock(sx->sx_lock); 204 return (0); 205 } 206 } 207 208 void 209 _sx_sunlock(struct sx *sx, const char *file, int line) 210 { 211 212 _sx_assert(sx, SX_SLOCKED, file, line); 213 mtx_lock(sx->sx_lock); 214 215 WITNESS_UNLOCK(&sx->sx_object, 0, file, line); 216 217 /* Release. */ 218 sx->sx_cnt--; 219 220 /* 221 * If we just released the last shared lock, wake any waiters up, giving 222 * exclusive lockers precedence. In order to make sure that exclusive 223 * lockers won't be blocked forever, don't wake shared lock waiters if 224 * there are exclusive lock waiters. 225 */ 226 if (sx->sx_excl_wcnt > 0) { 227 if (sx->sx_cnt == 0) 228 cv_signal(&sx->sx_excl_cv); 229 } else if (sx->sx_shrd_wcnt > 0) 230 cv_broadcast(&sx->sx_shrd_cv); 231 232 LOCK_LOG_LOCK("SUNLOCK", &sx->sx_object, 0, 0, file, line); 233 234 mtx_unlock(sx->sx_lock); 235 } 236 237 void 238 _sx_xunlock(struct sx *sx, const char *file, int line) 239 { 240 241 _sx_assert(sx, SX_XLOCKED, file, line); 242 mtx_lock(sx->sx_lock); 243 MPASS(sx->sx_cnt == -1); 244 245 WITNESS_UNLOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line); 246 247 /* Release. */ 248 sx->sx_cnt++; 249 sx->sx_xholder = NULL; 250 251 /* 252 * Wake up waiters if there are any. Give precedence to slock waiters. 253 */ 254 if (sx->sx_shrd_wcnt > 0) 255 cv_broadcast(&sx->sx_shrd_cv); 256 else if (sx->sx_excl_wcnt > 0) 257 cv_signal(&sx->sx_excl_cv); 258 259 LOCK_LOG_LOCK("XUNLOCK", &sx->sx_object, 0, 0, file, line); 260 261 mtx_unlock(sx->sx_lock); 262 } 263 264 int 265 _sx_try_upgrade(struct sx *sx, const char *file, int line) 266 { 267 268 _sx_assert(sx, SX_SLOCKED, file, line); 269 mtx_lock(sx->sx_lock); 270 271 if (sx->sx_cnt == 1) { 272 sx->sx_cnt = -1; 273 sx->sx_xholder = curthread; 274 275 LOCK_LOG_TRY("XUPGRADE", &sx->sx_object, 0, 1, file, line); 276 WITNESS_UPGRADE(&sx->sx_object, LOP_EXCLUSIVE | LOP_TRYLOCK, 277 file, line); 278 279 mtx_unlock(sx->sx_lock); 280 return (1); 281 } else { 282 LOCK_LOG_TRY("XUPGRADE", &sx->sx_object, 0, 0, file, line); 283 mtx_unlock(sx->sx_lock); 284 return (0); 285 } 286 } 287 288 void 289 _sx_downgrade(struct sx *sx, const char *file, int line) 290 { 291 292 _sx_assert(sx, SX_XLOCKED, file, line); 293 mtx_lock(sx->sx_lock); 294 MPASS(sx->sx_cnt == -1); 295 296 WITNESS_DOWNGRADE(&sx->sx_object, 0, file, line); 297 298 sx->sx_cnt = 1; 299 sx->sx_xholder = NULL; 300 if (sx->sx_shrd_wcnt > 0) 301 cv_broadcast(&sx->sx_shrd_cv); 302 303 LOCK_LOG_LOCK("XDOWNGRADE", &sx->sx_object, 0, 0, file, line); 304 305 mtx_unlock(sx->sx_lock); 306 } 307 308 #ifdef INVARIANT_SUPPORT 309 #ifndef INVARIANTS 310 #undef _sx_assert 311 #endif 312 313 /* 314 * In the non-WITNESS case, sx_assert() can only detect that at least 315 * *some* thread owns an slock, but it cannot guarantee that *this* 316 * thread owns an slock. 317 */ 318 void 319 _sx_assert(struct sx *sx, int what, const char *file, int line) 320 { 321 322 switch (what) { 323 case SX_LOCKED: 324 case SX_SLOCKED: 325 #ifdef WITNESS 326 witness_assert(&sx->sx_object, what, file, line); 327 #else 328 mtx_lock(sx->sx_lock); 329 if (sx->sx_cnt <= 0 && 330 (what == SX_SLOCKED || sx->sx_xholder != curthread)) 331 printf("Lock %s not %slocked @ %s:%d\n", 332 sx->sx_object.lo_name, (what == SX_SLOCKED) ? 333 "share " : "", file, line); 334 mtx_unlock(sx->sx_lock); 335 #endif 336 break; 337 case SX_XLOCKED: 338 mtx_lock(sx->sx_lock); 339 if (sx->sx_xholder != curthread) 340 printf("Lock %s not exclusively locked @ %s:%d\n", 341 sx->sx_object.lo_name, file, line); 342 mtx_unlock(sx->sx_lock); 343 break; 344 default: 345 panic("Unknown sx lock assertion: %d @ %s:%d", what, file, 346 line); 347 } 348 } 349 #endif /* INVARIANT_SUPPORT */ 350