1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * This file and its contents are supplied under the terms of the 4 * Common Development and Distribution License ("CDDL"), version 1.0. 5 * You may only use this file in accordance with the terms of version 6 * 1.0 of the CDDL. 7 * 8 * A full copy of the text of the CDDL should have accompanied this 9 * source. A copy of the CDDL is also available via the Internet at 10 * https://opensource.org/license/CDDL-1.0. 11 */ 12 /* 13 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 14 * Copyright (c) 2014, 2015 by Delphix. All rights reserved. 15 * Copyright 2016 The MathWorks, Inc. All rights reserved. 16 */ 17 18 /* 19 * A Zero Reference Lock (ZRL) is a reference count that can lock out new 20 * references only when the count is zero and only without waiting if the count 21 * is not already zero. It is similar to a read-write lock in that it allows 22 * multiple readers and only a single writer, but it does not allow a writer to 23 * block while waiting for readers to exit, and therefore the question of 24 * reader/writer priority is moot (no WRWANT bit). Since the equivalent of 25 * rw_enter(&lock, RW_WRITER) is disallowed and only tryenter() is allowed, it 26 * is perfectly safe for the same reader to acquire the same lock multiple 27 * times. The fact that a ZRL is reentrant for readers (through multiple calls 28 * to zrl_add()) makes it convenient for determining whether something is 29 * actively referenced without the fuss of flagging lock ownership across 30 * function calls. 31 */ 32 #include <sys/zrlock.h> 33 #include <sys/trace_zfs.h> 34 35 /* 36 * A ZRL can be locked only while there are zero references, so ZRL_LOCKED is 37 * treated as zero references. 38 */ 39 #define ZRL_LOCKED -1 40 #define ZRL_DESTROYED -2 41 42 void 43 zrl_init(zrlock_t *zrl) 44 { 45 mutex_init(&zrl->zr_mtx, NULL, MUTEX_DEFAULT, NULL); 46 zrl->zr_refcount = 0; 47 cv_init(&zrl->zr_cv, NULL, CV_DEFAULT, NULL); 48 #ifdef ZFS_DEBUG 49 zrl->zr_owner = NULL; 50 zrl->zr_caller = NULL; 51 #endif 52 } 53 54 void 55 zrl_destroy(zrlock_t *zrl) 56 { 57 ASSERT0(zrl->zr_refcount); 58 59 mutex_destroy(&zrl->zr_mtx); 60 zrl->zr_refcount = ZRL_DESTROYED; 61 cv_destroy(&zrl->zr_cv); 62 } 63 64 void 65 zrl_add_impl(zrlock_t *zrl, const char *zc) 66 { 67 for (;;) { 68 uint32_t n = (uint32_t)zrl->zr_refcount; 69 while (n != ZRL_LOCKED) { 70 uint32_t cas = atomic_cas_32( 71 (uint32_t *)&zrl->zr_refcount, n, n + 1); 72 if (cas == n) { 73 ASSERT3S((int32_t)n, >=, 0); 74 #ifdef ZFS_DEBUG 75 if (zrl->zr_owner == curthread) { 76 DTRACE_PROBE3(zrlock__reentry, 77 zrlock_t *, zrl, 78 kthread_t *, curthread, 79 uint32_t, n); 80 } 81 zrl->zr_owner = curthread; 82 zrl->zr_caller = zc; 83 #endif 84 return; 85 } 86 n = cas; 87 } 88 89 mutex_enter(&zrl->zr_mtx); 90 while (zrl->zr_refcount == ZRL_LOCKED) { 91 cv_wait(&zrl->zr_cv, &zrl->zr_mtx); 92 } 93 mutex_exit(&zrl->zr_mtx); 94 } 95 } 96 97 void 98 zrl_remove(zrlock_t *zrl) 99 { 100 #ifdef ZFS_DEBUG 101 if (zrl->zr_owner == curthread) { 102 zrl->zr_owner = NULL; 103 zrl->zr_caller = NULL; 104 } 105 int32_t n = atomic_dec_32_nv((uint32_t *)&zrl->zr_refcount); 106 ASSERT3S(n, >=, 0); 107 #else 108 atomic_dec_32((uint32_t *)&zrl->zr_refcount); 109 #endif 110 } 111 112 int 113 zrl_tryenter(zrlock_t *zrl) 114 { 115 uint32_t n = (uint32_t)zrl->zr_refcount; 116 117 if (n == 0) { 118 uint32_t cas = atomic_cas_32( 119 (uint32_t *)&zrl->zr_refcount, 0, ZRL_LOCKED); 120 if (cas == 0) { 121 #ifdef ZFS_DEBUG 122 ASSERT0P(zrl->zr_owner); 123 zrl->zr_owner = curthread; 124 #endif 125 return (1); 126 } 127 } 128 129 ASSERT3S((int32_t)n, >, ZRL_DESTROYED); 130 131 return (0); 132 } 133 134 void 135 zrl_exit(zrlock_t *zrl) 136 { 137 ASSERT3S(zrl->zr_refcount, ==, ZRL_LOCKED); 138 139 mutex_enter(&zrl->zr_mtx); 140 #ifdef ZFS_DEBUG 141 ASSERT3P(zrl->zr_owner, ==, curthread); 142 zrl->zr_owner = NULL; 143 membar_producer(); /* make sure the owner store happens first */ 144 #endif 145 zrl->zr_refcount = 0; 146 cv_broadcast(&zrl->zr_cv); 147 mutex_exit(&zrl->zr_mtx); 148 } 149 150 int 151 zrl_is_zero(zrlock_t *zrl) 152 { 153 ASSERT3S(zrl->zr_refcount, >, ZRL_DESTROYED); 154 155 return (zrl->zr_refcount <= 0); 156 } 157 158 int 159 zrl_is_locked(zrlock_t *zrl) 160 { 161 ASSERT3S(zrl->zr_refcount, >, ZRL_DESTROYED); 162 163 return (zrl->zr_refcount == ZRL_LOCKED); 164 } 165 166 #ifdef ZFS_DEBUG 167 kthread_t * 168 zrl_owner(zrlock_t *zrl) 169 { 170 return (zrl->zr_owner); 171 } 172 #endif 173 174 EXPORT_SYMBOL(zrl_add_impl); 175 EXPORT_SYMBOL(zrl_remove); 176