1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2014, 2015 by Delphix. All rights reserved.
25 * Copyright 2016 The MathWorks, Inc. All rights reserved.
26 */
27
28 /*
29 * A Zero Reference Lock (ZRL) is a reference count that can lock out new
30 * references only when the count is zero and only without waiting if the count
31 * is not already zero. It is similar to a read-write lock in that it allows
32 * multiple readers and only a single writer, but it does not allow a writer to
33 * block while waiting for readers to exit, and therefore the question of
34 * reader/writer priority is moot (no WRWANT bit). Since the equivalent of
35 * rw_enter(&lock, RW_WRITER) is disallowed and only tryenter() is allowed, it
36 * is perfectly safe for the same reader to acquire the same lock multiple
37 * times. The fact that a ZRL is reentrant for readers (through multiple calls
38 * to zrl_add()) makes it convenient for determining whether something is
39 * actively referenced without the fuss of flagging lock ownership across
40 * function calls.
41 */
42 #include <sys/zrlock.h>
43 #include <sys/trace_zfs.h>
44
45 /*
46 * A ZRL can be locked only while there are zero references, so ZRL_LOCKED is
47 * treated as zero references.
48 */
49 #define ZRL_LOCKED -1
50 #define ZRL_DESTROYED -2
51
52 void
zrl_init(zrlock_t * zrl)53 zrl_init(zrlock_t *zrl)
54 {
55 mutex_init(&zrl->zr_mtx, NULL, MUTEX_DEFAULT, NULL);
56 zrl->zr_refcount = 0;
57 cv_init(&zrl->zr_cv, NULL, CV_DEFAULT, NULL);
58 #ifdef ZFS_DEBUG
59 zrl->zr_owner = NULL;
60 zrl->zr_caller = NULL;
61 #endif
62 }
63
64 void
zrl_destroy(zrlock_t * zrl)65 zrl_destroy(zrlock_t *zrl)
66 {
67 ASSERT0(zrl->zr_refcount);
68
69 mutex_destroy(&zrl->zr_mtx);
70 zrl->zr_refcount = ZRL_DESTROYED;
71 cv_destroy(&zrl->zr_cv);
72 }
73
74 void
zrl_add_impl(zrlock_t * zrl,const char * zc)75 zrl_add_impl(zrlock_t *zrl, const char *zc)
76 {
77 for (;;) {
78 uint32_t n = (uint32_t)zrl->zr_refcount;
79 while (n != ZRL_LOCKED) {
80 uint32_t cas = atomic_cas_32(
81 (uint32_t *)&zrl->zr_refcount, n, n + 1);
82 if (cas == n) {
83 ASSERT3S((int32_t)n, >=, 0);
84 #ifdef ZFS_DEBUG
85 if (zrl->zr_owner == curthread) {
86 DTRACE_PROBE3(zrlock__reentry,
87 zrlock_t *, zrl,
88 kthread_t *, curthread,
89 uint32_t, n);
90 }
91 zrl->zr_owner = curthread;
92 zrl->zr_caller = zc;
93 #endif
94 return;
95 }
96 n = cas;
97 }
98
99 mutex_enter(&zrl->zr_mtx);
100 while (zrl->zr_refcount == ZRL_LOCKED) {
101 cv_wait(&zrl->zr_cv, &zrl->zr_mtx);
102 }
103 mutex_exit(&zrl->zr_mtx);
104 }
105 }
106
107 void
zrl_remove(zrlock_t * zrl)108 zrl_remove(zrlock_t *zrl)
109 {
110 #ifdef ZFS_DEBUG
111 if (zrl->zr_owner == curthread) {
112 zrl->zr_owner = NULL;
113 zrl->zr_caller = NULL;
114 }
115 int32_t n = atomic_dec_32_nv((uint32_t *)&zrl->zr_refcount);
116 ASSERT3S(n, >=, 0);
117 #else
118 atomic_dec_32((uint32_t *)&zrl->zr_refcount);
119 #endif
120 }
121
122 int
zrl_tryenter(zrlock_t * zrl)123 zrl_tryenter(zrlock_t *zrl)
124 {
125 uint32_t n = (uint32_t)zrl->zr_refcount;
126
127 if (n == 0) {
128 uint32_t cas = atomic_cas_32(
129 (uint32_t *)&zrl->zr_refcount, 0, ZRL_LOCKED);
130 if (cas == 0) {
131 #ifdef ZFS_DEBUG
132 ASSERT3P(zrl->zr_owner, ==, NULL);
133 zrl->zr_owner = curthread;
134 #endif
135 return (1);
136 }
137 }
138
139 ASSERT3S((int32_t)n, >, ZRL_DESTROYED);
140
141 return (0);
142 }
143
144 void
zrl_exit(zrlock_t * zrl)145 zrl_exit(zrlock_t *zrl)
146 {
147 ASSERT3S(zrl->zr_refcount, ==, ZRL_LOCKED);
148
149 mutex_enter(&zrl->zr_mtx);
150 #ifdef ZFS_DEBUG
151 ASSERT3P(zrl->zr_owner, ==, curthread);
152 zrl->zr_owner = NULL;
153 membar_producer(); /* make sure the owner store happens first */
154 #endif
155 zrl->zr_refcount = 0;
156 cv_broadcast(&zrl->zr_cv);
157 mutex_exit(&zrl->zr_mtx);
158 }
159
160 int
zrl_is_zero(zrlock_t * zrl)161 zrl_is_zero(zrlock_t *zrl)
162 {
163 ASSERT3S(zrl->zr_refcount, >, ZRL_DESTROYED);
164
165 return (zrl->zr_refcount <= 0);
166 }
167
168 int
zrl_is_locked(zrlock_t * zrl)169 zrl_is_locked(zrlock_t *zrl)
170 {
171 ASSERT3S(zrl->zr_refcount, >, ZRL_DESTROYED);
172
173 return (zrl->zr_refcount == ZRL_LOCKED);
174 }
175
176 #ifdef ZFS_DEBUG
177 kthread_t *
zrl_owner(zrlock_t * zrl)178 zrl_owner(zrlock_t *zrl)
179 {
180 return (zrl->zr_owner);
181 }
182 #endif
183
184 #if defined(_KERNEL)
185
186 EXPORT_SYMBOL(zrl_add_impl);
187 EXPORT_SYMBOL(zrl_remove);
188
189 #endif
190