1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12 /*
13 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
14 * Copyright (c) 2014, 2015 by Delphix. All rights reserved.
15 * Copyright 2016 The MathWorks, Inc. All rights reserved.
16 */
17
18 /*
19 * A Zero Reference Lock (ZRL) is a reference count that can lock out new
20 * references only when the count is zero and only without waiting if the count
21 * is not already zero. It is similar to a read-write lock in that it allows
22 * multiple readers and only a single writer, but it does not allow a writer to
23 * block while waiting for readers to exit, and therefore the question of
24 * reader/writer priority is moot (no WRWANT bit). Since the equivalent of
25 * rw_enter(&lock, RW_WRITER) is disallowed and only tryenter() is allowed, it
26 * is perfectly safe for the same reader to acquire the same lock multiple
27 * times. The fact that a ZRL is reentrant for readers (through multiple calls
28 * to zrl_add()) makes it convenient for determining whether something is
29 * actively referenced without the fuss of flagging lock ownership across
30 * function calls.
31 */
32 #include <sys/zrlock.h>
33 #include <sys/trace_zfs.h>
34
35 /*
36 * A ZRL can be locked only while there are zero references, so ZRL_LOCKED is
37 * treated as zero references.
38 */
39 #define ZRL_LOCKED -1
40 #define ZRL_DESTROYED -2
41
42 void
zrl_init(zrlock_t * zrl)43 zrl_init(zrlock_t *zrl)
44 {
45 mutex_init(&zrl->zr_mtx, NULL, MUTEX_DEFAULT, NULL);
46 zrl->zr_refcount = 0;
47 cv_init(&zrl->zr_cv, NULL, CV_DEFAULT, NULL);
48 #ifdef ZFS_DEBUG
49 zrl->zr_owner = NULL;
50 zrl->zr_caller = NULL;
51 #endif
52 }
53
54 void
zrl_destroy(zrlock_t * zrl)55 zrl_destroy(zrlock_t *zrl)
56 {
57 ASSERT0(zrl->zr_refcount);
58
59 mutex_destroy(&zrl->zr_mtx);
60 zrl->zr_refcount = ZRL_DESTROYED;
61 cv_destroy(&zrl->zr_cv);
62 }
63
64 void
zrl_add_impl(zrlock_t * zrl,const char * zc)65 zrl_add_impl(zrlock_t *zrl, const char *zc)
66 {
67 for (;;) {
68 uint32_t n = (uint32_t)zrl->zr_refcount;
69 while (n != ZRL_LOCKED) {
70 uint32_t cas = atomic_cas_32(
71 (uint32_t *)&zrl->zr_refcount, n, n + 1);
72 if (cas == n) {
73 ASSERT3S((int32_t)n, >=, 0);
74 #ifdef ZFS_DEBUG
75 if (zrl->zr_owner == curthread) {
76 DTRACE_PROBE3(zrlock__reentry,
77 zrlock_t *, zrl,
78 kthread_t *, curthread,
79 uint32_t, n);
80 }
81 zrl->zr_owner = curthread;
82 zrl->zr_caller = zc;
83 #endif
84 return;
85 }
86 n = cas;
87 }
88
89 mutex_enter(&zrl->zr_mtx);
90 while (zrl->zr_refcount == ZRL_LOCKED) {
91 cv_wait(&zrl->zr_cv, &zrl->zr_mtx);
92 }
93 mutex_exit(&zrl->zr_mtx);
94 }
95 }
96
97 void
zrl_remove(zrlock_t * zrl)98 zrl_remove(zrlock_t *zrl)
99 {
100 #ifdef ZFS_DEBUG
101 if (zrl->zr_owner == curthread) {
102 zrl->zr_owner = NULL;
103 zrl->zr_caller = NULL;
104 }
105 int32_t n = atomic_dec_32_nv((uint32_t *)&zrl->zr_refcount);
106 ASSERT3S(n, >=, 0);
107 #else
108 atomic_dec_32((uint32_t *)&zrl->zr_refcount);
109 #endif
110 }
111
112 int
zrl_tryenter(zrlock_t * zrl)113 zrl_tryenter(zrlock_t *zrl)
114 {
115 uint32_t n = (uint32_t)zrl->zr_refcount;
116
117 if (n == 0) {
118 uint32_t cas = atomic_cas_32(
119 (uint32_t *)&zrl->zr_refcount, 0, ZRL_LOCKED);
120 if (cas == 0) {
121 #ifdef ZFS_DEBUG
122 ASSERT0P(zrl->zr_owner);
123 zrl->zr_owner = curthread;
124 #endif
125 return (1);
126 }
127 }
128
129 ASSERT3S((int32_t)n, >, ZRL_DESTROYED);
130
131 return (0);
132 }
133
134 void
zrl_exit(zrlock_t * zrl)135 zrl_exit(zrlock_t *zrl)
136 {
137 ASSERT3S(zrl->zr_refcount, ==, ZRL_LOCKED);
138
139 mutex_enter(&zrl->zr_mtx);
140 #ifdef ZFS_DEBUG
141 ASSERT3P(zrl->zr_owner, ==, curthread);
142 zrl->zr_owner = NULL;
143 membar_producer(); /* make sure the owner store happens first */
144 #endif
145 zrl->zr_refcount = 0;
146 cv_broadcast(&zrl->zr_cv);
147 mutex_exit(&zrl->zr_mtx);
148 }
149
150 int
zrl_is_zero(zrlock_t * zrl)151 zrl_is_zero(zrlock_t *zrl)
152 {
153 ASSERT3S(zrl->zr_refcount, >, ZRL_DESTROYED);
154
155 return (zrl->zr_refcount <= 0);
156 }
157
158 int
zrl_is_locked(zrlock_t * zrl)159 zrl_is_locked(zrlock_t *zrl)
160 {
161 ASSERT3S(zrl->zr_refcount, >, ZRL_DESTROYED);
162
163 return (zrl->zr_refcount == ZRL_LOCKED);
164 }
165
166 #ifdef ZFS_DEBUG
167 kthread_t *
zrl_owner(zrlock_t * zrl)168 zrl_owner(zrlock_t *zrl)
169 {
170 return (zrl->zr_owner);
171 }
172 #endif
173
174 #if defined(_KERNEL)
175
176 EXPORT_SYMBOL(zrl_add_impl);
177 EXPORT_SYMBOL(zrl_remove);
178
179 #endif
180