1 /*
2 * Copyright 2011-2015 Samy Al Bahra.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #ifndef CK_EPOCH_H
28 #define CK_EPOCH_H
29
30 /*
31 * The implementation here is inspired from the work described in:
32 * Fraser, K. 2004. Practical Lock-Freedom. PhD Thesis, University
33 * of Cambridge Computing Laboratory.
34 */
35
36 #include <ck_cc.h>
37 #include <ck_md.h>
38 #include <ck_pr.h>
39 #include <ck_stack.h>
40 #include <ck_stdbool.h>
41
42 #ifndef CK_EPOCH_LENGTH
43 #define CK_EPOCH_LENGTH 4
44 #endif
45
46 /*
47 * This is used for sense detection with-respect to concurrent
48 * epoch sections.
49 */
50 #define CK_EPOCH_SENSE (2)
51
52 struct ck_epoch_entry;
53 typedef struct ck_epoch_entry ck_epoch_entry_t;
54 typedef void ck_epoch_cb_t(ck_epoch_entry_t *);
55
56 /*
57 * This should be embedded into objects you wish to be the target of
58 * ck_epoch_cb_t functions (with ck_epoch_call).
59 */
60 struct ck_epoch_entry {
61 ck_epoch_cb_t *function;
62 ck_stack_entry_t stack_entry;
63 };
64
65 /*
66 * A section object may be passed to every begin-end pair to allow for
67 * forward progress guarantees with-in prolonged active sections.
68 */
69 struct ck_epoch_section {
70 unsigned int bucket;
71 };
72 typedef struct ck_epoch_section ck_epoch_section_t;
73
74 /*
75 * Return pointer to ck_epoch_entry container object.
76 */
77 #define CK_EPOCH_CONTAINER(T, M, N) \
78 CK_CC_CONTAINER(struct ck_epoch_entry, T, M, N)
79
80 struct ck_epoch_ref {
81 unsigned int epoch;
82 unsigned int count;
83 };
84
85 struct ck_epoch_record {
86 ck_stack_entry_t record_next;
87 struct ck_epoch *global;
88 unsigned int state;
89 unsigned int epoch;
90 unsigned int active;
91 struct {
92 struct ck_epoch_ref bucket[CK_EPOCH_SENSE];
93 } local CK_CC_CACHELINE;
94 unsigned int n_pending;
95 unsigned int n_peak;
96 unsigned int n_dispatch;
97 void *ct;
98 ck_stack_t pending[CK_EPOCH_LENGTH];
99 } CK_CC_CACHELINE;
100 typedef struct ck_epoch_record ck_epoch_record_t;
101
102 struct ck_epoch {
103 unsigned int epoch;
104 unsigned int n_free;
105 ck_stack_t records;
106 };
107 typedef struct ck_epoch ck_epoch_t;
108
109 /*
110 * Internal functions.
111 */
112 void _ck_epoch_addref(ck_epoch_record_t *, ck_epoch_section_t *);
113 bool _ck_epoch_delref(ck_epoch_record_t *, ck_epoch_section_t *);
114
115 CK_CC_FORCE_INLINE static void *
ck_epoch_record_ct(const ck_epoch_record_t * record)116 ck_epoch_record_ct(const ck_epoch_record_t *record)
117 {
118
119 return ck_pr_load_ptr(&record->ct);
120 }
121
122 /*
123 * Marks the beginning of an epoch-protected section.
124 */
125 CK_CC_FORCE_INLINE static void
ck_epoch_begin(ck_epoch_record_t * record,ck_epoch_section_t * section)126 ck_epoch_begin(ck_epoch_record_t *record, ck_epoch_section_t *section)
127 {
128 struct ck_epoch *epoch = record->global;
129
130 /*
131 * Only observe new epoch if thread is not recursing into a read
132 * section.
133 */
134 if (record->active == 0) {
135 unsigned int g_epoch;
136
137 /*
138 * It is possible for loads to be re-ordered before the store
139 * is committed into the caller's epoch and active fields.
140 * For this reason, store to load serialization is necessary.
141 */
142 #if defined(CK_MD_TSO)
143 ck_pr_fas_uint(&record->active, 1);
144 ck_pr_fence_atomic_load();
145 #else
146 ck_pr_store_uint(&record->active, 1);
147 ck_pr_fence_memory();
148 #endif
149
150 /*
151 * This load is allowed to be re-ordered prior to setting
152 * active flag due to monotonic nature of the global epoch.
153 * However, stale values lead to measurable performance
154 * degradation in some torture tests so we disallow early load
155 * of global epoch.
156 */
157 g_epoch = ck_pr_load_uint(&epoch->epoch);
158 ck_pr_store_uint(&record->epoch, g_epoch);
159 } else {
160 ck_pr_store_uint(&record->active, record->active + 1);
161 }
162
163 if (section != NULL)
164 _ck_epoch_addref(record, section);
165
166 return;
167 }
168
169 /*
170 * Marks the end of an epoch-protected section. Returns true if no more
171 * sections exist for the caller.
172 */
173 CK_CC_FORCE_INLINE static bool
ck_epoch_end(ck_epoch_record_t * record,ck_epoch_section_t * section)174 ck_epoch_end(ck_epoch_record_t *record, ck_epoch_section_t *section)
175 {
176
177 ck_pr_fence_release();
178 ck_pr_store_uint(&record->active, record->active - 1);
179
180 if (section != NULL)
181 return _ck_epoch_delref(record, section);
182
183 return record->active == 0;
184 }
185
186 /*
187 * Defers the execution of the function pointed to by the "cb"
188 * argument until an epoch counter loop. This allows for a
189 * non-blocking deferral.
190 *
191 * We can get away without a fence here due to the monotonic nature
192 * of the epoch counter. Worst case, this will result in some delays
193 * before object destruction.
194 */
195 CK_CC_FORCE_INLINE static void
ck_epoch_call(ck_epoch_record_t * record,ck_epoch_entry_t * entry,ck_epoch_cb_t * function)196 ck_epoch_call(ck_epoch_record_t *record,
197 ck_epoch_entry_t *entry,
198 ck_epoch_cb_t *function)
199 {
200 struct ck_epoch *epoch = record->global;
201 unsigned int e = ck_pr_load_uint(&epoch->epoch);
202 unsigned int offset = e & (CK_EPOCH_LENGTH - 1);
203
204 record->n_pending++;
205 entry->function = function;
206 ck_stack_push_spnc(&record->pending[offset], &entry->stack_entry);
207 return;
208 }
209
210 /*
211 * Same as ck_epoch_call, but allows for records to be shared and is reentrant.
212 */
213 CK_CC_FORCE_INLINE static void
ck_epoch_call_strict(ck_epoch_record_t * record,ck_epoch_entry_t * entry,ck_epoch_cb_t * function)214 ck_epoch_call_strict(ck_epoch_record_t *record,
215 ck_epoch_entry_t *entry,
216 ck_epoch_cb_t *function)
217 {
218 struct ck_epoch *epoch = record->global;
219 unsigned int e = ck_pr_load_uint(&epoch->epoch);
220 unsigned int offset = e & (CK_EPOCH_LENGTH - 1);
221
222 ck_pr_inc_uint(&record->n_pending);
223 entry->function = function;
224
225 /* Store fence is implied by push operation. */
226 ck_stack_push_upmc(&record->pending[offset], &entry->stack_entry);
227 return;
228 }
229
230 /*
231 * This callback is used for synchronize_wait to allow for custom blocking
232 * behavior.
233 */
234 typedef void ck_epoch_wait_cb_t(ck_epoch_t *, ck_epoch_record_t *,
235 void *);
236
237 /*
238 * Return latest epoch value. This operation provides load ordering.
239 */
240 CK_CC_FORCE_INLINE static unsigned int
ck_epoch_value(const ck_epoch_t * ep)241 ck_epoch_value(const ck_epoch_t *ep)
242 {
243
244 ck_pr_fence_load();
245 return ck_pr_load_uint(&ep->epoch);
246 }
247
248 void ck_epoch_init(ck_epoch_t *);
249
250 /*
251 * Attempts to recycle an unused epoch record. If one is successfully
252 * allocated, the record context pointer is also updated.
253 */
254 ck_epoch_record_t *ck_epoch_recycle(ck_epoch_t *, void *);
255
256 /*
257 * Registers an epoch record. An optional context pointer may be passed that
258 * is retrievable with ck_epoch_record_ct.
259 */
260 void ck_epoch_register(ck_epoch_t *, ck_epoch_record_t *, void *);
261
262 /*
263 * Marks a record as available for re-use by a subsequent recycle operation.
264 * Note that the record cannot be physically destroyed.
265 */
266 void ck_epoch_unregister(ck_epoch_record_t *);
267
268 bool ck_epoch_poll(ck_epoch_record_t *);
269 bool ck_epoch_poll_deferred(struct ck_epoch_record *record, ck_stack_t *deferred);
270 void ck_epoch_synchronize(ck_epoch_record_t *);
271 void ck_epoch_synchronize_wait(ck_epoch_t *, ck_epoch_wait_cb_t *, void *);
272 void ck_epoch_barrier(ck_epoch_record_t *);
273 void ck_epoch_barrier_wait(ck_epoch_record_t *, ck_epoch_wait_cb_t *, void *);
274
275 /*
276 * Reclaim entries associated with a record. This is safe to call only on
277 * the caller's record or records that are using call_strict.
278 */
279 void ck_epoch_reclaim(ck_epoch_record_t *);
280
281 #endif /* CK_EPOCH_H */
282