151 _ck_epoch_delref(struct ck_epoch_record *record,  in CK_STACK_CONTAINER()
157 	current = &record->local.bucket[i];  in CK_STACK_CONTAINER()
172 	other = &record->local.bucket[(i + 1) & CK_EPOCH_SENSE_MASK];  in CK_STACK_CONTAINER()
179 		ck_pr_store_uint(&record->epoch, other->epoch);  in CK_STACK_CONTAINER()
186 _ck_epoch_addref(struct ck_epoch_record *record,  in _ck_epoch_addref()  argument
189 	struct ck_epoch *global = record->global;  in _ck_epoch_addref()
195 	ref = &record->local.bucket[i];  in _ck_epoch_addref()
211 		previous = &record->local.bucket[(i + 1) &  in _ck_epoch_addref()
242 	struct ck_epoch_record *record;  in ck_epoch_recycle()  local
250 		record = ck_epoch_record_container(cursor);  in ck_epoch_recycle()
252 		if (ck_pr_load_uint(&record->state) == CK_EPOCH_STATE_FREE) {  in ck_epoch_recycle()
255 			state = ck_pr_fas_uint(&record->state,  in ck_epoch_recycle()
259 				ck_pr_store_ptr(&record->ct, ct);  in ck_epoch_recycle()
265 				return record;  in ck_epoch_recycle()
274 ck_epoch_register(struct ck_epoch *global, struct ck_epoch_record *record,  in ck_epoch_register()  argument
279 	record->global = global;  in ck_epoch_register()
280 	record->state = CK_EPOCH_STATE_USED;  in ck_epoch_register()
281 	record->active = 0;  in ck_epoch_register()
282 	record->epoch = 0;  in ck_epoch_register()
283 	record->n_dispatch = 0;  in ck_epoch_register()
284 	record->n_peak = 0;  in ck_epoch_register()
285 	record->n_pending = 0;  in ck_epoch_register()
286 	record->ct = ct;  in ck_epoch_register()
287 	memset(&record->local, 0, sizeof record->local);  in ck_epoch_register()
290 		ck_stack_init(&record->pending[i]);  in ck_epoch_register()
293 	ck_stack_push_upmc(&global->records, &record->record_next);  in ck_epoch_register()
298 ck_epoch_unregister(struct ck_epoch_record *record)  in ck_epoch_unregister()  argument
300 	struct ck_epoch *global = record->global;  in ck_epoch_unregister()
303 	record->active = 0;  in ck_epoch_unregister()
304 	record->epoch = 0;  in ck_epoch_unregister()
305 	record->n_dispatch = 0;  in ck_epoch_unregister()
306 	record->n_peak = 0;  in ck_epoch_unregister()
307 	record->n_pending = 0;  in ck_epoch_unregister()
308 	memset(&record->local, 0, sizeof record->local);  in ck_epoch_unregister()
311 		ck_stack_init(&record->pending[i]);  in ck_epoch_unregister()
313 	ck_pr_store_ptr(&record->ct, NULL);  in ck_epoch_unregister()
315 	ck_pr_store_uint(&record->state, CK_EPOCH_STATE_FREE);  in ck_epoch_unregister()
360 ck_epoch_dispatch(struct ck_epoch_record *record, unsigned int e, ck_stack_t *deferred)  in ck_epoch_dispatch()  argument
367 	head = ck_stack_batch_pop_upmc(&record->pending[epoch]);  in ck_epoch_dispatch()
381 	n_peak = ck_pr_load_uint(&record->n_peak);  in ck_epoch_dispatch()
382 	n_pending = ck_pr_load_uint(&record->n_pending);  in ck_epoch_dispatch()
386 		ck_pr_store_uint(&record->n_peak, n_peak);  in ck_epoch_dispatch()
389 		ck_pr_add_uint(&record->n_dispatch, i);  in ck_epoch_dispatch()
390 		ck_pr_sub_uint(&record->n_pending, i);  in ck_epoch_dispatch()
400 ck_epoch_reclaim(struct ck_epoch_record *record)  in ck_epoch_reclaim()  argument
405 		ck_epoch_dispatch(record, epoch, NULL);  in ck_epoch_reclaim()
529 ck_epoch_synchronize(struct ck_epoch_record *record)  in ck_epoch_synchronize()  argument
532 	ck_epoch_synchronize_wait(record->global, NULL, NULL);  in ck_epoch_synchronize()
537 ck_epoch_barrier(struct ck_epoch_record *record)  in ck_epoch_barrier()  argument
540 	ck_epoch_synchronize(record);  in ck_epoch_barrier()
541 	ck_epoch_reclaim(record);  in ck_epoch_barrier()
546 ck_epoch_barrier_wait(struct ck_epoch_record *record, ck_epoch_wait_cb_t *cb,  in ck_epoch_barrier_wait()  argument
550 	ck_epoch_synchronize_wait(record->global, cb, ct);  in ck_epoch_barrier_wait()
551 	ck_epoch_reclaim(record);  in ck_epoch_barrier_wait()
566 ck_epoch_poll_deferred(struct ck_epoch_record *record, ck_stack_t *deferred)  in ck_epoch_poll_deferred()  argument
571 	struct ck_epoch *global = record->global;  in ck_epoch_poll_deferred()
589 	n_dispatch = ck_epoch_dispatch(record, epoch - 2, deferred);  in ck_epoch_poll_deferred()
597 		record->epoch = epoch;  in ck_epoch_poll_deferred()
599 			ck_epoch_dispatch(record, epoch, deferred);  in ck_epoch_poll_deferred()
614 	ck_epoch_dispatch(record, epoch - 1, deferred);  in ck_epoch_poll_deferred()
619 ck_epoch_poll(struct ck_epoch_record *record)  in ck_epoch_poll()  argument
622 	return ck_epoch_poll_deferred(record, NULL);  in ck_epoch_poll()