44  * acquired some snapshot (e) of the global epoch value (e_g) and set an active
189 	struct ck_epoch *global = record->global;  in _ck_epoch_addref()  local
193 	epoch = ck_pr_load_uint(&global->epoch);  in _ck_epoch_addref()
207 		 * On TSO architectures, the monoticity of the global counter  in _ck_epoch_addref()
229 ck_epoch_init(struct ck_epoch *global)  in ck_epoch_init()  argument
232 	ck_stack_init(&global->records);  in ck_epoch_init()
233 	global->epoch = 1;  in ck_epoch_init()
234 	global->n_free = 0;  in ck_epoch_init()
240 ck_epoch_recycle(struct ck_epoch *global, void *ct)  in ck_epoch_recycle()  argument
246 	if (ck_pr_load_uint(&global->n_free) == 0)  in ck_epoch_recycle()
249 	CK_STACK_FOREACH(&global->records, cursor) {  in ck_epoch_recycle()
258 				ck_pr_dec_uint(&global->n_free);  in ck_epoch_recycle()
274 ck_epoch_register(struct ck_epoch *global, struct ck_epoch_record *record,  in ck_epoch_register()  argument
279 	record->global = global;  in ck_epoch_register()
293 	ck_stack_push_upmc(&global->records, &record->record_next);  in ck_epoch_register()
300 	struct ck_epoch *global = record->global;  in ck_epoch_unregister()  local
316 	ck_pr_inc_uint(&global->n_free);  in ck_epoch_unregister()
321 ck_epoch_scan(struct ck_epoch *global,  in ck_epoch_scan()  argument
329 		cursor = CK_STACK_FIRST(&global->records);  in ck_epoch_scan()
411 epoch_block(struct ck_epoch *global, struct ck_epoch_record *cr,  in epoch_block()  argument
416 		cb(global, cr, ct);  in epoch_block()
425 ck_epoch_synchronize_wait(struct ck_epoch *global,  in ck_epoch_synchronize_wait()  argument
435 	 * The observation of the global epoch must be ordered with respect to  in ck_epoch_synchronize_wait()
437 	 * monoticity of global epoch counter.  in ck_epoch_synchronize_wait()
443 	delta = epoch = ck_pr_load_uint(&global->epoch);  in ck_epoch_synchronize_wait()
453 		while (cr = ck_epoch_scan(global, cr, delta, &active),  in ck_epoch_synchronize_wait()
463 			e_d = ck_pr_load_uint(&global->epoch);  in ck_epoch_synchronize_wait()
465 				epoch_block(global, cr, cb, ct);  in ck_epoch_synchronize_wait()
477 			epoch_block(global, cr, cb, ct);  in ck_epoch_synchronize_wait()
497 		 * same global epoch value snapshot.  in ck_epoch_synchronize_wait()
505 		r = ck_pr_cas_uint_value(&global->epoch, delta, delta + 1,  in ck_epoch_synchronize_wait()
532 	ck_epoch_synchronize_wait(record->global, NULL, NULL);  in ck_epoch_synchronize()
550 	ck_epoch_synchronize_wait(record->global, cb, ct);  in ck_epoch_barrier_wait()
571 	struct ck_epoch *global = record->global;  in ck_epoch_poll_deferred()  local
574 	epoch = ck_pr_load_uint(&global->epoch);  in ck_epoch_poll_deferred()
576 	/* Serialize epoch snapshots with respect to global epoch. */  in ck_epoch_poll_deferred()
580 	 * At this point, epoch is the current global epoch value.  in ck_epoch_poll_deferred()
586 	 * allow another thread to increment the global epoch before this  in ck_epoch_poll_deferred()
591 	cr = ck_epoch_scan(global, cr, epoch, &active);  in ck_epoch_poll_deferred()
612 	(void)ck_pr_cas_uint(&global->epoch, epoch, epoch + 1);  in ck_epoch_poll_deferred()