ck/src/ck_rhs.c

/*
 * Copyright 2014-2015 Olivier Houchard.
 * Copyright 2012-2015 Samy Al Bahra.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <ck_cc.h>
#include <ck_rhs.h>
#include <ck_limits.h>
#include <ck_md.h>
#include <ck_pr.h>
#include <ck_stdint.h>
#include <ck_stdbool.h>
#include <ck_string.h>

#include "ck_internal.h"

#ifndef CK_RHS_PROBE_L1_SHIFT
#define CK_RHS_PROBE_L1_SHIFT 3ULL
#endif /* CK_RHS_PROBE_L1_SHIFT */

#define CK_RHS_PROBE_L1 (1 << CK_RHS_PROBE_L1_SHIFT)
#define CK_RHS_PROBE_L1_MASK (CK_RHS_PROBE_L1 - 1)

#ifndef CK_RHS_PROBE_L1_DEFAULT
#define CK_RHS_PROBE_L1_DEFAULT CK_MD_CACHELINE
#endif

#define CK_RHS_VMA_MASK ((uintptr_t)((1ULL << CK_MD_VMA_BITS) - 1))
#define CK_RHS_VMA(x)	\
	((void *)((uintptr_t)(x) & CK_RHS_VMA_MASK))

#define CK_RHS_EMPTY     NULL
#define CK_RHS_G		(1024)
#define CK_RHS_G_MASK	(CK_RHS_G - 1)

#if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_STORE_8)
#define CK_RHS_WORD          uint8_t
#define CK_RHS_WORD_MAX	    UINT8_MAX
#define CK_RHS_STORE(x, y)   ck_pr_store_8(x, y)
#define CK_RHS_LOAD(x)       ck_pr_load_8(x)
#elif defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_STORE_16)
#define CK_RHS_WORD          uint16_t
#define CK_RHS_WORD_MAX	    UINT16_MAX
#define CK_RHS_STORE(x, y)   ck_pr_store_16(x, y)
#define CK_RHS_LOAD(x)       ck_pr_load_16(x)
#elif defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_STORE_32)
#define CK_RHS_WORD          uint32_t
#define CK_RHS_WORD_MAX	    UINT32_MAX
#define CK_RHS_STORE(x, y)   ck_pr_store_32(x, y)
#define CK_RHS_LOAD(x)       ck_pr_load_32(x)
#else
#error "ck_rhs is not supported on your platform."
#endif

#define CK_RHS_MAX_WANTED	0xffff

enum ck_rhs_probe_behavior {
	CK_RHS_PROBE = 0,	/* Default behavior. */
	CK_RHS_PROBE_RH,	/* Short-circuit if RH slot found. */
	CK_RHS_PROBE_INSERT,	/* Short-circuit on probe bound if tombstone found. */

	CK_RHS_PROBE_ROBIN_HOOD,/* Look for the first slot available for the entry we are about to replace, only used to internally implement Robin Hood */
	CK_RHS_PROBE_NO_RH,	/* Don't do the RH dance */
};
struct ck_rhs_entry_desc {
	unsigned int probes;
	unsigned short wanted;
	CK_RHS_WORD probe_bound;
	bool in_rh;
	const void *entry;
} CK_CC_ALIGN(16);

struct ck_rhs_no_entry_desc {
	unsigned int probes;
	unsigned short wanted;
	CK_RHS_WORD probe_bound;
	bool in_rh;
} CK_CC_ALIGN(8);

typedef long ck_rhs_probe_cb_t(struct ck_rhs *hs,
    struct ck_rhs_map *map,
    unsigned long *n_probes,
    long *priority,
    unsigned long h,
    const void *key,
    const void **object,
    unsigned long probe_limit,
    enum ck_rhs_probe_behavior behavior);

struct ck_rhs_map {
	unsigned int generation[CK_RHS_G];
	unsigned int probe_maximum;
	unsigned long mask;
	unsigned long step;
	unsigned int probe_limit;
	unsigned long n_entries;
	unsigned long capacity;
	unsigned long size;
	unsigned long max_entries;
	char offset_mask;
	union {
		struct ck_rhs_entry_desc *descs;
		struct ck_rhs_no_entry {
			const void **entries;
			struct ck_rhs_no_entry_desc *descs;
		} no_entries;
	} entries;
	bool read_mostly;
	ck_rhs_probe_cb_t *probe_func;
};

static CK_CC_INLINE const void *
ck_rhs_entry(struct ck_rhs_map *map, long offset)
{

	if (map->read_mostly)
		return (map->entries.no_entries.entries[offset]);
	else
		return (map->entries.descs[offset].entry);
}

static CK_CC_INLINE const void **
ck_rhs_entry_addr(struct ck_rhs_map *map, long offset)
{

	if (map->read_mostly)
		return (&map->entries.no_entries.entries[offset]);
	else
		return (&map->entries.descs[offset].entry);
}

static CK_CC_INLINE struct ck_rhs_entry_desc *
ck_rhs_desc(struct ck_rhs_map *map, long offset)
{

	if (CK_CC_UNLIKELY(map->read_mostly))
		return ((struct ck_rhs_entry_desc *)(void *)&map->entries.no_entries.descs[offset]);
	else
		return (&map->entries.descs[offset]);
}

static CK_CC_INLINE void
ck_rhs_wanted_inc(struct ck_rhs_map *map, long offset)
{

	if (CK_CC_UNLIKELY(map->read_mostly))
		map->entries.no_entries.descs[offset].wanted++;
	else
		map->entries.descs[offset].wanted++;
}

static CK_CC_INLINE unsigned int
ck_rhs_probes(struct ck_rhs_map *map, long offset)
{

	if (CK_CC_UNLIKELY(map->read_mostly))
		return (map->entries.no_entries.descs[offset].probes);
	else
		return (map->entries.descs[offset].probes);
}

static CK_CC_INLINE void
ck_rhs_set_probes(struct ck_rhs_map *map, long offset, unsigned int value)
{

	if (CK_CC_UNLIKELY(map->read_mostly))
		map->entries.no_entries.descs[offset].probes = value;
	else
		map->entries.descs[offset].probes = value;
}

static CK_CC_INLINE CK_RHS_WORD
ck_rhs_probe_bound(struct ck_rhs_map *map, long offset)
{

	if (CK_CC_UNLIKELY(map->read_mostly))
		return (map->entries.no_entries.descs[offset].probe_bound);
	else
		return (map->entries.descs[offset].probe_bound);
}

static CK_CC_INLINE CK_RHS_WORD *
ck_rhs_probe_bound_addr(struct ck_rhs_map *map, long offset)
{

	if (CK_CC_UNLIKELY(map->read_mostly))
		return (&map->entries.no_entries.descs[offset].probe_bound);
	else
		return (&map->entries.descs[offset].probe_bound);
}


static CK_CC_INLINE bool
ck_rhs_in_rh(struct ck_rhs_map *map, long offset)
{

	if (CK_CC_UNLIKELY(map->read_mostly))
		return (map->entries.no_entries.descs[offset].in_rh);
	else
		return (map->entries.descs[offset].in_rh);
}

static CK_CC_INLINE void
ck_rhs_set_rh(struct ck_rhs_map *map, long offset)
{

	if (CK_CC_UNLIKELY(map->read_mostly))
		map->entries.no_entries.descs[offset].in_rh = true;
	else
		map->entries.descs[offset].in_rh = true;
}

static CK_CC_INLINE void
ck_rhs_unset_rh(struct ck_rhs_map *map, long offset)
{

	if (CK_CC_UNLIKELY(map->read_mostly))
		map->entries.no_entries.descs[offset].in_rh = false;
	else
		map->entries.descs[offset].in_rh = false;
}


#define CK_RHS_DEFAULT_LOAD_FACTOR	50

static ck_rhs_probe_cb_t ck_rhs_map_probe;
static ck_rhs_probe_cb_t ck_rhs_map_probe_rm;

bool
ck_rhs_set_load_factor(struct ck_rhs *hs, unsigned int load_factor)
{
	struct ck_rhs_map *map = hs->map;

	if (load_factor == 0 || load_factor > 100)
		return false;

	hs->load_factor = load_factor;
	map->max_entries = (map->capacity * (unsigned long)hs->load_factor) / 100;
	while (map->n_entries > map->max_entries) {
		if (ck_rhs_grow(hs, map->capacity << 1) == false)
			return false;
		map = hs->map;
	}
	return true;
}

void
ck_rhs_iterator_init(struct ck_rhs_iterator *iterator)
{

	iterator->cursor = NULL;
	iterator->offset = 0;
	return;
}

bool
ck_rhs_next(struct ck_rhs *hs, struct ck_rhs_iterator *i, void **key)
{
	struct ck_rhs_map *map = hs->map;
	void *value;

	if (i->offset >= map->capacity)
		return false;

	do {
		value = CK_CC_DECONST_PTR(ck_rhs_entry(map, i->offset));
		if (value != CK_RHS_EMPTY) {
#ifdef CK_RHS_PP
			if (hs->mode & CK_RHS_MODE_OBJECT)
				value = CK_RHS_VMA(value);
#endif
			i->offset++;
			*key = value;
			return true;
		}
	} while (++i->offset < map->capacity);

	return false;
}

void
ck_rhs_stat(struct ck_rhs *hs, struct ck_rhs_stat *st)
{
	struct ck_rhs_map *map = hs->map;

	st->n_entries = map->n_entries;
	st->probe_maximum = map->probe_maximum;
	return;
}

unsigned long
ck_rhs_count(struct ck_rhs *hs)
{

	return hs->map->n_entries;
}

static void
ck_rhs_map_destroy(struct ck_malloc *m, struct ck_rhs_map *map, bool defer)
{

	m->free(map, map->size, defer);
	return;
}

void
ck_rhs_destroy(struct ck_rhs *hs)
{

	ck_rhs_map_destroy(hs->m, hs->map, false);
	return;
}

static struct ck_rhs_map *
ck_rhs_map_create(struct ck_rhs *hs, unsigned long entries)
{
	struct ck_rhs_map *map;
	unsigned long size, n_entries, limit;

	n_entries = ck_internal_power_2(entries);
	if (n_entries < CK_RHS_PROBE_L1)
		n_entries = CK_RHS_PROBE_L1;

	if (hs->mode & CK_RHS_MODE_READ_MOSTLY)
		size = sizeof(struct ck_rhs_map) +
		    (sizeof(void *) * n_entries +
		     sizeof(struct ck_rhs_no_entry_desc) * n_entries +
		     2 * CK_MD_CACHELINE - 1);
	else
		size = sizeof(struct ck_rhs_map) +
		    (sizeof(struct ck_rhs_entry_desc) * n_entries +
		     CK_MD_CACHELINE - 1);
	map = hs->m->malloc(size);
	if (map == NULL)
		return NULL;
	map->read_mostly = !!(hs->mode & CK_RHS_MODE_READ_MOSTLY);

	map->size = size;
	/* We should probably use a more intelligent heuristic for default probe length. */
	limit = ck_internal_max(n_entries >> (CK_RHS_PROBE_L1_SHIFT + 2), CK_RHS_PROBE_L1_DEFAULT);
	if (limit > UINT_MAX)
		limit = UINT_MAX;

	map->probe_limit = (unsigned int)limit;
	map->probe_maximum = 0;
	map->capacity = n_entries;
	map->step = ck_cc_ffsl(n_entries);
	map->mask = n_entries - 1;
	map->n_entries = 0;

	map->max_entries = (map->capacity * (unsigned long)hs->load_factor) / 100;
	/* Align map allocation to cache line. */
	if (map->read_mostly) {
		map->entries.no_entries.entries = (void *)(((uintptr_t)&map[1] +
		    CK_MD_CACHELINE - 1) & ~(CK_MD_CACHELINE - 1));
		map->entries.no_entries.descs = (void *)(((uintptr_t)map->entries.no_entries.entries + (sizeof(void *) * n_entries) + CK_MD_CACHELINE - 1) &~ (CK_MD_CACHELINE - 1));
		memset(map->entries.no_entries.entries, 0,
		    sizeof(void *) * n_entries);
		memset(map->entries.no_entries.descs, 0,
		    sizeof(struct ck_rhs_no_entry_desc) * n_entries);
		map->offset_mask = (CK_MD_CACHELINE / sizeof(void *)) - 1;
		map->probe_func = ck_rhs_map_probe_rm;

	} else {
		map->entries.descs = (void *)(((uintptr_t)&map[1] +
		    CK_MD_CACHELINE - 1) & ~(CK_MD_CACHELINE - 1));
		memset(map->entries.descs, 0, sizeof(struct ck_rhs_entry_desc) * n_entries);
		map->offset_mask = (CK_MD_CACHELINE / sizeof(struct ck_rhs_entry_desc)) - 1;
		map->probe_func = ck_rhs_map_probe;
	}
	memset(map->generation, 0, sizeof map->generation);

	/* Commit entries purge with respect to map publication. */
	ck_pr_fence_store();
	return map;
}

bool
ck_rhs_reset_size(struct ck_rhs *hs, unsigned long capacity)
{
	struct ck_rhs_map *map, *previous;

	previous = hs->map;
	map = ck_rhs_map_create(hs, capacity);
	if (map == NULL)
		return false;

	ck_pr_store_ptr(&hs->map, map);
	ck_rhs_map_destroy(hs->m, previous, true);
	return true;
}

bool
ck_rhs_reset(struct ck_rhs *hs)
{
	struct ck_rhs_map *previous;

	previous = hs->map;
	return ck_rhs_reset_size(hs, previous->capacity);
}

static inline unsigned long
ck_rhs_map_probe_next(struct ck_rhs_map *map,
    unsigned long offset,
    unsigned long probes)
{

	if (probes & map->offset_mask) {
		offset = (offset &~ map->offset_mask) +
		    ((offset + 1) & map->offset_mask);
		return offset;
	} else
		return (offset + probes) & map->mask;
}

static inline unsigned long
ck_rhs_map_probe_prev(struct ck_rhs_map *map, unsigned long offset,
    unsigned long probes)
{

	if (probes & map->offset_mask) {
		offset = (offset &~ map->offset_mask) + ((offset - 1) &
		    map->offset_mask);
		return offset;
	} else
		return ((offset - probes) & map->mask);
}


static inline void
ck_rhs_map_bound_set(struct ck_rhs_map *m,
    unsigned long h,
    unsigned long n_probes)
{
	unsigned long offset = h & m->mask;
	struct ck_rhs_entry_desc *desc;

	if (n_probes > m->probe_maximum)
		ck_pr_store_uint(&m->probe_maximum, n_probes);
	if (!(m->read_mostly)) {
		desc = &m->entries.descs[offset];

		if (desc->probe_bound < n_probes) {
			if (n_probes > CK_RHS_WORD_MAX)
				n_probes = CK_RHS_WORD_MAX;

			CK_RHS_STORE(&desc->probe_bound, n_probes);
			ck_pr_fence_store();
		}
	}

	return;
}

static inline unsigned int
ck_rhs_map_bound_get(struct ck_rhs_map *m, unsigned long h)
{
	unsigned long offset = h & m->mask;
	unsigned int r = CK_RHS_WORD_MAX;

	if (m->read_mostly)
		r = ck_pr_load_uint(&m->probe_maximum);
	else {
		r = CK_RHS_LOAD(&m->entries.descs[offset].probe_bound);
		if (r == CK_RHS_WORD_MAX)
			r = ck_pr_load_uint(&m->probe_maximum);
	}
	return r;
}

bool
ck_rhs_grow(struct ck_rhs *hs,
    unsigned long capacity)
{
	struct ck_rhs_map *map, *update;
	const void *previous, *prev_saved;
	unsigned long k, offset, probes;

restart:
	map = hs->map;
	if (map->capacity > capacity)
		return false;

	update = ck_rhs_map_create(hs, capacity);
	if (update == NULL)
		return false;

	for (k = 0; k < map->capacity; k++) {
		unsigned long h;

		prev_saved = previous = ck_rhs_entry(map, k);
		if (previous == CK_RHS_EMPTY)
			continue;

#ifdef CK_RHS_PP
		if (hs->mode & CK_RHS_MODE_OBJECT)
			previous = CK_RHS_VMA(previous);
#endif

		h = hs->hf(previous, hs->seed);
		offset = h & update->mask;
		probes = 0;

		for (;;) {
			const void **cursor = ck_rhs_entry_addr(update, offset);

			if (probes++ == update->probe_limit) {
				/*
				 * We have hit the probe limit, map needs to be even larger.
				 */
				ck_rhs_map_destroy(hs->m, update, false);
				capacity <<= 1;
				goto restart;
			}

			if (CK_CC_LIKELY(*cursor == CK_RHS_EMPTY)) {
				*cursor = prev_saved;
				update->n_entries++;
				ck_rhs_set_probes(update, offset, probes);
				ck_rhs_map_bound_set(update, h, probes);
				break;
			} else if (ck_rhs_probes(update, offset) < probes) {
				const void *tmp = prev_saved;
				unsigned int old_probes;
				prev_saved = previous = *cursor;
#ifdef CK_RHS_PP
				if (hs->mode & CK_RHS_MODE_OBJECT)
					previous = CK_RHS_VMA(previous);
#endif
				*cursor = tmp;
				ck_rhs_map_bound_set(update, h, probes);
				h = hs->hf(previous, hs->seed);
				old_probes = ck_rhs_probes(update, offset);
				ck_rhs_set_probes(update, offset, probes);
				probes = old_probes - 1;
				continue;
			}
			ck_rhs_wanted_inc(update, offset);
			offset = ck_rhs_map_probe_next(update, offset,  probes);
		}

	}

	ck_pr_fence_store();
	ck_pr_store_ptr(&hs->map, update);
	ck_rhs_map_destroy(hs->m, map, true);
	return true;
}

bool
ck_rhs_rebuild(struct ck_rhs *hs)
{

	return ck_rhs_grow(hs, hs->map->capacity);
}

static long
ck_rhs_map_probe_rm(struct ck_rhs *hs,
    struct ck_rhs_map *map,
    unsigned long *n_probes,
    long *priority,
    unsigned long h,
    const void *key,
    const void **object,
    unsigned long probe_limit,
    enum ck_rhs_probe_behavior behavior)
{
	const void *k;
	const void *compare;
	long pr = -1;
	unsigned long offset, probes, opl;

#ifdef CK_RHS_PP
	/* If we are storing object pointers, then we may leverage pointer packing. */
	unsigned long hv = 0;

	if (hs->mode & CK_RHS_MODE_OBJECT) {
		hv = (h >> 25) & CK_RHS_KEY_MASK;
		compare = CK_RHS_VMA(key);
	} else {
		compare = key;
	}
#else
	compare = key;
#endif
 	*object = NULL;
	if (behavior != CK_RHS_PROBE_ROBIN_HOOD) {
		probes = 0;
		offset = h & map->mask;
	} else {
		/* Restart from the bucket we were previously in */
		probes = *n_probes;
		offset = ck_rhs_map_probe_next(map, *priority,
		    probes);
	}
	opl = probe_limit;

	for (;;) {
		if (probes++ == probe_limit) {
			if (probe_limit == opl || pr != -1) {
				k = CK_RHS_EMPTY;
				goto leave;
			}
			/*
			 * If no eligible slot has been found yet, continue probe
			 * sequence with original probe limit.
			 */
			probe_limit = opl;
		}
		k = ck_pr_load_ptr(&map->entries.no_entries.entries[offset]);
		if (k == CK_RHS_EMPTY)
			goto leave;

		if (behavior != CK_RHS_PROBE_NO_RH) {
			struct ck_rhs_entry_desc *desc = (void *)&map->entries.no_entries.descs[offset];

			if (pr == -1 &&
			    desc->in_rh == false && desc->probes < probes) {
				pr = offset;
				*n_probes = probes;

				if (behavior == CK_RHS_PROBE_RH ||
				    behavior == CK_RHS_PROBE_ROBIN_HOOD) {
					k = CK_RHS_EMPTY;
					goto leave;
				}
			}
		}

		if (behavior != CK_RHS_PROBE_ROBIN_HOOD) {
#ifdef CK_RHS_PP
			if (hs->mode & CK_RHS_MODE_OBJECT) {
				if (((uintptr_t)k >> CK_MD_VMA_BITS) != hv) {
					offset = ck_rhs_map_probe_next(map, offset, probes);
					continue;
				}

				k = CK_RHS_VMA(k);
			}
#endif

			if (k == compare)
				goto leave;

			if (hs->compare == NULL) {
				offset = ck_rhs_map_probe_next(map, offset, probes);
				continue;
			}

			if (hs->compare(k, key) == true)
				goto leave;
		}
		offset = ck_rhs_map_probe_next(map, offset, probes);
	}
leave:
	if (probes > probe_limit) {
		offset = -1;
	} else {
		*object = k;
	}

	if (pr == -1)
		*n_probes = probes;

	*priority = pr;
	return offset;
}

static long
ck_rhs_map_probe(struct ck_rhs *hs,
    struct ck_rhs_map *map,
    unsigned long *n_probes,
    long *priority,
    unsigned long h,
    const void *key,
    const void **object,
    unsigned long probe_limit,
    enum ck_rhs_probe_behavior behavior)
{
	const void *k;
	const void *compare;
	long pr = -1;
	unsigned long offset, probes, opl;

#ifdef CK_RHS_PP
	/* If we are storing object pointers, then we may leverage pointer packing. */
	unsigned long hv = 0;

	if (hs->mode & CK_RHS_MODE_OBJECT) {
		hv = (h >> 25) & CK_RHS_KEY_MASK;
		compare = CK_RHS_VMA(key);
	} else {
		compare = key;
	}
#else
	compare = key;
#endif

 	*object = NULL;
	if (behavior != CK_RHS_PROBE_ROBIN_HOOD) {
		probes = 0;
		offset = h & map->mask;
	} else {
		/* Restart from the bucket we were previously in */
		probes = *n_probes;
		offset = ck_rhs_map_probe_next(map, *priority,
		    probes);
	}
	opl = probe_limit;
	if (behavior == CK_RHS_PROBE_INSERT)
		probe_limit = ck_rhs_map_bound_get(map, h);

	for (;;) {
		if (probes++ == probe_limit) {
			if (probe_limit == opl || pr != -1) {
				k = CK_RHS_EMPTY;
				goto leave;
			}
			/*
			 * If no eligible slot has been found yet, continue probe
			 * sequence with original probe limit.
			 */
			probe_limit = opl;
		}
		k = ck_pr_load_ptr(&map->entries.descs[offset].entry);
		if (k == CK_RHS_EMPTY)
			goto leave;
		if ((behavior != CK_RHS_PROBE_NO_RH)) {
			struct ck_rhs_entry_desc *desc = &map->entries.descs[offset];

			if (pr == -1 &&
			    desc->in_rh == false && desc->probes < probes) {
				pr = offset;
				*n_probes = probes;

				if (behavior == CK_RHS_PROBE_RH ||
				    behavior == CK_RHS_PROBE_ROBIN_HOOD) {
					k = CK_RHS_EMPTY;
					goto leave;
				}
			}
		}

		if (behavior != CK_RHS_PROBE_ROBIN_HOOD) {
#ifdef CK_RHS_PP
			if (hs->mode & CK_RHS_MODE_OBJECT) {
				if (((uintptr_t)k >> CK_MD_VMA_BITS) != hv) {
					offset = ck_rhs_map_probe_next(map, offset, probes);
					continue;
				}

				k = CK_RHS_VMA(k);
			}
#endif

			if (k == compare)
				goto leave;

			if (hs->compare == NULL) {
				offset = ck_rhs_map_probe_next(map, offset, probes);
				continue;
			}

			if (hs->compare(k, key) == true)
				goto leave;
		}
		offset = ck_rhs_map_probe_next(map, offset, probes);
	}
leave:
	if (probes > probe_limit) {
		offset = -1;
	} else {
		*object = k;
	}

	if (pr == -1)
		*n_probes = probes;

	*priority = pr;
	return offset;
}

static inline const void *
ck_rhs_marshal(unsigned int mode, const void *key, unsigned long h)
{
#ifdef CK_RHS_PP
	const void *insert;

	if (mode & CK_RHS_MODE_OBJECT) {
		insert = (void *)((uintptr_t)CK_RHS_VMA(key) | ((h >> 25) << CK_MD_VMA_BITS));
	} else {
		insert = key;
	}

	return insert;
#else
	(void)mode;
	(void)h;

	return key;
#endif
}

bool
ck_rhs_gc(struct ck_rhs *hs)
{
	unsigned long i;
	struct ck_rhs_map *map = hs->map;

	unsigned int max_probes = 0;
	for (i = 0; i < map->capacity; i++) {
		if (ck_rhs_probes(map, i) > max_probes)
			max_probes = ck_rhs_probes(map, i);
	}
	map->probe_maximum = max_probes;
	return true;
}

static void
ck_rhs_add_wanted(struct ck_rhs *hs, long end_offset, long old_slot,
	unsigned long h)
{
	struct ck_rhs_map *map = hs->map;
	long offset;
	unsigned int probes = 1;
	bool found_slot = false;
	struct ck_rhs_entry_desc *desc;

	offset = h & map->mask;

	if (old_slot == -1)
		found_slot = true;
	while (offset != end_offset) {
		if (offset == old_slot)
			found_slot = true;
		if (found_slot) {
			desc = ck_rhs_desc(map, offset);
			if (desc->wanted < CK_RHS_MAX_WANTED)
				desc->wanted++;
		}
		offset = ck_rhs_map_probe_next(map, offset, probes);
		probes++;
	}
}

static unsigned long
ck_rhs_remove_wanted(struct ck_rhs *hs, long offset, long limit)
{
	struct ck_rhs_map *map = hs->map;
	int probes = ck_rhs_probes(map, offset);
	bool do_remove = true;
	struct ck_rhs_entry_desc *desc;

	while (probes > 1) {
		probes--;
		offset = ck_rhs_map_probe_prev(map, offset, probes);
		if (offset == limit)
			do_remove = false;
		if (do_remove) {
			desc = ck_rhs_desc(map, offset);
			if (desc->wanted != CK_RHS_MAX_WANTED)
				desc->wanted--;
		}
	}
	return offset;
}

static long
ck_rhs_get_first_offset(struct ck_rhs_map *map, unsigned long offset, unsigned int probes)
{
	while (probes > (unsigned long)map->offset_mask + 1) {
		offset -= ((probes - 1) &~ map->offset_mask);
		offset &= map->mask;
		offset = (offset &~ map->offset_mask) +
		    ((offset - map->offset_mask) & map->offset_mask);
		probes -= map->offset_mask + 1;
	}
	return ((offset &~ map->offset_mask) + ((offset - (probes - 1)) & map->offset_mask));
}

#define CK_RHS_MAX_RH	512

static int
ck_rhs_put_robin_hood(struct ck_rhs *hs,
    long orig_slot, struct ck_rhs_entry_desc *desc)
{
	long slot, first;
	const void *object, *insert;
	unsigned long n_probes;
	struct ck_rhs_map *map;
	unsigned long h = 0;
	long prev;
	void *key;
	long prevs[CK_RHS_MAX_RH];
	unsigned int prevs_nb = 0;
	unsigned int i;

	map = hs->map;
	first = orig_slot;
	n_probes = desc->probes;
restart:
	key = CK_CC_DECONST_PTR(ck_rhs_entry(map, first));
	insert = key;
#ifdef CK_RHS_PP
	if (hs->mode & CK_RHS_MODE_OBJECT)
	    key = CK_RHS_VMA(key);
#endif
	orig_slot = first;
	ck_rhs_set_rh(map, orig_slot);

	slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object,
	    map->probe_limit, prevs_nb == CK_RHS_MAX_RH ?
	    CK_RHS_PROBE_NO_RH : CK_RHS_PROBE_ROBIN_HOOD);

	if (slot == -1 && first == -1) {
		if (ck_rhs_grow(hs, map->capacity << 1) == false) {
			desc->in_rh = false;

			for (i = 0; i < prevs_nb; i++)
				ck_rhs_unset_rh(map, prevs[i]);

			return -1;
		}

		return 1;
	}

	if (first != -1) {
		desc = ck_rhs_desc(map, first);
		int old_probes = desc->probes;

		desc->probes = n_probes;
		h = ck_rhs_get_first_offset(map, first, n_probes);
		ck_rhs_map_bound_set(map, h, n_probes);
		prev = orig_slot;
		prevs[prevs_nb++] = prev;
		n_probes = old_probes;
		goto restart;
	} else {
		/* An empty slot was found. */
		h =  ck_rhs_get_first_offset(map, slot, n_probes);
		ck_rhs_map_bound_set(map, h, n_probes);
		ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), insert);
		ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]);
		ck_pr_fence_atomic_store();
		ck_rhs_set_probes(map, slot, n_probes);
		desc->in_rh = 0;
		ck_rhs_add_wanted(hs, slot, orig_slot, h);
	}
	while (prevs_nb > 0) {
		prev = prevs[--prevs_nb];
		ck_pr_store_ptr(ck_rhs_entry_addr(map, orig_slot),
		    ck_rhs_entry(map, prev));
		h = ck_rhs_get_first_offset(map, orig_slot,
		    desc->probes);
		ck_rhs_add_wanted(hs, orig_slot, prev, h);
		ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]);
		ck_pr_fence_atomic_store();
		orig_slot = prev;
		desc->in_rh = false;
		desc = ck_rhs_desc(map, orig_slot);
	}
	return 0;
}

static void
ck_rhs_do_backward_shift_delete(struct ck_rhs *hs, long slot)
{
	struct ck_rhs_map *map = hs->map;
	struct ck_rhs_entry_desc *desc, *new_desc = NULL;
	unsigned long h;

	desc = ck_rhs_desc(map, slot);
	h = ck_rhs_remove_wanted(hs, slot, -1);
	while (desc->wanted > 0) {
		unsigned long offset = 0, tmp_offset;
		unsigned long wanted_probes = 1;
		unsigned int probe = 0;
		unsigned int max_probes;

		/* Find a successor */
		while (wanted_probes < map->probe_maximum) {
			probe = wanted_probes;
			offset = ck_rhs_map_probe_next(map, slot, probe);
			while (probe < map->probe_maximum) {
				new_desc = ck_rhs_desc(map, offset);
				if (new_desc->probes == probe + 1)
					break;
				probe++;
				offset = ck_rhs_map_probe_next(map, offset,
				    probe);
			}
			if (probe < map->probe_maximum)
				break;
			wanted_probes++;
		}
		if (!(wanted_probes < map->probe_maximum)) {
			desc->wanted = 0;
			break;
		}
		desc->probes = wanted_probes;
		h = ck_rhs_remove_wanted(hs, offset, slot);
		ck_pr_store_ptr(ck_rhs_entry_addr(map, slot),
		    ck_rhs_entry(map, offset));
		ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]);
		ck_pr_fence_atomic_store();
		if (wanted_probes < CK_RHS_WORD_MAX) {
			struct ck_rhs_entry_desc *hdesc = ck_rhs_desc(map, h);
			if (hdesc->wanted == 1)
				CK_RHS_STORE(&hdesc->probe_bound,
				    wanted_probes);
			else if (hdesc->probe_bound == CK_RHS_WORD_MAX ||
			    hdesc->probe_bound == new_desc->probes) {
				probe++;
				if (hdesc->probe_bound == CK_RHS_WORD_MAX)
					max_probes = map->probe_maximum;
				else {
					max_probes = hdesc->probe_bound;
					max_probes--;
				}
				tmp_offset = ck_rhs_map_probe_next(map, offset,
				    probe);
				while (probe < max_probes) {
					if (h == (unsigned long)ck_rhs_get_first_offset(map, tmp_offset, probe))
						break;
					probe++;
					tmp_offset = ck_rhs_map_probe_next(map, tmp_offset, probe);
				}
				if (probe == max_probes)
					CK_RHS_STORE(&hdesc->probe_bound,
					    wanted_probes);
			}
		}
		if (desc->wanted < CK_RHS_MAX_WANTED)
			desc->wanted--;
		slot = offset;
		desc = new_desc;
	}
	ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), CK_RHS_EMPTY);
	if ((desc->probes - 1) < CK_RHS_WORD_MAX)
		CK_RHS_STORE(ck_rhs_probe_bound_addr(map, h),
		    desc->probes - 1);
	desc->probes = 0;
}

bool
ck_rhs_fas(struct ck_rhs *hs,
    unsigned long h,
    const void *key,
    void **previous)
{
	long slot, first;
	const void *object;
	const void *insert;
	unsigned long n_probes;
	struct ck_rhs_map *map = hs->map;
	struct ck_rhs_entry_desc *desc, *desc2;

	*previous = NULL;
restart:
	slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object,
	    ck_rhs_map_bound_get(map, h), CK_RHS_PROBE);

	/* Replacement semantics presume existence. */
	if (object == NULL)
		return false;

	insert = ck_rhs_marshal(hs->mode, key, h);

	if (first != -1) {
		int ret;

		desc = ck_rhs_desc(map, slot);
		desc2 = ck_rhs_desc(map, first);
		desc->in_rh = true;
		ret = ck_rhs_put_robin_hood(hs, first, desc2);
		desc->in_rh = false;
		if (CK_CC_UNLIKELY(ret == 1))
			goto restart;
		else if (CK_CC_UNLIKELY(ret != 0))
			return false;
		ck_pr_store_ptr(ck_rhs_entry_addr(map, first), insert);
		ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]);
		ck_pr_fence_atomic_store();
		desc2->probes = n_probes;
		ck_rhs_add_wanted(hs, first, -1, h);
		ck_rhs_do_backward_shift_delete(hs, slot);
	} else {
		ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), insert);
		ck_rhs_set_probes(map, slot, n_probes);
	}
	*previous = CK_CC_DECONST_PTR(object);
	return true;
}

/*
 * An apply function takes two arguments. The first argument is a pointer to a
 * pre-existing object. The second argument is a pointer to the fifth argument
 * passed to ck_hs_apply. If a non-NULL pointer is passed to the first argument
 * and the return value of the apply function is NULL, then the pre-existing
 * value is deleted. If the return pointer is the same as the one passed to the
 * apply function then no changes are made to the hash table.  If the first
 * argument is non-NULL and the return pointer is different than that passed to
 * the apply function, then the pre-existing value is replaced. For
 * replacement, it is required that the value itself is identical to the
 * previous value.
 */
bool
ck_rhs_apply(struct ck_rhs *hs,
    unsigned long h,
    const void *key,
    ck_rhs_apply_fn_t *fn,
    void *cl)
{
	const void *insert;
	const void  *object, *delta = false;
	unsigned long n_probes;
	long slot, first;
	struct ck_rhs_map *map;
	bool delta_set = false;

restart:
	map = hs->map;

	slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object, map->probe_limit, CK_RHS_PROBE_INSERT);
	if (slot == -1 && first == -1) {
		if (ck_rhs_grow(hs, map->capacity << 1) == false)
			return false;

		goto restart;
	}
	if (!delta_set) {
		delta = fn(CK_CC_DECONST_PTR(object), cl);
		delta_set = true;
	}

	if (delta == NULL) {
		/*
		 * The apply function has requested deletion. If the object doesn't exist,
		 * then exit early.
		 */
		if (CK_CC_UNLIKELY(object == NULL))
			return true;

		/* Otherwise, delete it. */
		ck_rhs_do_backward_shift_delete(hs, slot);
		return true;
	}

	/* The apply function has not requested hash set modification so exit early. */
	if (delta == object)
		return true;

	/* A modification or insertion has been requested. */
	ck_rhs_map_bound_set(map, h, n_probes);

	insert = ck_rhs_marshal(hs->mode, delta, h);
	if (first != -1) {
		/*
		 * This follows the same semantics as ck_hs_set, please refer to that
		 * function for documentation.
		 */
		struct ck_rhs_entry_desc *desc = NULL, *desc2;
		if (slot != -1) {
			desc = ck_rhs_desc(map, slot);
			desc->in_rh = true;
		}
		desc2 = ck_rhs_desc(map, first);
		int ret = ck_rhs_put_robin_hood(hs, first, desc2);
		if (slot != -1)
			desc->in_rh = false;

		if (CK_CC_UNLIKELY(ret == 1))
			goto restart;
		if (CK_CC_UNLIKELY(ret == -1))
			return false;
		/* If an earlier bucket was found, then store entry there. */
		ck_pr_store_ptr(ck_rhs_entry_addr(map, first), insert);
		desc2->probes = n_probes;
		/*
		 * If a duplicate key was found, then delete it after
		 * signaling concurrent probes to restart. Optionally,
		 * it is possible to install tombstone after grace
		 * period if we can guarantee earlier position of
		 * duplicate key.
		 */
		ck_rhs_add_wanted(hs, first, -1, h);
		if (object != NULL) {
			ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]);
			ck_pr_fence_atomic_store();
			ck_rhs_do_backward_shift_delete(hs, slot);
		}
	} else {
		/*
		 * If we are storing into same slot, then atomic store is sufficient
		 * for replacement.
		 */
		ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), insert);
		ck_rhs_set_probes(map, slot, n_probes);
		if (object == NULL)
			ck_rhs_add_wanted(hs, slot, -1, h);
	}

	if (object == NULL) {
		map->n_entries++;
		if ((map->n_entries ) > map->max_entries)
			ck_rhs_grow(hs, map->capacity << 1);
	}
	return true;
}

bool
ck_rhs_set(struct ck_rhs *hs,
    unsigned long h,
    const void *key,
    void **previous)
{
	long slot, first;
	const void *object;
	const void *insert;
	unsigned long n_probes;
	struct ck_rhs_map *map;

	*previous = NULL;

restart:
	map = hs->map;

	slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object, map->probe_limit, CK_RHS_PROBE_INSERT);
	if (slot == -1 && first == -1) {
		if (ck_rhs_grow(hs, map->capacity << 1) == false)
			return false;

		goto restart;
	}
	ck_rhs_map_bound_set(map, h, n_probes);
	insert = ck_rhs_marshal(hs->mode, key, h);

	if (first != -1) {
		struct ck_rhs_entry_desc *desc = NULL, *desc2;
		if (slot != -1) {
			desc = ck_rhs_desc(map, slot);
			desc->in_rh = true;
		}
		desc2 = ck_rhs_desc(map, first);
		int ret = ck_rhs_put_robin_hood(hs, first, desc2);
		if (slot != -1)
			desc->in_rh = false;

		if (CK_CC_UNLIKELY(ret == 1))
			goto restart;
		if (CK_CC_UNLIKELY(ret == -1))
			return false;
		/* If an earlier bucket was found, then store entry there. */
		ck_pr_store_ptr(ck_rhs_entry_addr(map, first), insert);
		desc2->probes = n_probes;
		/*
		 * If a duplicate key was found, then delete it after
		 * signaling concurrent probes to restart. Optionally,
		 * it is possible to install tombstone after grace
		 * period if we can guarantee earlier position of
		 * duplicate key.
		 */
		ck_rhs_add_wanted(hs, first, -1, h);
		if (object != NULL) {
			ck_pr_inc_uint(&map->generation[h & CK_RHS_G_MASK]);
			ck_pr_fence_atomic_store();
			ck_rhs_do_backward_shift_delete(hs, slot);
		}

	} else {
		/*
		 * If we are storing into same slot, then atomic store is sufficient
		 * for replacement.
		 */
		ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), insert);
		ck_rhs_set_probes(map, slot, n_probes);
		if (object == NULL)
			ck_rhs_add_wanted(hs, slot, -1, h);
	}

	if (object == NULL) {
		map->n_entries++;
		if ((map->n_entries ) > map->max_entries)
			ck_rhs_grow(hs, map->capacity << 1);
	}

	*previous = CK_CC_DECONST_PTR(object);
	return true;
}

static bool
ck_rhs_put_internal(struct ck_rhs *hs,
    unsigned long h,
    const void *key,
    enum ck_rhs_probe_behavior behavior)
{
	long slot, first;
	const void *object;
	const void *insert;
	unsigned long n_probes;
	struct ck_rhs_map *map;

restart:
	map = hs->map;

	slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object,
	    map->probe_limit, behavior);

	if (slot == -1 && first == -1) {
		if (ck_rhs_grow(hs, map->capacity << 1) == false)
			return false;

		goto restart;
	}

	/* Fail operation if a match was found. */
	if (object != NULL)
		return false;

	ck_rhs_map_bound_set(map, h, n_probes);
	insert = ck_rhs_marshal(hs->mode, key, h);

	if (first != -1) {
		struct ck_rhs_entry_desc *desc = ck_rhs_desc(map, first);
		int ret = ck_rhs_put_robin_hood(hs, first, desc);
		if (CK_CC_UNLIKELY(ret == 1))
			return ck_rhs_put_internal(hs, h, key, behavior);
		else if (CK_CC_UNLIKELY(ret == -1))
			return false;
		/* Insert key into first bucket in probe sequence. */
		ck_pr_store_ptr(ck_rhs_entry_addr(map, first), insert);
		desc->probes = n_probes;
		ck_rhs_add_wanted(hs, first, -1, h);
	} else {
		/* An empty slot was found. */
		ck_pr_store_ptr(ck_rhs_entry_addr(map, slot), insert);
		ck_rhs_set_probes(map, slot, n_probes);
		ck_rhs_add_wanted(hs, slot, -1, h);
	}

	map->n_entries++;
	if ((map->n_entries ) > map->max_entries)
		ck_rhs_grow(hs, map->capacity << 1);
	return true;
}

bool
ck_rhs_put(struct ck_rhs *hs,
    unsigned long h,
    const void *key)
{

	return ck_rhs_put_internal(hs, h, key, CK_RHS_PROBE_INSERT);
}

bool
ck_rhs_put_unique(struct ck_rhs *hs,
    unsigned long h,
    const void *key)
{

	return ck_rhs_put_internal(hs, h, key, CK_RHS_PROBE_RH);
}

void *
ck_rhs_get(struct ck_rhs *hs,
    unsigned long h,
    const void *key)
{
	long first;
	const void *object;
	struct ck_rhs_map *map;
	unsigned long n_probes;
	unsigned int g, g_p, probe;
	unsigned int *generation;

	do {
		map = ck_pr_load_ptr(&hs->map);
		generation = &map->generation[h & CK_RHS_G_MASK];
		g = ck_pr_load_uint(generation);
		probe  = ck_rhs_map_bound_get(map, h);
		ck_pr_fence_load();

		first = -1;
		map->probe_func(hs, map, &n_probes, &first, h, key, &object, probe, CK_RHS_PROBE_NO_RH);

		ck_pr_fence_load();
		g_p = ck_pr_load_uint(generation);
	} while (g != g_p);

	return CK_CC_DECONST_PTR(object);
}

void *
ck_rhs_remove(struct ck_rhs *hs,
    unsigned long h,
    const void *key)
{
	long slot, first;
	const void *object;
	struct ck_rhs_map *map = hs->map;
	unsigned long n_probes;

	slot = map->probe_func(hs, map, &n_probes, &first, h, key, &object,
	    ck_rhs_map_bound_get(map, h), CK_RHS_PROBE_NO_RH);
	if (object == NULL)
		return NULL;

	map->n_entries--;
	ck_rhs_do_backward_shift_delete(hs, slot);
	return CK_CC_DECONST_PTR(object);
}

bool
ck_rhs_move(struct ck_rhs *hs,
    struct ck_rhs *source,
    ck_rhs_hash_cb_t *hf,
    ck_rhs_compare_cb_t *compare,
    struct ck_malloc *m)
{

	if (m == NULL || m->malloc == NULL || m->free == NULL || hf == NULL)
		return false;

	hs->mode = source->mode;
	hs->seed = source->seed;
	hs->map = source->map;
	hs->load_factor = source->load_factor;
	hs->m = m;
	hs->hf = hf;
	hs->compare = compare;
	return true;
}

bool
ck_rhs_init(struct ck_rhs *hs,
    unsigned int mode,
    ck_rhs_hash_cb_t *hf,
    ck_rhs_compare_cb_t *compare,
    struct ck_malloc *m,
    unsigned long n_entries,
    unsigned long seed)
{

	if (m == NULL || m->malloc == NULL || m->free == NULL || hf == NULL)
		return false;

	hs->m = m;
	hs->mode = mode;
	hs->seed = seed;
	hs->hf = hf;
	hs->compare = compare;
	hs->load_factor = CK_RHS_DEFAULT_LOAD_FACTOR;

	hs->map = ck_rhs_map_create(hs, n_entries);
	return hs->map != NULL;
}