xref: /freebsd/sys/geom/eli/g_eli_key_cache.c (revision 39b7ca4533a5cbdb182b0388955d4c7d37a74ab8)
11e09ff3dSPawel Jakub Dawidek /*-
21e09ff3dSPawel Jakub Dawidek  * Copyright (c) 2011 Pawel Jakub Dawidek <pawel@dawidek.net>
31e09ff3dSPawel Jakub Dawidek  * All rights reserved.
41e09ff3dSPawel Jakub Dawidek  *
51e09ff3dSPawel Jakub Dawidek  * Redistribution and use in source and binary forms, with or without
61e09ff3dSPawel Jakub Dawidek  * modification, are permitted provided that the following conditions
71e09ff3dSPawel Jakub Dawidek  * are met:
81e09ff3dSPawel Jakub Dawidek  * 1. Redistributions of source code must retain the above copyright
91e09ff3dSPawel Jakub Dawidek  *    notice, this list of conditions and the following disclaimer.
101e09ff3dSPawel Jakub Dawidek  * 2. Redistributions in binary form must reproduce the above copyright
111e09ff3dSPawel Jakub Dawidek  *    notice, this list of conditions and the following disclaimer in the
121e09ff3dSPawel Jakub Dawidek  *    documentation and/or other materials provided with the distribution.
131e09ff3dSPawel Jakub Dawidek  *
141e09ff3dSPawel Jakub Dawidek  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
151e09ff3dSPawel Jakub Dawidek  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
161e09ff3dSPawel Jakub Dawidek  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
171e09ff3dSPawel Jakub Dawidek  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
181e09ff3dSPawel Jakub Dawidek  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
191e09ff3dSPawel Jakub Dawidek  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
201e09ff3dSPawel Jakub Dawidek  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
211e09ff3dSPawel Jakub Dawidek  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
221e09ff3dSPawel Jakub Dawidek  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
231e09ff3dSPawel Jakub Dawidek  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
241e09ff3dSPawel Jakub Dawidek  * SUCH DAMAGE.
251e09ff3dSPawel Jakub Dawidek  */
261e09ff3dSPawel Jakub Dawidek 
271e09ff3dSPawel Jakub Dawidek #include <sys/cdefs.h>
281e09ff3dSPawel Jakub Dawidek __FBSDID("$FreeBSD$");
291e09ff3dSPawel Jakub Dawidek 
301e09ff3dSPawel Jakub Dawidek #include <sys/param.h>
314332fecaSAllan Jude #ifdef _KERNEL
321e09ff3dSPawel Jakub Dawidek #include <sys/kernel.h>
331e09ff3dSPawel Jakub Dawidek #include <sys/malloc.h>
341e09ff3dSPawel Jakub Dawidek #include <sys/sysctl.h>
351e09ff3dSPawel Jakub Dawidek #include <sys/systm.h>
364332fecaSAllan Jude #endif /* _KERNEL */
374332fecaSAllan Jude #include <sys/queue.h>
381e09ff3dSPawel Jakub Dawidek #include <sys/tree.h>
391e09ff3dSPawel Jakub Dawidek 
401e09ff3dSPawel Jakub Dawidek #include <geom/geom.h>
411e09ff3dSPawel Jakub Dawidek 
421e09ff3dSPawel Jakub Dawidek #include <geom/eli/g_eli.h>
431e09ff3dSPawel Jakub Dawidek 
444332fecaSAllan Jude #ifdef _KERNEL
451e09ff3dSPawel Jakub Dawidek MALLOC_DECLARE(M_ELI);
461e09ff3dSPawel Jakub Dawidek 
471e09ff3dSPawel Jakub Dawidek SYSCTL_DECL(_kern_geom_eli);
481e09ff3dSPawel Jakub Dawidek /*
491e09ff3dSPawel Jakub Dawidek  * The default limit (8192 keys) will allow to cache all keys for 4TB
501e09ff3dSPawel Jakub Dawidek  * provider with 512 bytes sectors and will take around 1MB of memory.
511e09ff3dSPawel Jakub Dawidek  */
521e09ff3dSPawel Jakub Dawidek static u_int g_eli_key_cache_limit = 8192;
531e09ff3dSPawel Jakub Dawidek SYSCTL_UINT(_kern_geom_eli, OID_AUTO, key_cache_limit, CTLFLAG_RDTUN,
541e09ff3dSPawel Jakub Dawidek     &g_eli_key_cache_limit, 0, "Maximum number of encryption keys to cache");
551e09ff3dSPawel Jakub Dawidek static uint64_t g_eli_key_cache_hits;
561e09ff3dSPawel Jakub Dawidek SYSCTL_UQUAD(_kern_geom_eli, OID_AUTO, key_cache_hits, CTLFLAG_RW,
571e09ff3dSPawel Jakub Dawidek     &g_eli_key_cache_hits, 0, "Key cache hits");
581e09ff3dSPawel Jakub Dawidek static uint64_t g_eli_key_cache_misses;
591e09ff3dSPawel Jakub Dawidek SYSCTL_UQUAD(_kern_geom_eli, OID_AUTO, key_cache_misses, CTLFLAG_RW,
601e09ff3dSPawel Jakub Dawidek     &g_eli_key_cache_misses, 0, "Key cache misses");
611e09ff3dSPawel Jakub Dawidek 
624332fecaSAllan Jude #endif /* _KERNEL */
631e09ff3dSPawel Jakub Dawidek 
641e09ff3dSPawel Jakub Dawidek static int
651e09ff3dSPawel Jakub Dawidek g_eli_key_cmp(const struct g_eli_key *a, const struct g_eli_key *b)
661e09ff3dSPawel Jakub Dawidek {
671e09ff3dSPawel Jakub Dawidek 
681e09ff3dSPawel Jakub Dawidek 	if (a->gek_keyno > b->gek_keyno)
691e09ff3dSPawel Jakub Dawidek 		return (1);
701e09ff3dSPawel Jakub Dawidek 	else if (a->gek_keyno < b->gek_keyno)
711e09ff3dSPawel Jakub Dawidek 		return (-1);
721e09ff3dSPawel Jakub Dawidek 	return (0);
731e09ff3dSPawel Jakub Dawidek }
741e09ff3dSPawel Jakub Dawidek 
754332fecaSAllan Jude void
761e09ff3dSPawel Jakub Dawidek g_eli_key_fill(struct g_eli_softc *sc, struct g_eli_key *key, uint64_t keyno)
771e09ff3dSPawel Jakub Dawidek {
78457bbc4fSPawel Jakub Dawidek 	const uint8_t *ekey;
791e09ff3dSPawel Jakub Dawidek 	struct {
801e09ff3dSPawel Jakub Dawidek 		char magic[4];
811e09ff3dSPawel Jakub Dawidek 		uint8_t keyno[8];
821e09ff3dSPawel Jakub Dawidek 	} __packed hmacdata;
831e09ff3dSPawel Jakub Dawidek 
84457bbc4fSPawel Jakub Dawidek 	if ((sc->sc_flags & G_ELI_FLAG_ENC_IVKEY) != 0)
85457bbc4fSPawel Jakub Dawidek 		ekey = sc->sc_mkey;
86457bbc4fSPawel Jakub Dawidek 	else
87457bbc4fSPawel Jakub Dawidek 		ekey = sc->sc_ekey;
88457bbc4fSPawel Jakub Dawidek 
891e09ff3dSPawel Jakub Dawidek 	bcopy("ekey", hmacdata.magic, 4);
901e09ff3dSPawel Jakub Dawidek 	le64enc(hmacdata.keyno, keyno);
91457bbc4fSPawel Jakub Dawidek 	g_eli_crypto_hmac(ekey, G_ELI_MAXKEYLEN, (uint8_t *)&hmacdata,
921e09ff3dSPawel Jakub Dawidek 	    sizeof(hmacdata), key->gek_key, 0);
931e09ff3dSPawel Jakub Dawidek 	key->gek_keyno = keyno;
941e09ff3dSPawel Jakub Dawidek 	key->gek_count = 0;
959104c920SPawel Jakub Dawidek 	key->gek_magic = G_ELI_KEY_MAGIC;
961e09ff3dSPawel Jakub Dawidek }
971e09ff3dSPawel Jakub Dawidek 
984332fecaSAllan Jude #ifdef _KERNEL
994332fecaSAllan Jude RB_PROTOTYPE(g_eli_key_tree, g_eli_key, gek_link, g_eli_key_cmp);
1004332fecaSAllan Jude RB_GENERATE(g_eli_key_tree, g_eli_key, gek_link, g_eli_key_cmp);
1014332fecaSAllan Jude 
1021e09ff3dSPawel Jakub Dawidek static struct g_eli_key *
1031e09ff3dSPawel Jakub Dawidek g_eli_key_allocate(struct g_eli_softc *sc, uint64_t keyno)
1041e09ff3dSPawel Jakub Dawidek {
1051e09ff3dSPawel Jakub Dawidek 	struct g_eli_key *key, *ekey, keysearch;
1061e09ff3dSPawel Jakub Dawidek 
1071e09ff3dSPawel Jakub Dawidek 	mtx_assert(&sc->sc_ekeys_lock, MA_OWNED);
1081e09ff3dSPawel Jakub Dawidek 	mtx_unlock(&sc->sc_ekeys_lock);
1091e09ff3dSPawel Jakub Dawidek 
1101e09ff3dSPawel Jakub Dawidek 	key = malloc(sizeof(*key), M_ELI, M_WAITOK);
1111e09ff3dSPawel Jakub Dawidek 	g_eli_key_fill(sc, key, keyno);
1121e09ff3dSPawel Jakub Dawidek 
1131e09ff3dSPawel Jakub Dawidek 	mtx_lock(&sc->sc_ekeys_lock);
1141e09ff3dSPawel Jakub Dawidek 	/*
1151e09ff3dSPawel Jakub Dawidek 	 * Recheck if the key wasn't added while we weren't holding the lock.
1161e09ff3dSPawel Jakub Dawidek 	 */
1171e09ff3dSPawel Jakub Dawidek 	keysearch.gek_keyno = keyno;
1181e09ff3dSPawel Jakub Dawidek 	ekey = RB_FIND(g_eli_key_tree, &sc->sc_ekeys_tree, &keysearch);
1191e09ff3dSPawel Jakub Dawidek 	if (ekey != NULL) {
120*39b7ca45SAllan Jude 		explicit_bzero(key, sizeof(*key));
12176cc7f6dSMikolaj Golub 		free(key, M_ELI);
1221e09ff3dSPawel Jakub Dawidek 		key = ekey;
1231e09ff3dSPawel Jakub Dawidek 		TAILQ_REMOVE(&sc->sc_ekeys_queue, key, gek_next);
1241e09ff3dSPawel Jakub Dawidek 	} else {
1251e09ff3dSPawel Jakub Dawidek 		RB_INSERT(g_eli_key_tree, &sc->sc_ekeys_tree, key);
1261e09ff3dSPawel Jakub Dawidek 		sc->sc_ekeys_allocated++;
1271e09ff3dSPawel Jakub Dawidek 	}
1281e09ff3dSPawel Jakub Dawidek 	TAILQ_INSERT_TAIL(&sc->sc_ekeys_queue, key, gek_next);
1291e09ff3dSPawel Jakub Dawidek 
1301e09ff3dSPawel Jakub Dawidek 	return (key);
1311e09ff3dSPawel Jakub Dawidek }
1321e09ff3dSPawel Jakub Dawidek 
1331e09ff3dSPawel Jakub Dawidek static struct g_eli_key *
1341e09ff3dSPawel Jakub Dawidek g_eli_key_find_last(struct g_eli_softc *sc)
1351e09ff3dSPawel Jakub Dawidek {
1361e09ff3dSPawel Jakub Dawidek 	struct g_eli_key *key;
1371e09ff3dSPawel Jakub Dawidek 
1381e09ff3dSPawel Jakub Dawidek 	mtx_assert(&sc->sc_ekeys_lock, MA_OWNED);
1391e09ff3dSPawel Jakub Dawidek 
1401e09ff3dSPawel Jakub Dawidek 	TAILQ_FOREACH(key, &sc->sc_ekeys_queue, gek_next) {
1411e09ff3dSPawel Jakub Dawidek 		if (key->gek_count == 0)
1421e09ff3dSPawel Jakub Dawidek 			break;
1431e09ff3dSPawel Jakub Dawidek 	}
1441e09ff3dSPawel Jakub Dawidek 
1451e09ff3dSPawel Jakub Dawidek 	return (key);
1461e09ff3dSPawel Jakub Dawidek }
1471e09ff3dSPawel Jakub Dawidek 
1481e09ff3dSPawel Jakub Dawidek static void
1491e09ff3dSPawel Jakub Dawidek g_eli_key_replace(struct g_eli_softc *sc, struct g_eli_key *key, uint64_t keyno)
1501e09ff3dSPawel Jakub Dawidek {
1511e09ff3dSPawel Jakub Dawidek 
1521e09ff3dSPawel Jakub Dawidek 	mtx_assert(&sc->sc_ekeys_lock, MA_OWNED);
1539104c920SPawel Jakub Dawidek 	KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, ("Invalid magic."));
1541e09ff3dSPawel Jakub Dawidek 
1551e09ff3dSPawel Jakub Dawidek 	RB_REMOVE(g_eli_key_tree, &sc->sc_ekeys_tree, key);
1561e09ff3dSPawel Jakub Dawidek 	TAILQ_REMOVE(&sc->sc_ekeys_queue, key, gek_next);
1571e09ff3dSPawel Jakub Dawidek 
1581e09ff3dSPawel Jakub Dawidek 	KASSERT(key->gek_count == 0, ("gek_count=%d", key->gek_count));
1591e09ff3dSPawel Jakub Dawidek 
1601e09ff3dSPawel Jakub Dawidek 	g_eli_key_fill(sc, key, keyno);
1611e09ff3dSPawel Jakub Dawidek 
1621e09ff3dSPawel Jakub Dawidek 	RB_INSERT(g_eli_key_tree, &sc->sc_ekeys_tree, key);
1631e09ff3dSPawel Jakub Dawidek 	TAILQ_INSERT_TAIL(&sc->sc_ekeys_queue, key, gek_next);
1641e09ff3dSPawel Jakub Dawidek }
1651e09ff3dSPawel Jakub Dawidek 
1661e09ff3dSPawel Jakub Dawidek static void
1671e09ff3dSPawel Jakub Dawidek g_eli_key_remove(struct g_eli_softc *sc, struct g_eli_key *key)
1681e09ff3dSPawel Jakub Dawidek {
1691e09ff3dSPawel Jakub Dawidek 
1701e09ff3dSPawel Jakub Dawidek 	mtx_assert(&sc->sc_ekeys_lock, MA_OWNED);
1719104c920SPawel Jakub Dawidek 	KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, ("Invalid magic."));
1721e09ff3dSPawel Jakub Dawidek 	KASSERT(key->gek_count == 0, ("gek_count=%d", key->gek_count));
1731e09ff3dSPawel Jakub Dawidek 
1741e09ff3dSPawel Jakub Dawidek 	RB_REMOVE(g_eli_key_tree, &sc->sc_ekeys_tree, key);
1751e09ff3dSPawel Jakub Dawidek 	TAILQ_REMOVE(&sc->sc_ekeys_queue, key, gek_next);
1761e09ff3dSPawel Jakub Dawidek 	sc->sc_ekeys_allocated--;
177*39b7ca45SAllan Jude 	explicit_bzero(key, sizeof(*key));
1781e09ff3dSPawel Jakub Dawidek 	free(key, M_ELI);
1791e09ff3dSPawel Jakub Dawidek }
1801e09ff3dSPawel Jakub Dawidek 
1811e09ff3dSPawel Jakub Dawidek void
1821e09ff3dSPawel Jakub Dawidek g_eli_key_init(struct g_eli_softc *sc)
1831e09ff3dSPawel Jakub Dawidek {
1841e09ff3dSPawel Jakub Dawidek 	uint8_t *mkey;
1851e09ff3dSPawel Jakub Dawidek 
1869d180439SPawel Jakub Dawidek 	mtx_lock(&sc->sc_ekeys_lock);
1871e09ff3dSPawel Jakub Dawidek 
1889d180439SPawel Jakub Dawidek 	mkey = sc->sc_mkey + sizeof(sc->sc_ivkey);
1891e09ff3dSPawel Jakub Dawidek 	if ((sc->sc_flags & G_ELI_FLAG_AUTH) == 0)
1901e09ff3dSPawel Jakub Dawidek 		bcopy(mkey, sc->sc_ekey, G_ELI_DATAKEYLEN);
1911e09ff3dSPawel Jakub Dawidek 	else {
1921e09ff3dSPawel Jakub Dawidek 		/*
1933d47ea33SPawel Jakub Dawidek 		 * The encryption key is: ekey = HMAC_SHA512(Data-Key, 0x10)
1941e09ff3dSPawel Jakub Dawidek 		 */
1951e09ff3dSPawel Jakub Dawidek 		g_eli_crypto_hmac(mkey, G_ELI_MAXKEYLEN, "\x10", 1,
1961e09ff3dSPawel Jakub Dawidek 		    sc->sc_ekey, 0);
1971e09ff3dSPawel Jakub Dawidek 	}
1989d180439SPawel Jakub Dawidek 
1999d180439SPawel Jakub Dawidek 	if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0) {
2009d180439SPawel Jakub Dawidek 		sc->sc_ekeys_total = 1;
2019d180439SPawel Jakub Dawidek 		sc->sc_ekeys_allocated = 0;
2021e09ff3dSPawel Jakub Dawidek 	} else {
2031e09ff3dSPawel Jakub Dawidek 		off_t mediasize;
2041e09ff3dSPawel Jakub Dawidek 		size_t blocksize;
2051e09ff3dSPawel Jakub Dawidek 
2061e09ff3dSPawel Jakub Dawidek 		if ((sc->sc_flags & G_ELI_FLAG_AUTH) != 0) {
2071e09ff3dSPawel Jakub Dawidek 			struct g_provider *pp;
2081e09ff3dSPawel Jakub Dawidek 
2091e09ff3dSPawel Jakub Dawidek 			pp = LIST_FIRST(&sc->sc_geom->consumer)->provider;
2101e09ff3dSPawel Jakub Dawidek 			mediasize = pp->mediasize;
2111e09ff3dSPawel Jakub Dawidek 			blocksize = pp->sectorsize;
2121e09ff3dSPawel Jakub Dawidek 		} else {
2131e09ff3dSPawel Jakub Dawidek 			mediasize = sc->sc_mediasize;
2141e09ff3dSPawel Jakub Dawidek 			blocksize = sc->sc_sectorsize;
2151e09ff3dSPawel Jakub Dawidek 		}
2161e09ff3dSPawel Jakub Dawidek 		sc->sc_ekeys_total =
2171e09ff3dSPawel Jakub Dawidek 		    ((mediasize - 1) >> G_ELI_KEY_SHIFT) / blocksize + 1;
2181e09ff3dSPawel Jakub Dawidek 		sc->sc_ekeys_allocated = 0;
2191e09ff3dSPawel Jakub Dawidek 		TAILQ_INIT(&sc->sc_ekeys_queue);
2201e09ff3dSPawel Jakub Dawidek 		RB_INIT(&sc->sc_ekeys_tree);
2215bd8adc7SPawel Jakub Dawidek 		if (sc->sc_ekeys_total <= g_eli_key_cache_limit) {
2225bd8adc7SPawel Jakub Dawidek 			uint64_t keyno;
2235bd8adc7SPawel Jakub Dawidek 
2245bd8adc7SPawel Jakub Dawidek 			for (keyno = 0; keyno < sc->sc_ekeys_total; keyno++)
2255bd8adc7SPawel Jakub Dawidek 				(void)g_eli_key_allocate(sc, keyno);
2265bd8adc7SPawel Jakub Dawidek 			KASSERT(sc->sc_ekeys_total == sc->sc_ekeys_allocated,
2275bd8adc7SPawel Jakub Dawidek 			    ("sc_ekeys_total=%ju != sc_ekeys_allocated=%ju",
2285bd8adc7SPawel Jakub Dawidek 			    (uintmax_t)sc->sc_ekeys_total,
2295bd8adc7SPawel Jakub Dawidek 			    (uintmax_t)sc->sc_ekeys_allocated));
2305bd8adc7SPawel Jakub Dawidek 		}
2311e09ff3dSPawel Jakub Dawidek 	}
2329d180439SPawel Jakub Dawidek 
2331e09ff3dSPawel Jakub Dawidek 	mtx_unlock(&sc->sc_ekeys_lock);
2341e09ff3dSPawel Jakub Dawidek }
2351e09ff3dSPawel Jakub Dawidek 
2361e09ff3dSPawel Jakub Dawidek void
2371e09ff3dSPawel Jakub Dawidek g_eli_key_destroy(struct g_eli_softc *sc)
2381e09ff3dSPawel Jakub Dawidek {
2391e09ff3dSPawel Jakub Dawidek 
2401e09ff3dSPawel Jakub Dawidek 	mtx_lock(&sc->sc_ekeys_lock);
2411e09ff3dSPawel Jakub Dawidek 	if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0) {
242*39b7ca45SAllan Jude 		explicit_bzero(sc->sc_ekey, sizeof(sc->sc_ekey));
2431e09ff3dSPawel Jakub Dawidek 	} else {
2441e09ff3dSPawel Jakub Dawidek 		struct g_eli_key *key;
2451e09ff3dSPawel Jakub Dawidek 
2461e09ff3dSPawel Jakub Dawidek 		while ((key = TAILQ_FIRST(&sc->sc_ekeys_queue)) != NULL)
2471e09ff3dSPawel Jakub Dawidek 			g_eli_key_remove(sc, key);
2481e09ff3dSPawel Jakub Dawidek 		TAILQ_INIT(&sc->sc_ekeys_queue);
2491e09ff3dSPawel Jakub Dawidek 		RB_INIT(&sc->sc_ekeys_tree);
2501e09ff3dSPawel Jakub Dawidek 	}
2511e09ff3dSPawel Jakub Dawidek 	mtx_unlock(&sc->sc_ekeys_lock);
2521e09ff3dSPawel Jakub Dawidek }
2531e09ff3dSPawel Jakub Dawidek 
2541e09ff3dSPawel Jakub Dawidek /*
2551e09ff3dSPawel Jakub Dawidek  * Select encryption key. If G_ELI_FLAG_SINGLE_KEY is present we only have one
2561e09ff3dSPawel Jakub Dawidek  * key available for all the data. If the flag is not present select the key
2571e09ff3dSPawel Jakub Dawidek  * based on data offset.
2581e09ff3dSPawel Jakub Dawidek  */
2591e09ff3dSPawel Jakub Dawidek uint8_t *
2601e09ff3dSPawel Jakub Dawidek g_eli_key_hold(struct g_eli_softc *sc, off_t offset, size_t blocksize)
2611e09ff3dSPawel Jakub Dawidek {
2621e09ff3dSPawel Jakub Dawidek 	struct g_eli_key *key, keysearch;
2631e09ff3dSPawel Jakub Dawidek 	uint64_t keyno;
2641e09ff3dSPawel Jakub Dawidek 
2651e09ff3dSPawel Jakub Dawidek 	if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0)
2661e09ff3dSPawel Jakub Dawidek 		return (sc->sc_ekey);
2671e09ff3dSPawel Jakub Dawidek 
2681e09ff3dSPawel Jakub Dawidek 	/* We switch key every 2^G_ELI_KEY_SHIFT blocks. */
2691e09ff3dSPawel Jakub Dawidek 	keyno = (offset >> G_ELI_KEY_SHIFT) / blocksize;
2701e09ff3dSPawel Jakub Dawidek 
2711e09ff3dSPawel Jakub Dawidek 	KASSERT(keyno < sc->sc_ekeys_total,
2721e09ff3dSPawel Jakub Dawidek 	    ("%s: keyno=%ju >= sc_ekeys_total=%ju",
2731e09ff3dSPawel Jakub Dawidek 	    __func__, (uintmax_t)keyno, (uintmax_t)sc->sc_ekeys_total));
2741e09ff3dSPawel Jakub Dawidek 
2751e09ff3dSPawel Jakub Dawidek 	keysearch.gek_keyno = keyno;
2761e09ff3dSPawel Jakub Dawidek 
2775bd8adc7SPawel Jakub Dawidek 	if (sc->sc_ekeys_total == sc->sc_ekeys_allocated) {
2785bd8adc7SPawel Jakub Dawidek 		/* We have all the keys, so avoid some overhead. */
2795bd8adc7SPawel Jakub Dawidek 		key = RB_FIND(g_eli_key_tree, &sc->sc_ekeys_tree, &keysearch);
2805bd8adc7SPawel Jakub Dawidek 		KASSERT(key != NULL, ("No key %ju found.", (uintmax_t)keyno));
2819104c920SPawel Jakub Dawidek 		KASSERT(key->gek_magic == G_ELI_KEY_MAGIC,
2829104c920SPawel Jakub Dawidek 		    ("Invalid key magic."));
2835bd8adc7SPawel Jakub Dawidek 		return (key->gek_key);
2845bd8adc7SPawel Jakub Dawidek 	}
2855bd8adc7SPawel Jakub Dawidek 
2861e09ff3dSPawel Jakub Dawidek 	mtx_lock(&sc->sc_ekeys_lock);
2871e09ff3dSPawel Jakub Dawidek 	key = RB_FIND(g_eli_key_tree, &sc->sc_ekeys_tree, &keysearch);
2881e09ff3dSPawel Jakub Dawidek 	if (key != NULL) {
2891e09ff3dSPawel Jakub Dawidek 		g_eli_key_cache_hits++;
2901e09ff3dSPawel Jakub Dawidek 		TAILQ_REMOVE(&sc->sc_ekeys_queue, key, gek_next);
2911e09ff3dSPawel Jakub Dawidek 		TAILQ_INSERT_TAIL(&sc->sc_ekeys_queue, key, gek_next);
2921e09ff3dSPawel Jakub Dawidek 	} else {
2931e09ff3dSPawel Jakub Dawidek 		/*
2941e09ff3dSPawel Jakub Dawidek 		 * No key in cache, find the least recently unreferenced key
2951e09ff3dSPawel Jakub Dawidek 		 * or allocate one if we haven't reached our limit yet.
2961e09ff3dSPawel Jakub Dawidek 		 */
2971e09ff3dSPawel Jakub Dawidek 		if (sc->sc_ekeys_allocated < g_eli_key_cache_limit) {
2981e09ff3dSPawel Jakub Dawidek 			key = g_eli_key_allocate(sc, keyno);
2991e09ff3dSPawel Jakub Dawidek 		} else {
3001e09ff3dSPawel Jakub Dawidek 			g_eli_key_cache_misses++;
3011e09ff3dSPawel Jakub Dawidek 			key = g_eli_key_find_last(sc);
3021e09ff3dSPawel Jakub Dawidek 			if (key != NULL) {
3031e09ff3dSPawel Jakub Dawidek 				g_eli_key_replace(sc, key, keyno);
3041e09ff3dSPawel Jakub Dawidek 			} else {
3051e09ff3dSPawel Jakub Dawidek 				/* All keys are referenced? Allocate one. */
3061e09ff3dSPawel Jakub Dawidek 				key = g_eli_key_allocate(sc, keyno);
3071e09ff3dSPawel Jakub Dawidek 			}
3081e09ff3dSPawel Jakub Dawidek 		}
3091e09ff3dSPawel Jakub Dawidek 	}
3101e09ff3dSPawel Jakub Dawidek 	key->gek_count++;
3111e09ff3dSPawel Jakub Dawidek 	mtx_unlock(&sc->sc_ekeys_lock);
3121e09ff3dSPawel Jakub Dawidek 
3139104c920SPawel Jakub Dawidek 	KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, ("Invalid key magic."));
3149104c920SPawel Jakub Dawidek 
3151e09ff3dSPawel Jakub Dawidek 	return (key->gek_key);
3161e09ff3dSPawel Jakub Dawidek }
3171e09ff3dSPawel Jakub Dawidek 
3181e09ff3dSPawel Jakub Dawidek void
3191e09ff3dSPawel Jakub Dawidek g_eli_key_drop(struct g_eli_softc *sc, uint8_t *rawkey)
3201e09ff3dSPawel Jakub Dawidek {
3211e09ff3dSPawel Jakub Dawidek 	struct g_eli_key *key = (struct g_eli_key *)rawkey;
3221e09ff3dSPawel Jakub Dawidek 
3231e09ff3dSPawel Jakub Dawidek 	if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) != 0)
3241e09ff3dSPawel Jakub Dawidek 		return;
3251e09ff3dSPawel Jakub Dawidek 
3269104c920SPawel Jakub Dawidek 	KASSERT(key->gek_magic == G_ELI_KEY_MAGIC, ("Invalid key magic."));
3279104c920SPawel Jakub Dawidek 
3285bd8adc7SPawel Jakub Dawidek 	if (sc->sc_ekeys_total == sc->sc_ekeys_allocated)
3295bd8adc7SPawel Jakub Dawidek 		return;
3305bd8adc7SPawel Jakub Dawidek 
3311e09ff3dSPawel Jakub Dawidek 	mtx_lock(&sc->sc_ekeys_lock);
3321e09ff3dSPawel Jakub Dawidek 	KASSERT(key->gek_count > 0, ("key->gek_count=%d", key->gek_count));
3331e09ff3dSPawel Jakub Dawidek 	key->gek_count--;
3341e09ff3dSPawel Jakub Dawidek 	while (sc->sc_ekeys_allocated > g_eli_key_cache_limit) {
3351e09ff3dSPawel Jakub Dawidek 		key = g_eli_key_find_last(sc);
3361e09ff3dSPawel Jakub Dawidek 		if (key == NULL)
3371e09ff3dSPawel Jakub Dawidek 			break;
3381e09ff3dSPawel Jakub Dawidek 		g_eli_key_remove(sc, key);
3391e09ff3dSPawel Jakub Dawidek 	}
3401e09ff3dSPawel Jakub Dawidek 	mtx_unlock(&sc->sc_ekeys_lock);
3411e09ff3dSPawel Jakub Dawidek }
3424332fecaSAllan Jude #endif /* _KERNEL */
343