xref: /titanic_52/usr/src/uts/common/crypto/io/swrand.c (revision f79b63829052fcaae660db7f78ce17efd13e15bc)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * Software based random number provider for the Kernel Cryptographic
28  * Framework (KCF). This provider periodically collects unpredictable input
29  * from external sources and processes it into a pool of entropy (randomness)
30  * in order to satisfy requests for random bits from kCF. It implements
31  * software-based mixing, extraction, and generation algorithms.
32  *
33  * A history note: The software-based algorithms in this file used to be
34  * part of the /dev/random driver.
35  */
36 
37 #include <sys/types.h>
38 #include <sys/errno.h>
39 #include <sys/debug.h>
40 #include <vm/seg_kmem.h>
41 #include <vm/hat.h>
42 #include <sys/systm.h>
43 #include <sys/memlist.h>
44 #include <sys/cmn_err.h>
45 #include <sys/ksynch.h>
46 #include <sys/random.h>
47 #include <sys/ddi.h>
48 #include <sys/mman.h>
49 #include <sys/sysmacros.h>
50 #include <sys/mem_config.h>
51 #include <sys/time.h>
52 #include <sys/crypto/spi.h>
53 #include <sys/sha1.h>
54 #include <sys/sunddi.h>
55 #include <sys/modctl.h>
56 #include <sys/hold_page.h>
57 #include <rng/fips_random.h>
58 
59 #define	RNDPOOLSIZE		1024	/* Pool size in bytes */
60 #define	HASHBUFSIZE		64	/* Buffer size used for pool mixing */
61 #define	MAXMEMBLOCKS		16384	/* Number of memory blocks to scan */
62 #define	MEMBLOCKSIZE		4096	/* Size of memory block to read */
63 #define	MINEXTRACTBITS		160	/* Min entropy level for extraction */
64 #define	TIMEOUT_INTERVAL	5	/* Periodic mixing interval in secs */
65 
66 /* Hash-algo generic definitions. For now, they are SHA1's. */
67 #define	HASHSIZE		20
68 #define	HASH_CTX		SHA1_CTX
69 #define	HashInit(ctx)		SHA1Init((ctx))
70 #define	HashUpdate(ctx, p, s)	SHA1Update((ctx), (p), (s))
71 #define	HashFinal(d, ctx)	SHA1Final((d), (ctx))
72 
73 /* Physical memory entropy source */
74 typedef struct physmem_entsrc_s {
75 	uint8_t *parity;		/* parity bit vector */
76 	caddr_t pmbuf;			/* buffer for memory block */
77 	uint32_t nblocks;		/* number of  memory blocks */
78 	int entperblock;		/* entropy bits per block read */
79 	hrtime_t last_diff;		/* previous time to process a block */
80 	hrtime_t last_delta;		/* previous time delta */
81 	hrtime_t last_delta2;		/* previous 2nd order time delta */
82 } physmem_entsrc_t;
83 
84 static uint32_t srndpool[RNDPOOLSIZE/4];	/* Pool of random bits */
85 static uint32_t buffer[RNDPOOLSIZE/4];	/* entropy mixed in later */
86 static int buffer_bytes;		/* bytes written to buffer */
87 static uint32_t entropy_bits;		/* pool's current amount of entropy */
88 static kmutex_t srndpool_lock;		/* protects r/w accesses to the pool, */
89 					/* and the global variables */
90 static kmutex_t buffer_lock;		/* protects r/w accesses to buffer */
91 static kcondvar_t srndpool_read_cv;	/* serializes poll/read syscalls */
92 static int pindex;			/* Global index for adding/extracting */
93 					/* from the pool */
94 static int bstart, bindex;		/* Global vars for adding/extracting */
95 					/* from the buffer */
96 static uint8_t leftover[HASHSIZE];	/* leftover output */
97 static uint32_t	swrand_XKEY[6];		/* one extra word for getentropy */
98 static int leftover_bytes;		/* leftover length */
99 static uint32_t previous_bytes[HASHSIZE/BYTES_IN_WORD];	/* prev random bytes */
100 
101 static physmem_entsrc_t entsrc;		/* Physical mem as an entropy source */
102 static timeout_id_t rnd_timeout_id;
103 static int snum_waiters;
104 static crypto_kcf_provider_handle_t swrand_prov_handle = NULL;
105 swrand_stats_t swrand_stats;
106 
107 static int physmem_ent_init(physmem_entsrc_t *);
108 static void physmem_ent_fini(physmem_entsrc_t *);
109 static void physmem_ent_gen(physmem_entsrc_t *);
110 static int physmem_parity_update(uint8_t *, uint32_t, int);
111 static void physmem_count_blocks();
112 static void rnd_dr_callback_post_add(void *, pgcnt_t);
113 static int rnd_dr_callback_pre_del(void *, pgcnt_t);
114 static void rnd_dr_callback_post_del(void *, pgcnt_t, int);
115 static void rnd_handler(void *arg);
116 static void swrand_init();
117 static void swrand_schedule_timeout(void);
118 static int swrand_get_entropy(uint8_t *ptr, size_t len, boolean_t);
119 static void swrand_add_entropy(uint8_t *ptr, size_t len, uint16_t entropy_est);
120 static void swrand_add_entropy_later(uint8_t *ptr, size_t len);
121 
122 /* Dynamic Reconfiguration related declarations */
123 kphysm_setup_vector_t rnd_dr_callback_vec = {
124 	KPHYSM_SETUP_VECTOR_VERSION,
125 	rnd_dr_callback_post_add,
126 	rnd_dr_callback_pre_del,
127 	rnd_dr_callback_post_del
128 };
129 
130 extern struct mod_ops mod_cryptoops;
131 
132 /*
133  * Module linkage information for the kernel.
134  */
135 static struct modlcrypto modlcrypto = {
136 	&mod_cryptoops,
137 	"Kernel Random number Provider"
138 };
139 
140 static struct modlinkage modlinkage = {
141 	MODREV_1,
142 	(void *)&modlcrypto,
143 	NULL
144 };
145 
146 /*
147  * CSPI information (entry points, provider info, etc.)
148  */
149 static void swrand_provider_status(crypto_provider_handle_t, uint_t *);
150 
151 static crypto_control_ops_t swrand_control_ops = {
152 	swrand_provider_status
153 };
154 
155 static int swrand_seed_random(crypto_provider_handle_t, crypto_session_id_t,
156     uchar_t *, size_t, uint_t, uint32_t, crypto_req_handle_t);
157 static int swrand_generate_random(crypto_provider_handle_t,
158     crypto_session_id_t, uchar_t *, size_t, crypto_req_handle_t);
159 
160 static crypto_random_number_ops_t swrand_random_number_ops = {
161 	swrand_seed_random,
162 	swrand_generate_random
163 };
164 
165 static void swrand_POST(int *);
166 
167 static crypto_fips140_ops_t swrand_fips140_ops = {
168 	swrand_POST
169 };
170 
171 static crypto_ops_t swrand_crypto_ops = {
172 	&swrand_control_ops,
173 	NULL,
174 	NULL,
175 	NULL,
176 	NULL,
177 	NULL,
178 	NULL,
179 	NULL,
180 	&swrand_random_number_ops,
181 	NULL,
182 	NULL,
183 	NULL,
184 	NULL,
185 	NULL,
186 	NULL,
187 	NULL,
188 	&swrand_fips140_ops
189 };
190 
191 static crypto_provider_info_t swrand_prov_info = {
192 	CRYPTO_SPI_VERSION_4,
193 	"Kernel Random Number Provider",
194 	CRYPTO_SW_PROVIDER,
195 	{&modlinkage},
196 	NULL,
197 	&swrand_crypto_ops,
198 	0,
199 	NULL
200 };
201 
202 int
203 _init(void)
204 {
205 	int ret;
206 	hrtime_t ts;
207 	time_t now;
208 
209 	/*
210 	 * Register with KCF. If the registration fails, return error.
211 	 */
212 	if ((ret = crypto_register_provider(&swrand_prov_info,
213 	    &swrand_prov_handle)) != CRYPTO_SUCCESS) {
214 		cmn_err(CE_WARN, "swrand : Kernel Random Number Provider "
215 		    "disabled for /dev/random use");
216 		return (EACCES);
217 	}
218 
219 	mutex_init(&srndpool_lock, NULL, MUTEX_DEFAULT, NULL);
220 	mutex_init(&buffer_lock, NULL, MUTEX_DEFAULT, NULL);
221 	cv_init(&srndpool_read_cv, NULL, CV_DEFAULT, NULL);
222 	entropy_bits = 0;
223 	pindex = 0;
224 	bindex = 0;
225 	bstart = 0;
226 	snum_waiters = 0;
227 	leftover_bytes = 0;
228 	buffer_bytes = 0;
229 
230 	/*
231 	 * Initialize the pool using
232 	 * . 2 unpredictable times: high resolution time since the boot-time,
233 	 *   and the current time-of-the day.
234 	 * . The initial physical memory state.
235 	 */
236 	ts = gethrtime();
237 	swrand_add_entropy((uint8_t *)&ts, sizeof (ts), 0);
238 
239 	(void) drv_getparm(TIME, &now);
240 	swrand_add_entropy((uint8_t *)&now, sizeof (now), 0);
241 
242 	ret = kphysm_setup_func_register(&rnd_dr_callback_vec, NULL);
243 	ASSERT(ret == 0);
244 
245 	if (physmem_ent_init(&entsrc) != 0) {
246 		mutex_destroy(&srndpool_lock);
247 		mutex_destroy(&buffer_lock);
248 		cv_destroy(&srndpool_read_cv);
249 		(void) crypto_unregister_provider(swrand_prov_handle);
250 		return (ENOMEM);
251 	}
252 
253 	if ((ret = mod_install(&modlinkage)) != 0) {
254 		mutex_destroy(&srndpool_lock);
255 		mutex_destroy(&buffer_lock);
256 		cv_destroy(&srndpool_read_cv);
257 		physmem_ent_fini(&entsrc);
258 		(void) crypto_unregister_provider(swrand_prov_handle);
259 		return (ret);
260 	}
261 
262 	/* Schedule periodic mixing of the pool. */
263 	mutex_enter(&srndpool_lock);
264 	swrand_schedule_timeout();
265 	mutex_exit(&srndpool_lock);
266 	(void) swrand_get_entropy((uint8_t *)swrand_XKEY, HASHSIZE, B_TRUE);
267 	bcopy(swrand_XKEY, previous_bytes, HASHSIZE);
268 
269 	return (0);
270 }
271 
272 int
273 _info(struct modinfo *modinfop)
274 {
275 	return (mod_info(&modlinkage, modinfop));
276 }
277 
278 /*
279  * Control entry points.
280  */
281 /* ARGSUSED */
282 static void
283 swrand_provider_status(crypto_provider_handle_t provider, uint_t *status)
284 {
285 	*status = CRYPTO_PROVIDER_READY;
286 }
287 
288 /*
289  * Random number entry points.
290  */
291 /* ARGSUSED */
292 static int
293 swrand_seed_random(crypto_provider_handle_t provider, crypto_session_id_t sid,
294     uchar_t *buf, size_t len, uint_t entropy_est, uint32_t flags,
295     crypto_req_handle_t req)
296 {
297 	/* The entropy estimate is always 0 in this path */
298 	if (flags & CRYPTO_SEED_NOW)
299 		swrand_add_entropy(buf, len, 0);
300 	else
301 		swrand_add_entropy_later(buf, len);
302 	return (CRYPTO_SUCCESS);
303 }
304 
305 /* ARGSUSED */
306 static int
307 swrand_generate_random(crypto_provider_handle_t provider,
308     crypto_session_id_t sid, uchar_t *buf, size_t len, crypto_req_handle_t req)
309 {
310 	if (crypto_kmflag(req) == KM_NOSLEEP)
311 		(void) swrand_get_entropy(buf, len, B_TRUE);
312 	else
313 		(void) swrand_get_entropy(buf, len, B_FALSE);
314 
315 	return (CRYPTO_SUCCESS);
316 }
317 
318 /*
319  * Extraction of entropy from the pool.
320  *
321  * Returns "len" random bytes in *ptr.
322  * Try to gather some more entropy by calling physmem_ent_gen() when less than
323  * MINEXTRACTBITS are present in the pool.
324  * Will block if not enough entropy was available and the call is blocking.
325  */
326 static int
327 swrand_get_entropy(uint8_t *ptr, size_t len, boolean_t nonblock)
328 {
329 	int i, bytes;
330 	HASH_CTX hashctx;
331 	uint8_t digest[HASHSIZE], *pool;
332 	uint32_t tempout[HASHSIZE/BYTES_IN_WORD];
333 	int size;
334 
335 	mutex_enter(&srndpool_lock);
336 	if (leftover_bytes > 0) {
337 		bytes = min(len, leftover_bytes);
338 		bcopy(leftover, ptr, bytes);
339 		len -= bytes;
340 		ptr += bytes;
341 		leftover_bytes -= bytes;
342 		if (leftover_bytes > 0)
343 			ovbcopy(leftover+bytes, leftover, leftover_bytes);
344 	}
345 
346 	while (len > 0) {
347 		/* Check if there is enough entropy */
348 		while (entropy_bits < MINEXTRACTBITS) {
349 
350 			physmem_ent_gen(&entsrc);
351 
352 			if (entropy_bits < MINEXTRACTBITS &&
353 			    nonblock == B_TRUE) {
354 				mutex_exit(&srndpool_lock);
355 				return (EAGAIN);
356 			}
357 
358 			if (entropy_bits < MINEXTRACTBITS) {
359 				ASSERT(nonblock == B_FALSE);
360 				snum_waiters++;
361 				if (cv_wait_sig(&srndpool_read_cv,
362 				    &srndpool_lock) == 0) {
363 					snum_waiters--;
364 					mutex_exit(&srndpool_lock);
365 					return (EINTR);
366 				}
367 				snum_waiters--;
368 			}
369 		}
370 
371 		/* Figure out how many bytes to extract */
372 		bytes = min(HASHSIZE, len);
373 		bytes = min(bytes, CRYPTO_BITS2BYTES(entropy_bits));
374 		entropy_bits -= CRYPTO_BYTES2BITS(bytes);
375 		BUMP_SWRAND_STATS(ss_entOut, CRYPTO_BYTES2BITS(bytes));
376 		swrand_stats.ss_entEst = entropy_bits;
377 
378 		/* Extract entropy by hashing pool content */
379 		HashInit(&hashctx);
380 		HashUpdate(&hashctx, (uint8_t *)srndpool, RNDPOOLSIZE);
381 		HashFinal(digest, &hashctx);
382 
383 		/*
384 		 * Feed the digest back into the pool so next
385 		 * extraction produces different result
386 		 */
387 		pool = (uint8_t *)srndpool;
388 		for (i = 0; i < HASHSIZE; i++) {
389 			pool[pindex++] ^= digest[i];
390 			/* pindex modulo RNDPOOLSIZE */
391 			pindex &= (RNDPOOLSIZE - 1);
392 		}
393 
394 		/* LINTED E_BAD_PTR_CAST_ALIGN */
395 		fips_random_inner(swrand_XKEY, tempout, (uint32_t *)digest);
396 
397 		if (len >= HASHSIZE) {
398 			size = HASHSIZE;
399 		} else {
400 			size = min(bytes, HASHSIZE);
401 		}
402 
403 		/*
404 		 * FIPS 140-2: Continuous RNG test - each generation
405 		 * of an n-bit block shall be compared with the previously
406 		 * generated block. Test shall fail if any two compared
407 		 * n-bit blocks are equal.
408 		 */
409 		for (i = 0; i < HASHSIZE/BYTES_IN_WORD; i++) {
410 			if (tempout[i] != previous_bytes[i])
411 				break;
412 		}
413 
414 		if (i == HASHSIZE/BYTES_IN_WORD) {
415 			cmn_err(CE_WARN, "swrand: The value of 160-bit block "
416 			    "random bytes are same as the previous one.\n");
417 			/* discard random bytes and return error */
418 			return (EIO);
419 		}
420 
421 		bcopy(tempout, previous_bytes, HASHSIZE);
422 
423 		bcopy(tempout, ptr, size);
424 		if (len < HASHSIZE) {
425 			leftover_bytes = HASHSIZE - bytes;
426 			bcopy((uint8_t *)tempout + bytes, leftover,
427 			    leftover_bytes);
428 		}
429 
430 		ptr += size;
431 		len -= size;
432 		BUMP_SWRAND_STATS(ss_bytesOut, size);
433 	}
434 
435 	/* Zero out sensitive information */
436 	bzero(digest, HASHSIZE);
437 	bzero(tempout, HASHSIZE);
438 	mutex_exit(&srndpool_lock);
439 	return (0);
440 }
441 
442 #define	SWRAND_ADD_BYTES(ptr, len, i, pool)		\
443 	ASSERT((ptr) != NULL && (len) > 0);		\
444 	BUMP_SWRAND_STATS(ss_bytesIn, (len));		\
445 	while ((len)--) {				\
446 		(pool)[(i)++] ^= *(ptr);		\
447 		(ptr)++;				\
448 		(i) &= (RNDPOOLSIZE - 1);		\
449 	}
450 
451 /* Write some more user-provided entropy to the pool */
452 static void
453 swrand_add_bytes(uint8_t *ptr, size_t len)
454 {
455 	uint8_t *pool = (uint8_t *)srndpool;
456 
457 	ASSERT(MUTEX_HELD(&srndpool_lock));
458 	SWRAND_ADD_BYTES(ptr, len, pindex, pool);
459 }
460 
461 /*
462  * Add bytes to buffer. Adding the buffer to the random pool
463  * is deferred until the random pool is mixed.
464  */
465 static void
466 swrand_add_bytes_later(uint8_t *ptr, size_t len)
467 {
468 	uint8_t *pool = (uint8_t *)buffer;
469 
470 	ASSERT(MUTEX_HELD(&buffer_lock));
471 	SWRAND_ADD_BYTES(ptr, len, bindex, pool);
472 	buffer_bytes += len;
473 }
474 
475 #undef SWRAND_ADD_BYTES
476 
477 /* Mix the pool */
478 static void
479 swrand_mix_pool(uint16_t entropy_est)
480 {
481 	int i, j, k, start;
482 	HASH_CTX hashctx;
483 	uint8_t digest[HASHSIZE];
484 	uint8_t *pool = (uint8_t *)srndpool;
485 	uint8_t *bp = (uint8_t *)buffer;
486 
487 	ASSERT(MUTEX_HELD(&srndpool_lock));
488 
489 	/* add deferred bytes */
490 	mutex_enter(&buffer_lock);
491 	if (buffer_bytes > 0) {
492 		if (buffer_bytes >= RNDPOOLSIZE) {
493 			for (i = 0; i < RNDPOOLSIZE/4; i++) {
494 				srndpool[i] ^= buffer[i];
495 				buffer[i] = 0;
496 			}
497 			bstart = bindex = 0;
498 		} else {
499 			for (i = 0; i < buffer_bytes; i++) {
500 				pool[pindex++] ^= bp[bstart];
501 				bp[bstart++] = 0;
502 				pindex &= (RNDPOOLSIZE - 1);
503 				bstart &= (RNDPOOLSIZE - 1);
504 			}
505 			ASSERT(bstart == bindex);
506 		}
507 		buffer_bytes = 0;
508 	}
509 	mutex_exit(&buffer_lock);
510 
511 	start = 0;
512 	for (i = 0; i < RNDPOOLSIZE/HASHSIZE + 1; i++) {
513 		HashInit(&hashctx);
514 
515 		/* Hash a buffer centered on a block in the pool */
516 		if (start + HASHBUFSIZE <= RNDPOOLSIZE)
517 			HashUpdate(&hashctx, &pool[start], HASHBUFSIZE);
518 		else {
519 			HashUpdate(&hashctx, &pool[start],
520 			    RNDPOOLSIZE - start);
521 			HashUpdate(&hashctx, pool,
522 			    HASHBUFSIZE - RNDPOOLSIZE + start);
523 		}
524 		HashFinal(digest, &hashctx);
525 
526 		/* XOR the hash result back into the block */
527 		k = (start + HASHSIZE) & (RNDPOOLSIZE - 1);
528 		for (j = 0; j < HASHSIZE; j++) {
529 			pool[k++] ^= digest[j];
530 			k &= (RNDPOOLSIZE - 1);
531 		}
532 
533 		/* Slide the hash buffer and repeat with next block */
534 		start = (start + HASHSIZE) & (RNDPOOLSIZE - 1);
535 	}
536 
537 	entropy_bits += entropy_est;
538 	if (entropy_bits > CRYPTO_BYTES2BITS(RNDPOOLSIZE))
539 		entropy_bits = CRYPTO_BYTES2BITS(RNDPOOLSIZE);
540 
541 	swrand_stats.ss_entEst = entropy_bits;
542 	BUMP_SWRAND_STATS(ss_entIn, entropy_est);
543 }
544 
545 static void
546 swrand_add_entropy_later(uint8_t *ptr, size_t len)
547 {
548 	mutex_enter(&buffer_lock);
549 	swrand_add_bytes_later(ptr, len);
550 	mutex_exit(&buffer_lock);
551 }
552 
553 static void
554 swrand_add_entropy(uint8_t *ptr, size_t len, uint16_t entropy_est)
555 {
556 	mutex_enter(&srndpool_lock);
557 	swrand_add_bytes(ptr, len);
558 	swrand_mix_pool(entropy_est);
559 	mutex_exit(&srndpool_lock);
560 }
561 
562 /*
563  * The physmem_* routines below generate entropy by reading blocks of
564  * physical memory.  Entropy is gathered in a couple of ways:
565  *
566  *  - By reading blocks of physical memory and detecting if changes
567  *    occurred in the blocks read.
568  *
569  *  - By measuring the time it takes to load and hash a block of memory
570  *    and computing the differences in the measured time.
571  *
572  * The first method was used in the CryptoRand implementation.  Physical
573  * memory is divided into blocks of fixed size.  A block of memory is
574  * chosen from the possible blocks and hashed to produce a digest.  This
575  * digest is then mixed into the pool.  A single bit from the digest is
576  * used as a parity bit or "checksum" and compared against the previous
577  * "checksum" computed for the block.  If the single-bit checksum has not
578  * changed, no entropy is credited to the pool.  If there is a change,
579  * then the assumption is that at least one bit in the block has changed.
580  * The possible locations within the memory block of where the bit change
581  * occurred is used as a measure of entropy.  For example, if a block
582  * size of 4096 bytes is used, about log_2(4096*8)=15 bits worth of
583  * entropy is available.  Because the single-bit checksum will miss half
584  * of the changes, the amount of entropy credited to the pool is doubled
585  * when a change is detected.  With a 4096 byte block size, a block
586  * change will add a total of 30 bits of entropy to the pool.
587  *
588  * The second method measures the amount of time it takes to read and
589  * hash a physical memory block (as described above).  The time measured
590  * can vary depending on system load, scheduling and other factors.
591  * Differences between consecutive measurements are computed to come up
592  * with an entropy estimate.  The first, second, and third order delta is
593  * calculated to determine the minimum delta value.  The number of bits
594  * present in this minimum delta value is the entropy estimate.  This
595  * entropy estimation technique using time deltas is similar to that used
596  * in /dev/random implementations from Linux/BSD.
597  */
598 
599 static int
600 physmem_ent_init(physmem_entsrc_t *entsrc)
601 {
602 	uint8_t *ptr;
603 	int i;
604 
605 	bzero(entsrc, sizeof (*entsrc));
606 
607 	/*
608 	 * The maximum entropy amount in bits per block of memory read is
609 	 * log_2(MEMBLOCKSIZE * 8);
610 	 */
611 	i = CRYPTO_BYTES2BITS(MEMBLOCKSIZE);
612 	while (i >>= 1)
613 		entsrc->entperblock++;
614 
615 	/* Initialize entsrc->nblocks */
616 	physmem_count_blocks();
617 
618 	if (entsrc->nblocks == 0) {
619 		cmn_err(CE_WARN, "no memory blocks to scan!");
620 		return (-1);
621 	}
622 
623 	/* Allocate space for the parity vector and memory page */
624 	entsrc->parity = kmem_alloc(howmany(entsrc->nblocks, 8),
625 	    KM_SLEEP);
626 	entsrc->pmbuf = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
627 
628 
629 	/* Initialize parity vector with bits from the pool */
630 	i = howmany(entsrc->nblocks, 8);
631 	ptr = entsrc->parity;
632 	while (i > 0) {
633 		if (i > RNDPOOLSIZE) {
634 			bcopy(srndpool, ptr, RNDPOOLSIZE);
635 			mutex_enter(&srndpool_lock);
636 			swrand_mix_pool(0);
637 			mutex_exit(&srndpool_lock);
638 			ptr += RNDPOOLSIZE;
639 			i -= RNDPOOLSIZE;
640 		} else {
641 			bcopy(srndpool, ptr, i);
642 			break;
643 		}
644 	}
645 
646 	/* Generate some entropy to further initialize the pool */
647 	mutex_enter(&srndpool_lock);
648 	physmem_ent_gen(entsrc);
649 	entropy_bits = 0;
650 	mutex_exit(&srndpool_lock);
651 
652 	return (0);
653 }
654 
655 static void
656 physmem_ent_fini(physmem_entsrc_t *entsrc)
657 {
658 	if (entsrc->pmbuf != NULL)
659 		vmem_free(heap_arena, entsrc->pmbuf, PAGESIZE);
660 	if (entsrc->parity != NULL)
661 		kmem_free(entsrc->parity, howmany(entsrc->nblocks, 8));
662 	bzero(entsrc, sizeof (*entsrc));
663 }
664 
665 static void
666 physmem_ent_gen(physmem_entsrc_t *entsrc)
667 {
668 	struct memlist *pmem;
669 	offset_t offset, poffset;
670 	pfn_t pfn;
671 	int i, nbytes, len, ent = 0;
672 	uint32_t block, oblock;
673 	hrtime_t ts1, ts2, diff, delta, delta2, delta3;
674 	uint8_t digest[HASHSIZE];
675 	HASH_CTX ctx;
676 	page_t *pp;
677 
678 	/*
679 	 * Use each 32-bit quantity in the pool to pick a memory
680 	 * block to read.
681 	 */
682 	for (i = 0; i < RNDPOOLSIZE/4; i++) {
683 
684 		/* If the pool is "full", stop after one block */
685 		if (entropy_bits + ent >= CRYPTO_BYTES2BITS(RNDPOOLSIZE)) {
686 			if (i > 0)
687 				break;
688 		}
689 
690 		/*
691 		 * This lock protects reading of phys_install.
692 		 * Any changes to this list, by DR, are done while
693 		 * holding this lock. So, holding this lock is sufficient
694 		 * to handle DR also.
695 		 */
696 		memlist_read_lock();
697 
698 		/* We're left with less than 4K of memory after DR */
699 		ASSERT(entsrc->nblocks > 0);
700 
701 		/* Pick a memory block to read */
702 		block = oblock = srndpool[i] % entsrc->nblocks;
703 
704 		for (pmem = phys_install; pmem != NULL; pmem = pmem->ml_next) {
705 			if (block < pmem->ml_size / MEMBLOCKSIZE)
706 				break;
707 			block -= pmem->ml_size / MEMBLOCKSIZE;
708 		}
709 
710 		ASSERT(pmem != NULL);
711 
712 		offset = pmem->ml_address + block * MEMBLOCKSIZE;
713 
714 		if (!address_in_memlist(phys_install, offset, MEMBLOCKSIZE)) {
715 			memlist_read_unlock();
716 			continue;
717 		}
718 
719 		/*
720 		 * Do an initial check to see if the address is safe
721 		 */
722 		if (plat_hold_page(offset >> PAGESHIFT, PLAT_HOLD_NO_LOCK, NULL)
723 		    == PLAT_HOLD_FAIL) {
724 			memlist_read_unlock();
725 			continue;
726 		}
727 
728 		/*
729 		 * Figure out which page to load to read the
730 		 * memory block.  Load the page and compute the
731 		 * hash of the memory block.
732 		 */
733 		len = MEMBLOCKSIZE;
734 		ts1 = gethrtime();
735 		HashInit(&ctx);
736 		while (len) {
737 			pfn = offset >> PAGESHIFT;
738 			poffset = offset & PAGEOFFSET;
739 			nbytes = PAGESIZE - poffset < len ?
740 			    PAGESIZE - poffset : len;
741 
742 			/*
743 			 * Re-check the offset, and lock the frame.  If the
744 			 * page was given away after the above check, we'll
745 			 * just bail out.
746 			 */
747 			if (plat_hold_page(pfn, PLAT_HOLD_LOCK, &pp) ==
748 			    PLAT_HOLD_FAIL)
749 				break;
750 
751 			hat_devload(kas.a_hat, entsrc->pmbuf,
752 			    PAGESIZE, pfn, PROT_READ,
753 			    HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK);
754 
755 			HashUpdate(&ctx, (uint8_t *)entsrc->pmbuf + poffset,
756 			    nbytes);
757 
758 			hat_unload(kas.a_hat, entsrc->pmbuf, PAGESIZE,
759 			    HAT_UNLOAD_UNLOCK);
760 
761 			plat_release_page(pp);
762 
763 			len -= nbytes;
764 			offset += nbytes;
765 		}
766 		/* We got our pages. Let the DR roll */
767 		memlist_read_unlock();
768 
769 		/* See if we had to bail out due to a page being given away */
770 		if (len)
771 			continue;
772 
773 		HashFinal(digest, &ctx);
774 		ts2 = gethrtime();
775 
776 		/*
777 		 * Compute the time it took to load and hash the
778 		 * block and compare it against the previous
779 		 * measurement. The delta of the time values
780 		 * provides a small amount of entropy.  The
781 		 * minimum of the first, second, and third order
782 		 * delta is used to estimate how much entropy
783 		 * is present.
784 		 */
785 		diff = ts2 - ts1;
786 		delta = diff - entsrc->last_diff;
787 		if (delta < 0)
788 			delta = -delta;
789 		delta2 = delta - entsrc->last_delta;
790 		if (delta2 < 0)
791 			delta2 = -delta2;
792 		delta3 = delta2 - entsrc->last_delta2;
793 		if (delta3 < 0)
794 			delta3 = -delta3;
795 		entsrc->last_diff = diff;
796 		entsrc->last_delta = delta;
797 		entsrc->last_delta2 = delta2;
798 
799 		if (delta > delta2)
800 			delta = delta2;
801 		if (delta > delta3)
802 			delta = delta3;
803 		delta2 = 0;
804 		while (delta >>= 1)
805 			delta2++;
806 		ent += delta2;
807 
808 		/*
809 		 * If the memory block has changed, credit the pool with
810 		 * the entropy estimate.  The entropy estimate is doubled
811 		 * because the single-bit checksum misses half the change
812 		 * on average.
813 		 */
814 		if (physmem_parity_update(entsrc->parity, oblock,
815 		    digest[0] & 1))
816 			ent += 2 * entsrc->entperblock;
817 
818 		/* Add the entropy bytes to the pool */
819 		swrand_add_bytes(digest, HASHSIZE);
820 		swrand_add_bytes((uint8_t *)&ts1, sizeof (ts1));
821 		swrand_add_bytes((uint8_t *)&ts2, sizeof (ts2));
822 	}
823 
824 	swrand_mix_pool(ent);
825 }
826 
827 static int
828 physmem_parity_update(uint8_t *parity_vec, uint32_t block, int parity)
829 {
830 	/* Test and set the parity bit, return 1 if changed */
831 	if (parity == ((parity_vec[block >> 3] >> (block & 7)) & 1))
832 		return (0);
833 	parity_vec[block >> 3] ^= 1 << (block & 7);
834 	return (1);
835 }
836 
837 /* Compute number of memory blocks available to scan */
838 static void
839 physmem_count_blocks()
840 {
841 	struct memlist *pmem;
842 
843 	memlist_read_lock();
844 	entsrc.nblocks = 0;
845 	for (pmem = phys_install; pmem != NULL; pmem = pmem->ml_next) {
846 		entsrc.nblocks += pmem->ml_size / MEMBLOCKSIZE;
847 		if (entsrc.nblocks > MAXMEMBLOCKS) {
848 			entsrc.nblocks = MAXMEMBLOCKS;
849 			break;
850 		}
851 	}
852 	memlist_read_unlock();
853 }
854 
855 /*
856  * Dynamic Reconfiguration call-back functions
857  */
858 
859 /* ARGSUSED */
860 static void
861 rnd_dr_callback_post_add(void *arg, pgcnt_t delta)
862 {
863 	/* More memory is available now, so update entsrc->nblocks. */
864 	physmem_count_blocks();
865 }
866 
867 /* Call-back routine invoked before the DR starts a memory removal. */
868 /* ARGSUSED */
869 static int
870 rnd_dr_callback_pre_del(void *arg, pgcnt_t delta)
871 {
872 	return (0);
873 }
874 
875 /* Call-back routine invoked after the DR starts a memory removal. */
876 /* ARGSUSED */
877 static void
878 rnd_dr_callback_post_del(void *arg, pgcnt_t delta, int cancelled)
879 {
880 	/* Memory has shrunk, so update entsrc->nblocks. */
881 	physmem_count_blocks();
882 }
883 
884 /* Timeout handling to gather entropy from physmem events */
885 static void
886 swrand_schedule_timeout(void)
887 {
888 	clock_t ut;	/* time in microseconds */
889 
890 	ASSERT(MUTEX_HELD(&srndpool_lock));
891 	/*
892 	 * The new timeout value is taken from the pool of random bits.
893 	 * We're merely reading the first 32 bits from the pool here, not
894 	 * consuming any entropy.
895 	 * This routine is usually called right after stirring the pool, so
896 	 * srndpool[0] will have a *fresh* random value each time.
897 	 * The timeout multiplier value is a random value between 0.7 sec and
898 	 * 1.748575 sec (0.7 sec + 0xFFFFF microseconds).
899 	 * The new timeout is TIMEOUT_INTERVAL times that multiplier.
900 	 */
901 	ut = 700000 + (clock_t)(srndpool[0] & 0xFFFFF);
902 	rnd_timeout_id = timeout(rnd_handler, NULL,
903 	    TIMEOUT_INTERVAL * drv_usectohz(ut));
904 }
905 
906 /*ARGSUSED*/
907 static void
908 rnd_handler(void *arg)
909 {
910 	mutex_enter(&srndpool_lock);
911 
912 	physmem_ent_gen(&entsrc);
913 	if (snum_waiters > 0)
914 		cv_broadcast(&srndpool_read_cv);
915 	swrand_schedule_timeout();
916 
917 	mutex_exit(&srndpool_lock);
918 }
919 
920 /*
921  * Swrand Power-Up Self-Test
922  */
923 void
924 swrand_POST(int *rc)
925 {
926 
927 	*rc = fips_rng_post();
928 
929 }
930