1 /*-
2 * Copyright (c) 2017 Oliver Pinter
3 * Copyright (c) 2017 W. Dean Freeman
4 * Copyright (c) 2000-2015 Mark R V Murray
5 * Copyright (c) 2013 Arthur Mesh
6 * Copyright (c) 2004 Robert N. M. Watson
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer
14 * in this position and unchanged.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 */
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/ck.h>
35 #include <sys/conf.h>
36 #include <sys/epoch.h>
37 #include <sys/eventhandler.h>
38 #include <sys/hash.h>
39 #include <sys/kernel.h>
40 #include <sys/kthread.h>
41 #include <sys/linker.h>
42 #include <sys/lock.h>
43 #include <sys/malloc.h>
44 #include <sys/module.h>
45 #include <sys/mutex.h>
46 #include <sys/random.h>
47 #include <sys/sbuf.h>
48 #include <sys/sysctl.h>
49 #include <sys/unistd.h>
50
51 #include <machine/atomic.h>
52 #include <machine/cpu.h>
53
54 #include <crypto/rijndael/rijndael-api-fst.h>
55 #include <crypto/sha2/sha256.h>
56
57 #include <dev/random/fortuna.h>
58 #include <dev/random/hash.h>
59 #include <dev/random/randomdev.h>
60 #include <dev/random/random_harvestq.h>
61
62 #if defined(RANDOM_ENABLE_ETHER)
63 #define _RANDOM_HARVEST_ETHER_OFF 0
64 #else
65 #define _RANDOM_HARVEST_ETHER_OFF (1u << RANDOM_NET_ETHER)
66 #endif
67 #if defined(RANDOM_ENABLE_UMA)
68 #define _RANDOM_HARVEST_UMA_OFF 0
69 #else
70 #define _RANDOM_HARVEST_UMA_OFF (1u << RANDOM_UMA)
71 #endif
72
73 /*
74 * Note that random_sources_feed() will also use this to try and split up
75 * entropy into a subset of pools per iteration with the goal of feeding
76 * HARVESTSIZE into every pool at least once per second.
77 */
78 #define RANDOM_KTHREAD_HZ 10
79
80 static void random_kthread(void);
81 static void random_sources_feed(void);
82
83 /*
84 * Random must initialize much earlier than epoch, but we can initialize the
85 * epoch code before SMP starts. Prior to SMP, we can safely bypass
86 * concurrency primitives.
87 */
88 static __read_mostly bool epoch_inited;
89 static __read_mostly epoch_t rs_epoch;
90
91 static const char *random_source_descr[ENTROPYSOURCE];
92
93 /*
94 * How many events to queue up. We create this many items in
95 * an 'empty' queue, then transfer them to the 'harvest' queue with
96 * supplied junk. When used, they are transferred back to the
97 * 'empty' queue.
98 */
99 #define RANDOM_RING_MAX 1024
100 #define RANDOM_ACCUM_MAX 8
101
102 /* 1 to let the kernel thread run, 0 to terminate, -1 to mark completion */
103 volatile int random_kthread_control;
104
105
106 /*
107 * Allow the sysadmin to select the broad category of entropy types to harvest.
108 *
109 * Updates are synchronized by the harvest mutex.
110 */
111 __read_frequently u_int hc_source_mask;
112
113 struct random_sources {
114 CK_LIST_ENTRY(random_sources) rrs_entries;
115 const struct random_source *rrs_source;
116 };
117
118 static CK_LIST_HEAD(sources_head, random_sources) source_list =
119 CK_LIST_HEAD_INITIALIZER(source_list);
120
121 SYSCTL_NODE(_kern_random, OID_AUTO, harvest, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
122 "Entropy Device Parameters");
123
124 /*
125 * Put all the harvest queue context stuff in one place.
126 * this make is a bit easier to lock and protect.
127 */
128 static struct harvest_context {
129 /* The harvest mutex protects all of harvest_context and
130 * the related data.
131 */
132 struct mtx hc_mtx;
133 /* Round-robin destination cache. */
134 u_int hc_destination[ENTROPYSOURCE];
135 /* The context of the kernel thread processing harvested entropy */
136 struct proc *hc_kthread_proc;
137 /*
138 * A pair of buffers for queued events. New events are added to the
139 * active queue while the kthread processes the other one in parallel.
140 */
141 struct entropy_buffer {
142 struct harvest_event ring[RANDOM_RING_MAX];
143 u_int pos;
144 } hc_entropy_buf[2];
145 u_int hc_active_buf;
146 struct fast_entropy_accumulator {
147 volatile u_int pos;
148 uint32_t buf[RANDOM_ACCUM_MAX];
149 } hc_entropy_fast_accumulator;
150 } harvest_context;
151
152 #define RANDOM_HARVEST_INIT_LOCK() mtx_init(&harvest_context.hc_mtx, \
153 "entropy harvest mutex", NULL, MTX_SPIN)
154 #define RANDOM_HARVEST_LOCK() mtx_lock_spin(&harvest_context.hc_mtx)
155 #define RANDOM_HARVEST_UNLOCK() mtx_unlock_spin(&harvest_context.hc_mtx)
156
157 static struct kproc_desc random_proc_kp = {
158 "rand_harvestq",
159 random_kthread,
160 &harvest_context.hc_kthread_proc,
161 };
162
163 /* Pass the given event straight through to Fortuna/Whatever. */
164 static __inline void
random_harvestq_fast_process_event(struct harvest_event * event)165 random_harvestq_fast_process_event(struct harvest_event *event)
166 {
167 p_random_alg_context->ra_event_processor(event);
168 explicit_bzero(event, sizeof(*event));
169 }
170
171 static void
random_kthread(void)172 random_kthread(void)
173 {
174 struct harvest_context *hc;
175
176 hc = &harvest_context;
177 for (random_kthread_control = 1; random_kthread_control;) {
178 struct entropy_buffer *buf;
179 u_int entries;
180
181 /* Deal with queued events. */
182 RANDOM_HARVEST_LOCK();
183 buf = &hc->hc_entropy_buf[hc->hc_active_buf];
184 entries = buf->pos;
185 buf->pos = 0;
186 hc->hc_active_buf = (hc->hc_active_buf + 1) %
187 nitems(hc->hc_entropy_buf);
188 RANDOM_HARVEST_UNLOCK();
189 for (u_int i = 0; i < entries; i++)
190 random_harvestq_fast_process_event(&buf->ring[i]);
191
192 /* Poll sources of noise. */
193 random_sources_feed();
194
195 /* XXX: FIX!! Increase the high-performance data rate? Need some measurements first. */
196 for (u_int i = 0; i < RANDOM_ACCUM_MAX; i++) {
197 if (hc->hc_entropy_fast_accumulator.buf[i]) {
198 random_harvest_direct(&hc->hc_entropy_fast_accumulator.buf[i],
199 sizeof(hc->hc_entropy_fast_accumulator.buf[0]), RANDOM_UMA);
200 hc->hc_entropy_fast_accumulator.buf[i] = 0;
201 }
202 }
203 /* XXX: FIX!! This is a *great* place to pass hardware/live entropy to random(9) */
204 tsleep_sbt(&hc->hc_kthread_proc, 0, "-",
205 SBT_1S/RANDOM_KTHREAD_HZ, 0, C_PREL(1));
206 }
207 random_kthread_control = -1;
208 wakeup(&hc->hc_kthread_proc);
209 kproc_exit(0);
210 /* NOTREACHED */
211 }
212 SYSINIT(random_device_h_proc, SI_SUB_KICK_SCHEDULER, SI_ORDER_ANY, kproc_start,
213 &random_proc_kp);
214 _Static_assert(SI_SUB_KICK_SCHEDULER > SI_SUB_RANDOM,
215 "random kthread starting before subsystem initialization");
216
217 static void
rs_epoch_init(void * dummy __unused)218 rs_epoch_init(void *dummy __unused)
219 {
220 rs_epoch = epoch_alloc("Random Sources", EPOCH_PREEMPT);
221 epoch_inited = true;
222 }
223 SYSINIT(rs_epoch_init, SI_SUB_EPOCH, SI_ORDER_ANY, rs_epoch_init, NULL);
224
225 /*
226 * Run through all fast sources reading entropy for the given
227 * number of rounds, which should be a multiple of the number
228 * of entropy accumulation pools in use; it is 32 for Fortuna.
229 */
230 static void
random_sources_feed(void)231 random_sources_feed(void)
232 {
233 uint32_t entropy[HARVESTSIZE];
234 struct epoch_tracker et;
235 struct random_sources *rrs;
236 u_int i, n, npools;
237 bool rse_warm;
238
239 rse_warm = epoch_inited;
240
241 /*
242 * Evenly-ish distribute pool population across the second based on how
243 * frequently random_kthread iterates.
244 *
245 * For Fortuna, the math currently works out as such:
246 *
247 * 64 bits * 4 pools = 256 bits per iteration
248 * 256 bits * 10 Hz = 2560 bits per second, 320 B/s
249 *
250 */
251 npools = howmany(p_random_alg_context->ra_poolcount, RANDOM_KTHREAD_HZ);
252
253 /*-
254 * If we're not seeded yet, attempt to perform a "full seed", filling
255 * all of the PRNG's pools with entropy; if there is enough entropy
256 * available from "fast" entropy sources this will allow us to finish
257 * seeding and unblock the boot process immediately rather than being
258 * stuck for a few seconds with random_kthread gradually collecting a
259 * small chunk of entropy every 1 / RANDOM_KTHREAD_HZ seconds.
260 *
261 * We collect RANDOM_FORTUNA_DEFPOOLSIZE bytes per pool, i.e. enough
262 * to fill Fortuna's pools in the default configuration. With another
263 * PRNG or smaller pools for Fortuna, we might collect more entropy
264 * than needed to fill the pools, but this is harmless; alternatively,
265 * a different PRNG, larger pools, or fast entropy sources which are
266 * not able to provide as much entropy as we request may result in the
267 * not being fully seeded (and thus remaining blocked) but in that
268 * case we will return here after 1 / RANDOM_KTHREAD_HZ seconds and
269 * try again for a large amount of entropy.
270 */
271 if (!p_random_alg_context->ra_seeded())
272 npools = howmany(p_random_alg_context->ra_poolcount *
273 RANDOM_FORTUNA_DEFPOOLSIZE, sizeof(entropy));
274
275 /*
276 * Step over all of live entropy sources, and feed their output
277 * to the system-wide RNG.
278 */
279 if (rse_warm)
280 epoch_enter_preempt(rs_epoch, &et);
281 CK_LIST_FOREACH(rrs, &source_list, rrs_entries) {
282 for (i = 0; i < npools; i++) {
283 if (rrs->rrs_source->rs_read == NULL) {
284 /* Source pushes entropy asynchronously. */
285 continue;
286 }
287 n = rrs->rrs_source->rs_read(entropy, sizeof(entropy));
288 KASSERT((n <= sizeof(entropy)),
289 ("%s: rs_read returned too much data (%u > %zu)",
290 __func__, n, sizeof(entropy)));
291
292 /*
293 * Sometimes the HW entropy source doesn't have anything
294 * ready for us. This isn't necessarily untrustworthy.
295 * We don't perform any other verification of an entropy
296 * source (i.e., length is allowed to be anywhere from 1
297 * to sizeof(entropy), quality is unchecked, etc), so
298 * don't balk verbosely at slow random sources either.
299 * There are reports that RDSEED on x86 metal falls
300 * behind the rate at which we query it, for example.
301 * But it's still a better entropy source than RDRAND.
302 */
303 if (n == 0)
304 continue;
305 random_harvest_direct(entropy, n, rrs->rrs_source->rs_source);
306 }
307 }
308 if (rse_warm)
309 epoch_exit_preempt(rs_epoch, &et);
310 explicit_bzero(entropy, sizeof(entropy));
311 }
312
313 /*
314 * State used for conducting NIST SP 800-90B health tests on entropy sources.
315 */
316 static struct health_test_softc {
317 uint32_t ht_rct_value[HARVESTSIZE + 1];
318 u_int ht_rct_count; /* number of samples with the same value */
319 u_int ht_rct_limit; /* constant after init */
320
321 uint32_t ht_apt_value[HARVESTSIZE + 1];
322 u_int ht_apt_count; /* number of samples with the same value */
323 u_int ht_apt_seq; /* sequence number of the last sample */
324 u_int ht_apt_cutoff; /* constant after init */
325
326 uint64_t ht_total_samples;
327 bool ondemand; /* Set to true to restart the state machine */
328 enum {
329 INIT = 0, /* initial state */
330 DISABLED, /* health checking is disabled */
331 STARTUP, /* doing startup tests, samples are discarded */
332 STEADY, /* steady-state operation */
333 FAILED, /* health check failed, discard samples */
334 } ht_state;
335 } healthtest[ENTROPYSOURCE];
336
337 #define RANDOM_SELFTEST_STARTUP_SAMPLES 1024 /* 4.3, requirement 4 */
338 #define RANDOM_SELFTEST_APT_WINDOW 512 /* 4.4.2 */
339
340 static void
copy_event(uint32_t dst[static HARVESTSIZE+1],const struct harvest_event * event)341 copy_event(uint32_t dst[static HARVESTSIZE + 1],
342 const struct harvest_event *event)
343 {
344 memset(dst, 0, sizeof(uint32_t) * (HARVESTSIZE + 1));
345 memcpy(dst, event->he_entropy, event->he_size);
346 if (event->he_source <= RANDOM_ENVIRONMENTAL_END) {
347 /*
348 * For pure entropy sources the timestamp counter is generally
349 * quite determinstic since samples are taken at regular
350 * intervals, so does not contribute much to the entropy. To
351 * make health tests more effective, exclude it from the sample,
352 * since it might otherwise defeat the health tests in a
353 * scenario where the source is stuck.
354 */
355 dst[HARVESTSIZE] = event->he_somecounter;
356 }
357 }
358
359 static void
random_healthtest_rct_init(struct health_test_softc * ht,const struct harvest_event * event)360 random_healthtest_rct_init(struct health_test_softc *ht,
361 const struct harvest_event *event)
362 {
363 ht->ht_rct_count = 1;
364 copy_event(ht->ht_rct_value, event);
365 }
366
367 /*
368 * Apply the repitition count test to a sample.
369 *
370 * Return false if the test failed, i.e., we observed >= C consecutive samples
371 * with the same value, and true otherwise.
372 */
373 static bool
random_healthtest_rct_next(struct health_test_softc * ht,const struct harvest_event * event)374 random_healthtest_rct_next(struct health_test_softc *ht,
375 const struct harvest_event *event)
376 {
377 uint32_t val[HARVESTSIZE + 1];
378
379 copy_event(val, event);
380 if (memcmp(val, ht->ht_rct_value, sizeof(ht->ht_rct_value)) != 0) {
381 ht->ht_rct_count = 1;
382 memcpy(ht->ht_rct_value, val, sizeof(ht->ht_rct_value));
383 return (true);
384 } else {
385 ht->ht_rct_count++;
386 return (ht->ht_rct_count < ht->ht_rct_limit);
387 }
388 }
389
390 static void
random_healthtest_apt_init(struct health_test_softc * ht,const struct harvest_event * event)391 random_healthtest_apt_init(struct health_test_softc *ht,
392 const struct harvest_event *event)
393 {
394 ht->ht_apt_count = 1;
395 ht->ht_apt_seq = 1;
396 copy_event(ht->ht_apt_value, event);
397 }
398
399 static bool
random_healthtest_apt_next(struct health_test_softc * ht,const struct harvest_event * event)400 random_healthtest_apt_next(struct health_test_softc *ht,
401 const struct harvest_event *event)
402 {
403 uint32_t val[HARVESTSIZE + 1];
404
405 if (ht->ht_apt_seq == 0) {
406 random_healthtest_apt_init(ht, event);
407 return (true);
408 }
409
410 copy_event(val, event);
411 if (memcmp(val, ht->ht_apt_value, sizeof(ht->ht_apt_value)) == 0) {
412 ht->ht_apt_count++;
413 if (ht->ht_apt_count >= ht->ht_apt_cutoff)
414 return (false);
415 }
416
417 ht->ht_apt_seq++;
418 if (ht->ht_apt_seq == RANDOM_SELFTEST_APT_WINDOW)
419 ht->ht_apt_seq = 0;
420
421 return (true);
422 }
423
424 /*
425 * Run the health tests for the given event. This is assumed to be called from
426 * a serialized context.
427 */
428 bool
random_harvest_healthtest(const struct harvest_event * event)429 random_harvest_healthtest(const struct harvest_event *event)
430 {
431 struct health_test_softc *ht;
432
433 ht = &healthtest[event->he_source];
434
435 /*
436 * Was on-demand testing requested? Restart the state machine if so,
437 * restarting the startup tests.
438 */
439 if (atomic_load_bool(&ht->ondemand)) {
440 atomic_store_bool(&ht->ondemand, false);
441 ht->ht_state = INIT;
442 }
443
444 switch (ht->ht_state) {
445 case __predict_false(INIT):
446 /* Store the first sample and initialize test state. */
447 random_healthtest_rct_init(ht, event);
448 random_healthtest_apt_init(ht, event);
449 ht->ht_total_samples = 0;
450 ht->ht_state = STARTUP;
451 return (false);
452 case DISABLED:
453 /* No health testing for this source. */
454 return (true);
455 case STEADY:
456 case STARTUP:
457 ht->ht_total_samples++;
458 if (random_healthtest_rct_next(ht, event) &&
459 random_healthtest_apt_next(ht, event)) {
460 if (ht->ht_state == STARTUP &&
461 ht->ht_total_samples >=
462 RANDOM_SELFTEST_STARTUP_SAMPLES) {
463 printf(
464 "random: health test passed for source %s\n",
465 random_source_descr[event->he_source]);
466 ht->ht_state = STEADY;
467 }
468 return (ht->ht_state == STEADY);
469 }
470 ht->ht_state = FAILED;
471 printf(
472 "random: health test failed for source %s, discarding samples\n",
473 random_source_descr[event->he_source]);
474 /* FALLTHROUGH */
475 case FAILED:
476 return (false);
477 }
478 }
479
480 static bool nist_healthtest_enabled = false;
481 SYSCTL_BOOL(_kern_random, OID_AUTO, nist_healthtest_enabled,
482 CTLFLAG_RDTUN, &nist_healthtest_enabled, 0,
483 "Enable NIST SP 800-90B health tests for noise sources");
484
485 static void
random_healthtest_init(enum random_entropy_source source,int min_entropy)486 random_healthtest_init(enum random_entropy_source source, int min_entropy)
487 {
488 struct health_test_softc *ht;
489
490 ht = &healthtest[source];
491 memset(ht, 0, sizeof(*ht));
492 KASSERT(ht->ht_state == INIT,
493 ("%s: health test state is %d for source %d",
494 __func__, ht->ht_state, source));
495
496 /*
497 * If health-testing is enabled, validate all sources except CACHED and
498 * VMGENID: they are deterministic sources used only a small, fixed
499 * number of times, so statistical testing is not applicable.
500 */
501 if (!nist_healthtest_enabled ||
502 source == RANDOM_CACHED || source == RANDOM_PURE_VMGENID) {
503 ht->ht_state = DISABLED;
504 return;
505 }
506
507 /*
508 * Set cutoff values for the two tests, given a min-entropy estimate for
509 * the source and allowing for an error rate of 1 in 2^{34}. With a
510 * min-entropy estimate of 1 bit and a sample rate of RANDOM_KTHREAD_HZ,
511 * we expect to see an false positive once in ~54.5 years.
512 *
513 * The RCT limit comes from the formula in section 4.4.1.
514 *
515 * The APT cutoffs are calculated using the formula in section 4.4.2
516 * footnote 10 with the number of Bernoulli trials changed from W to
517 * W-1, since the test as written counts the number of samples equal to
518 * the first sample in the window, and thus tests W-1 samples. We
519 * provide cutoffs for estimates up to sizeof(uint32_t)*HARVESTSIZE*8
520 * bits.
521 */
522 const int apt_cutoffs[] = {
523 [1] = 329,
524 [2] = 195,
525 [3] = 118,
526 [4] = 73,
527 [5] = 48,
528 [6] = 33,
529 [7] = 23,
530 [8] = 17,
531 [9] = 13,
532 [10] = 11,
533 [11] = 9,
534 [12] = 8,
535 [13] = 7,
536 [14] = 6,
537 [15] = 5,
538 [16] = 5,
539 [17 ... 19] = 4,
540 [20 ... 25] = 3,
541 [26 ... 42] = 2,
542 [43 ... 64] = 1,
543 };
544 const int error_rate = 34;
545
546 if (min_entropy == 0) {
547 /*
548 * For environmental sources, the main source of entropy is the
549 * associated timecounter value. Since these sources can be
550 * influenced by unprivileged users, we conservatively use a
551 * min-entropy estimate of 1 bit per sample. For "pure"
552 * sources, we assume 8 bits per sample, as such sources provide
553 * a variable amount of data per read and in particular might
554 * only provide a single byte at a time.
555 */
556 min_entropy = source >= RANDOM_PURE_START ? 8 : 1;
557 } else if (min_entropy < 0 || min_entropy >= nitems(apt_cutoffs)) {
558 panic("invalid min_entropy %d for %s", min_entropy,
559 random_source_descr[source]);
560 }
561
562 ht->ht_rct_limit = 1 + howmany(error_rate, min_entropy);
563 ht->ht_apt_cutoff = apt_cutoffs[min_entropy];
564 }
565
566 static int
random_healthtest_ondemand(SYSCTL_HANDLER_ARGS)567 random_healthtest_ondemand(SYSCTL_HANDLER_ARGS)
568 {
569 u_int mask, source;
570 int error;
571
572 mask = 0;
573 error = sysctl_handle_int(oidp, &mask, 0, req);
574 if (error != 0 || req->newptr == NULL)
575 return (error);
576
577 while (mask != 0) {
578 source = ffs(mask) - 1;
579 if (source < nitems(healthtest))
580 atomic_store_bool(&healthtest[source].ondemand, true);
581 mask &= ~(1u << source);
582 }
583 return (0);
584 }
585 SYSCTL_PROC(_kern_random, OID_AUTO, nist_healthtest_ondemand,
586 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0,
587 random_healthtest_ondemand, "I",
588 "Re-run NIST SP 800-90B startup health tests for a noise source");
589
590 static int
random_check_uint_harvestmask(SYSCTL_HANDLER_ARGS)591 random_check_uint_harvestmask(SYSCTL_HANDLER_ARGS)
592 {
593 static const u_int user_immutable_mask =
594 (((1 << ENTROPYSOURCE) - 1) & (-1UL << RANDOM_PURE_START)) |
595 _RANDOM_HARVEST_ETHER_OFF | _RANDOM_HARVEST_UMA_OFF;
596
597 int error;
598 u_int value;
599
600 value = atomic_load_int(&hc_source_mask);
601 error = sysctl_handle_int(oidp, &value, 0, req);
602 if (error != 0 || req->newptr == NULL)
603 return (error);
604
605 if (flsl(value) > ENTROPYSOURCE)
606 return (EINVAL);
607
608 /*
609 * Disallow userspace modification of pure entropy sources.
610 */
611 RANDOM_HARVEST_LOCK();
612 hc_source_mask = (value & ~user_immutable_mask) |
613 (hc_source_mask & user_immutable_mask);
614 RANDOM_HARVEST_UNLOCK();
615 return (0);
616 }
617 SYSCTL_PROC(_kern_random_harvest, OID_AUTO, mask,
618 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0,
619 random_check_uint_harvestmask, "IU",
620 "Entropy harvesting mask");
621
622 static int
random_print_harvestmask(SYSCTL_HANDLER_ARGS)623 random_print_harvestmask(SYSCTL_HANDLER_ARGS)
624 {
625 struct sbuf sbuf;
626 int error, i;
627
628 error = sysctl_wire_old_buffer(req, 0);
629 if (error == 0) {
630 u_int mask;
631
632 sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
633 mask = atomic_load_int(&hc_source_mask);
634 for (i = ENTROPYSOURCE - 1; i >= 0; i--) {
635 bool present;
636
637 present = (mask & (1u << i)) != 0;
638 sbuf_cat(&sbuf, present ? "1" : "0");
639 }
640 error = sbuf_finish(&sbuf);
641 sbuf_delete(&sbuf);
642 }
643 return (error);
644 }
645 SYSCTL_PROC(_kern_random_harvest, OID_AUTO, mask_bin,
646 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
647 random_print_harvestmask, "A",
648 "Entropy harvesting mask (printable)");
649
650 static const char *random_source_descr[ENTROPYSOURCE] = {
651 [RANDOM_CACHED] = "CACHED",
652 [RANDOM_ATTACH] = "ATTACH",
653 [RANDOM_KEYBOARD] = "KEYBOARD",
654 [RANDOM_MOUSE] = "MOUSE",
655 [RANDOM_NET_TUN] = "NET_TUN",
656 [RANDOM_NET_ETHER] = "NET_ETHER",
657 [RANDOM_NET_NG] = "NET_NG",
658 [RANDOM_INTERRUPT] = "INTERRUPT",
659 [RANDOM_SWI] = "SWI",
660 [RANDOM_FS_ATIME] = "FS_ATIME",
661 [RANDOM_UMA] = "UMA",
662 [RANDOM_CALLOUT] = "CALLOUT",
663 [RANDOM_RANDOMDEV] = "RANDOMDEV", /* ENVIRONMENTAL_END */
664 [RANDOM_PURE_SAFE] = "PURE_SAFE", /* PURE_START */
665 [RANDOM_PURE_GLXSB] = "PURE_GLXSB",
666 [RANDOM_PURE_RDRAND] = "PURE_RDRAND",
667 [RANDOM_PURE_RDSEED] = "PURE_RDSEED",
668 [RANDOM_PURE_NEHEMIAH] = "PURE_NEHEMIAH",
669 [RANDOM_PURE_RNDTEST] = "PURE_RNDTEST",
670 [RANDOM_PURE_VIRTIO] = "PURE_VIRTIO",
671 [RANDOM_PURE_BROADCOM] = "PURE_BROADCOM",
672 [RANDOM_PURE_CCP] = "PURE_CCP",
673 [RANDOM_PURE_DARN] = "PURE_DARN",
674 [RANDOM_PURE_TPM] = "PURE_TPM",
675 [RANDOM_PURE_VMGENID] = "PURE_VMGENID",
676 [RANDOM_PURE_QUALCOMM] = "PURE_QUALCOMM",
677 [RANDOM_PURE_ARMV8] = "PURE_ARMV8",
678 [RANDOM_PURE_ARM_TRNG] = "PURE_ARM_TRNG",
679 /* "ENTROPYSOURCE" */
680 };
681
682 static int
random_print_harvestmask_symbolic(SYSCTL_HANDLER_ARGS)683 random_print_harvestmask_symbolic(SYSCTL_HANDLER_ARGS)
684 {
685 struct sbuf sbuf;
686 int error, i;
687 bool first;
688
689 first = true;
690 error = sysctl_wire_old_buffer(req, 0);
691 if (error == 0) {
692 u_int mask;
693
694 sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
695 mask = atomic_load_int(&hc_source_mask);
696 for (i = ENTROPYSOURCE - 1; i >= 0; i--) {
697 bool present;
698
699 present = (mask & (1u << i)) != 0;
700 if (i >= RANDOM_PURE_START && !present)
701 continue;
702 if (!first)
703 sbuf_cat(&sbuf, ",");
704 sbuf_cat(&sbuf, !present ? "[" : "");
705 sbuf_cat(&sbuf, random_source_descr[i]);
706 sbuf_cat(&sbuf, !present ? "]" : "");
707 first = false;
708 }
709 error = sbuf_finish(&sbuf);
710 sbuf_delete(&sbuf);
711 }
712 return (error);
713 }
714 SYSCTL_PROC(_kern_random_harvest, OID_AUTO, mask_symbolic,
715 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
716 random_print_harvestmask_symbolic, "A",
717 "Entropy harvesting mask (symbolic)");
718
719 static void
random_harvestq_init(void * unused __unused)720 random_harvestq_init(void *unused __unused)
721 {
722 static const u_int almost_everything_mask =
723 (((1 << (RANDOM_ENVIRONMENTAL_END + 1)) - 1) &
724 ~_RANDOM_HARVEST_ETHER_OFF & ~_RANDOM_HARVEST_UMA_OFF);
725
726 hc_source_mask = almost_everything_mask;
727 RANDOM_HARVEST_INIT_LOCK();
728 harvest_context.hc_active_buf = 0;
729
730 for (int i = RANDOM_START; i <= RANDOM_ENVIRONMENTAL_END; i++)
731 random_healthtest_init(i, 0);
732 }
733 SYSINIT(random_device_h_init, SI_SUB_RANDOM, SI_ORDER_THIRD, random_harvestq_init, NULL);
734
735 /*
736 * Subroutine to slice up a contiguous chunk of 'entropy' and feed it into the
737 * underlying algorithm. Returns number of bytes actually fed into underlying
738 * algorithm.
739 */
740 static size_t
random_early_prime(char * entropy,size_t len)741 random_early_prime(char *entropy, size_t len)
742 {
743 struct harvest_event event;
744 size_t i;
745
746 len = rounddown(len, sizeof(event.he_entropy));
747 if (len == 0)
748 return (0);
749
750 for (i = 0; i < len; i += sizeof(event.he_entropy)) {
751 event.he_somecounter = random_get_cyclecount();
752 event.he_size = sizeof(event.he_entropy);
753 event.he_source = RANDOM_CACHED;
754 event.he_destination =
755 harvest_context.hc_destination[RANDOM_CACHED]++;
756 memcpy(event.he_entropy, entropy + i, sizeof(event.he_entropy));
757 random_harvestq_fast_process_event(&event);
758 }
759 explicit_bzero(entropy, len);
760 return (len);
761 }
762
763 /*
764 * Subroutine to search for known loader-loaded files in memory and feed them
765 * into the underlying algorithm early in boot. Returns the number of bytes
766 * loaded (zero if none were loaded).
767 */
768 static size_t
random_prime_loader_file(const char * type)769 random_prime_loader_file(const char *type)
770 {
771 uint8_t *keyfile, *data;
772 size_t size;
773
774 keyfile = preload_search_by_type(type);
775 if (keyfile == NULL)
776 return (0);
777
778 data = preload_fetch_addr(keyfile);
779 size = preload_fetch_size(keyfile);
780 if (data == NULL)
781 return (0);
782
783 return (random_early_prime(data, size));
784 }
785
786 /*
787 * This is used to prime the RNG by grabbing any early random stuff
788 * known to the kernel, and inserting it directly into the hashing
789 * module, currently Fortuna.
790 */
791 static void
random_harvestq_prime(void * unused __unused)792 random_harvestq_prime(void *unused __unused)
793 {
794 size_t size;
795
796 /*
797 * Get entropy that may have been preloaded by loader(8)
798 * and use it to pre-charge the entropy harvest queue.
799 */
800 size = random_prime_loader_file(RANDOM_CACHED_BOOT_ENTROPY_MODULE);
801 if (bootverbose) {
802 if (size > 0)
803 printf("random: read %zu bytes from preloaded cache\n",
804 size);
805 else
806 printf("random: no preloaded entropy cache\n");
807 }
808 size = random_prime_loader_file(RANDOM_PLATFORM_BOOT_ENTROPY_MODULE);
809 if (bootverbose) {
810 if (size > 0)
811 printf("random: read %zu bytes from platform bootloader\n",
812 size);
813 else
814 printf("random: no platform bootloader entropy\n");
815 }
816 }
817 SYSINIT(random_device_prime, SI_SUB_RANDOM, SI_ORDER_MIDDLE, random_harvestq_prime, NULL);
818
819 static void
random_harvestq_deinit(void * unused __unused)820 random_harvestq_deinit(void *unused __unused)
821 {
822
823 /* Command the hash/reseed thread to end and wait for it to finish */
824 random_kthread_control = 0;
825 while (random_kthread_control >= 0)
826 tsleep(&harvest_context.hc_kthread_proc, 0, "harvqterm", hz/5);
827 }
828 SYSUNINIT(random_device_h_init, SI_SUB_RANDOM, SI_ORDER_THIRD, random_harvestq_deinit, NULL);
829
830 /*-
831 * Entropy harvesting queue routine.
832 *
833 * This is supposed to be fast; do not do anything slow in here!
834 * It is also illegal (and morally reprehensible) to insert any
835 * high-rate data here. "High-rate" is defined as a data source
836 * that is likely to fill up the buffer in much less than 100ms.
837 * This includes the "always-on" sources like the Intel "rdrand"
838 * or the VIA Nehamiah "xstore" sources.
839 */
840 /* XXXRW: get_cyclecount() is cheap on most modern hardware, where cycle
841 * counters are built in, but on older hardware it will do a real time clock
842 * read which can be quite expensive.
843 */
844 void
random_harvest_queue_(const void * entropy,u_int size,enum random_entropy_source origin)845 random_harvest_queue_(const void *entropy, u_int size, enum random_entropy_source origin)
846 {
847 struct harvest_context *hc;
848 struct entropy_buffer *buf;
849 struct harvest_event *event;
850
851 KASSERT(origin >= RANDOM_START && origin < ENTROPYSOURCE,
852 ("%s: origin %d invalid", __func__, origin));
853
854 hc = &harvest_context;
855 RANDOM_HARVEST_LOCK();
856 buf = &hc->hc_entropy_buf[hc->hc_active_buf];
857 if (buf->pos < RANDOM_RING_MAX) {
858 event = &buf->ring[buf->pos++];
859 event->he_somecounter = random_get_cyclecount();
860 event->he_source = origin;
861 event->he_destination = hc->hc_destination[origin]++;
862 if (size <= sizeof(event->he_entropy)) {
863 event->he_size = size;
864 memcpy(event->he_entropy, entropy, size);
865 } else {
866 /* Big event, so squash it */
867 event->he_size = sizeof(event->he_entropy[0]);
868 event->he_entropy[0] = jenkins_hash(entropy, size, (uint32_t)(uintptr_t)event);
869 }
870 }
871 RANDOM_HARVEST_UNLOCK();
872 }
873
874 /*-
875 * Entropy harvesting fast routine.
876 *
877 * This is supposed to be very fast; do not do anything slow in here!
878 * This is the right place for high-rate harvested data.
879 */
880 void
random_harvest_fast_(const void * entropy,u_int size)881 random_harvest_fast_(const void *entropy, u_int size)
882 {
883 u_int pos;
884
885 pos = harvest_context.hc_entropy_fast_accumulator.pos;
886 harvest_context.hc_entropy_fast_accumulator.buf[pos] ^=
887 jenkins_hash(entropy, size, random_get_cyclecount());
888 harvest_context.hc_entropy_fast_accumulator.pos = (pos + 1)%RANDOM_ACCUM_MAX;
889 }
890
891 /*-
892 * Entropy harvesting direct routine.
893 *
894 * This is not supposed to be fast, but will only be used during
895 * (e.g.) booting when initial entropy is being gathered.
896 */
897 void
random_harvest_direct_(const void * entropy,u_int size,enum random_entropy_source origin)898 random_harvest_direct_(const void *entropy, u_int size, enum random_entropy_source origin)
899 {
900 struct harvest_event event;
901
902 KASSERT(origin >= RANDOM_START && origin < ENTROPYSOURCE, ("%s: origin %d invalid\n", __func__, origin));
903 size = MIN(size, sizeof(event.he_entropy));
904 event.he_somecounter = random_get_cyclecount();
905 event.he_size = size;
906 event.he_source = origin;
907 event.he_destination = harvest_context.hc_destination[origin]++;
908 memcpy(event.he_entropy, entropy, size);
909 random_harvestq_fast_process_event(&event);
910 }
911
912 void
random_source_register(const struct random_source * rsource)913 random_source_register(const struct random_source *rsource)
914 {
915 struct random_sources *rrs;
916
917 KASSERT(rsource != NULL, ("invalid input to %s", __func__));
918
919 rrs = malloc(sizeof(*rrs), M_ENTROPY, M_WAITOK);
920 rrs->rrs_source = rsource;
921
922 printf("random: registering fast source %s\n", rsource->rs_ident);
923
924 random_healthtest_init(rsource->rs_source, rsource->rs_min_entropy);
925
926 RANDOM_HARVEST_LOCK();
927 hc_source_mask |= (1 << rsource->rs_source);
928 CK_LIST_INSERT_HEAD(&source_list, rrs, rrs_entries);
929 RANDOM_HARVEST_UNLOCK();
930 }
931
932 void
random_source_deregister(const struct random_source * rsource)933 random_source_deregister(const struct random_source *rsource)
934 {
935 struct random_sources *rrs = NULL;
936
937 KASSERT(rsource != NULL, ("invalid input to %s", __func__));
938
939 RANDOM_HARVEST_LOCK();
940 hc_source_mask &= ~(1 << rsource->rs_source);
941 CK_LIST_FOREACH(rrs, &source_list, rrs_entries)
942 if (rrs->rrs_source == rsource) {
943 CK_LIST_REMOVE(rrs, rrs_entries);
944 break;
945 }
946 RANDOM_HARVEST_UNLOCK();
947
948 if (rrs != NULL && epoch_inited)
949 epoch_wait_preempt(rs_epoch);
950 free(rrs, M_ENTROPY);
951 }
952
953 static int
random_source_handler(SYSCTL_HANDLER_ARGS)954 random_source_handler(SYSCTL_HANDLER_ARGS)
955 {
956 struct epoch_tracker et;
957 struct random_sources *rrs;
958 struct sbuf sbuf;
959 int error, count;
960
961 error = sysctl_wire_old_buffer(req, 0);
962 if (error != 0)
963 return (error);
964
965 sbuf_new_for_sysctl(&sbuf, NULL, 64, req);
966 count = 0;
967 epoch_enter_preempt(rs_epoch, &et);
968 CK_LIST_FOREACH(rrs, &source_list, rrs_entries) {
969 sbuf_cat(&sbuf, (count++ ? ",'" : "'"));
970 sbuf_cat(&sbuf, rrs->rrs_source->rs_ident);
971 sbuf_cat(&sbuf, "'");
972 }
973 epoch_exit_preempt(rs_epoch, &et);
974 error = sbuf_finish(&sbuf);
975 sbuf_delete(&sbuf);
976 return (error);
977 }
978 SYSCTL_PROC(_kern_random, OID_AUTO, random_sources, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
979 NULL, 0, random_source_handler, "A",
980 "List of active fast entropy sources.");
981
982 MODULE_VERSION(random_harvestq, 1);
983