1 /*-
2 * Copyright (c) 2017 Oliver Pinter
3 * Copyright (c) 2017 W. Dean Freeman
4 * Copyright (c) 2000-2015 Mark R V Murray
5 * Copyright (c) 2013 Arthur Mesh
6 * Copyright (c) 2004 Robert N. M. Watson
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer
14 * in this position and unchanged.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 */
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/ck.h>
35 #include <sys/conf.h>
36 #include <sys/epoch.h>
37 #include <sys/eventhandler.h>
38 #include <sys/hash.h>
39 #include <sys/kernel.h>
40 #include <sys/kthread.h>
41 #include <sys/linker.h>
42 #include <sys/lock.h>
43 #include <sys/malloc.h>
44 #include <sys/module.h>
45 #include <sys/mutex.h>
46 #include <sys/random.h>
47 #include <sys/sbuf.h>
48 #include <sys/sysctl.h>
49 #include <sys/unistd.h>
50
51 #include <machine/atomic.h>
52 #include <machine/cpu.h>
53
54 #include <crypto/rijndael/rijndael-api-fst.h>
55 #include <crypto/sha2/sha256.h>
56
57 #include <dev/random/fortuna.h>
58 #include <dev/random/hash.h>
59 #include <dev/random/randomdev.h>
60 #include <dev/random/random_harvestq.h>
61
62 #if defined(RANDOM_ENABLE_ETHER)
63 #define _RANDOM_HARVEST_ETHER_OFF 0
64 #else
65 #define _RANDOM_HARVEST_ETHER_OFF (1u << RANDOM_NET_ETHER)
66 #endif
67 #if defined(RANDOM_ENABLE_UMA)
68 #define _RANDOM_HARVEST_UMA_OFF 0
69 #else
70 #define _RANDOM_HARVEST_UMA_OFF (1u << RANDOM_UMA)
71 #endif
72
73 /*
74 * Note that random_sources_feed() will also use this to try and split up
75 * entropy into a subset of pools per iteration with the goal of feeding
76 * HARVESTSIZE into every pool at least once per second.
77 */
78 #define RANDOM_KTHREAD_HZ 10
79
80 static void random_kthread(void);
81 static void random_sources_feed(void);
82
83 /*
84 * Random must initialize much earlier than epoch, but we can initialize the
85 * epoch code before SMP starts. Prior to SMP, we can safely bypass
86 * concurrency primitives.
87 */
88 static __read_mostly bool epoch_inited;
89 static __read_mostly epoch_t rs_epoch;
90
91 static const char *random_source_descr[ENTROPYSOURCE];
92
93 /*
94 * How many events to queue up. We create this many items in
95 * an 'empty' queue, then transfer them to the 'harvest' queue with
96 * supplied junk. When used, they are transferred back to the
97 * 'empty' queue.
98 */
99 #define RANDOM_RING_MAX 1024
100 #define RANDOM_ACCUM_MAX 8
101
102 /* 1 to let the kernel thread run, 0 to terminate, -1 to mark completion */
103 volatile int random_kthread_control;
104
105
106 /*
107 * Allow the sysadmin to select the broad category of entropy types to harvest.
108 *
109 * Updates are synchronized by the harvest mutex.
110 */
111 __read_frequently u_int hc_source_mask;
112
113 struct random_sources {
114 CK_LIST_ENTRY(random_sources) rrs_entries;
115 const struct random_source *rrs_source;
116 };
117
118 static CK_LIST_HEAD(sources_head, random_sources) source_list =
119 CK_LIST_HEAD_INITIALIZER(source_list);
120
121 SYSCTL_NODE(_kern_random, OID_AUTO, harvest, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
122 "Entropy Device Parameters");
123
124 /*
125 * Put all the harvest queue context stuff in one place.
126 * this make is a bit easier to lock and protect.
127 */
128 static struct harvest_context {
129 /* The harvest mutex protects all of harvest_context and
130 * the related data.
131 */
132 struct mtx hc_mtx;
133 /* Round-robin destination cache. */
134 u_int hc_destination[ENTROPYSOURCE];
135 /* The context of the kernel thread processing harvested entropy */
136 struct proc *hc_kthread_proc;
137 /*
138 * A pair of buffers for queued events. New events are added to the
139 * active queue while the kthread processes the other one in parallel.
140 */
141 struct entropy_buffer {
142 struct harvest_event ring[RANDOM_RING_MAX];
143 u_int pos;
144 } hc_entropy_buf[2];
145 u_int hc_active_buf;
146 struct fast_entropy_accumulator {
147 volatile u_int pos;
148 uint32_t buf[RANDOM_ACCUM_MAX];
149 } hc_entropy_fast_accumulator;
150 } harvest_context;
151
152 #define RANDOM_HARVEST_INIT_LOCK() mtx_init(&harvest_context.hc_mtx, \
153 "entropy harvest mutex", NULL, MTX_SPIN)
154 #define RANDOM_HARVEST_LOCK() mtx_lock_spin(&harvest_context.hc_mtx)
155 #define RANDOM_HARVEST_UNLOCK() mtx_unlock_spin(&harvest_context.hc_mtx)
156
157 static struct kproc_desc random_proc_kp = {
158 "rand_harvestq",
159 random_kthread,
160 &harvest_context.hc_kthread_proc,
161 };
162
163 /* Pass the given event straight through to Fortuna/Whatever. */
164 static __inline void
random_harvestq_fast_process_event(struct harvest_event * event)165 random_harvestq_fast_process_event(struct harvest_event *event)
166 {
167 p_random_alg_context->ra_event_processor(event);
168 explicit_bzero(event, sizeof(*event));
169 }
170
171 static void
random_kthread(void)172 random_kthread(void)
173 {
174 struct harvest_context *hc;
175
176 hc = &harvest_context;
177 for (random_kthread_control = 1; random_kthread_control;) {
178 struct entropy_buffer *buf;
179 u_int entries;
180
181 /* Deal with queued events. */
182 RANDOM_HARVEST_LOCK();
183 buf = &hc->hc_entropy_buf[hc->hc_active_buf];
184 entries = buf->pos;
185 buf->pos = 0;
186 hc->hc_active_buf = (hc->hc_active_buf + 1) %
187 nitems(hc->hc_entropy_buf);
188 RANDOM_HARVEST_UNLOCK();
189 for (u_int i = 0; i < entries; i++)
190 random_harvestq_fast_process_event(&buf->ring[i]);
191
192 /* Poll sources of noise. */
193 random_sources_feed();
194
195 /* XXX: FIX!! Increase the high-performance data rate? Need some measurements first. */
196 for (u_int i = 0; i < RANDOM_ACCUM_MAX; i++) {
197 if (hc->hc_entropy_fast_accumulator.buf[i]) {
198 random_harvest_direct(&hc->hc_entropy_fast_accumulator.buf[i],
199 sizeof(hc->hc_entropy_fast_accumulator.buf[0]), RANDOM_UMA);
200 hc->hc_entropy_fast_accumulator.buf[i] = 0;
201 }
202 }
203 /* XXX: FIX!! This is a *great* place to pass hardware/live entropy to random(9) */
204 tsleep_sbt(&hc->hc_kthread_proc, 0, "-",
205 SBT_1S/RANDOM_KTHREAD_HZ, 0, C_PREL(1));
206 }
207 random_kthread_control = -1;
208 wakeup(&hc->hc_kthread_proc);
209 kproc_exit(0);
210 /* NOTREACHED */
211 }
212 SYSINIT(random_device_h_proc, SI_SUB_KICK_SCHEDULER, SI_ORDER_ANY, kproc_start,
213 &random_proc_kp);
214 _Static_assert(SI_SUB_KICK_SCHEDULER > SI_SUB_RANDOM,
215 "random kthread starting before subsystem initialization");
216
217 static void
rs_epoch_init(void * dummy __unused)218 rs_epoch_init(void *dummy __unused)
219 {
220 rs_epoch = epoch_alloc("Random Sources", EPOCH_PREEMPT);
221 epoch_inited = true;
222 }
223 SYSINIT(rs_epoch_init, SI_SUB_EPOCH, SI_ORDER_ANY, rs_epoch_init, NULL);
224
225 /*
226 * Run through all fast sources reading entropy for the given
227 * number of rounds, which should be a multiple of the number
228 * of entropy accumulation pools in use; it is 32 for Fortuna.
229 */
230 static void
random_sources_feed(void)231 random_sources_feed(void)
232 {
233 uint32_t entropy[HARVESTSIZE];
234 struct epoch_tracker et;
235 struct random_sources *rrs;
236 u_int i, n, npools;
237 bool rse_warm;
238
239 rse_warm = epoch_inited;
240
241 /*
242 * Evenly-ish distribute pool population across the second based on how
243 * frequently random_kthread iterates.
244 *
245 * For Fortuna, the math currently works out as such:
246 *
247 * 64 bits * 4 pools = 256 bits per iteration
248 * 256 bits * 10 Hz = 2560 bits per second, 320 B/s
249 *
250 */
251 npools = howmany(p_random_alg_context->ra_poolcount, RANDOM_KTHREAD_HZ);
252
253 /*-
254 * If we're not seeded yet, attempt to perform a "full seed", filling
255 * all of the PRNG's pools with entropy; if there is enough entropy
256 * available from "fast" entropy sources this will allow us to finish
257 * seeding and unblock the boot process immediately rather than being
258 * stuck for a few seconds with random_kthread gradually collecting a
259 * small chunk of entropy every 1 / RANDOM_KTHREAD_HZ seconds.
260 *
261 * We collect RANDOM_FORTUNA_DEFPOOLSIZE bytes per pool, i.e. enough
262 * to fill Fortuna's pools in the default configuration. With another
263 * PRNG or smaller pools for Fortuna, we might collect more entropy
264 * than needed to fill the pools, but this is harmless; alternatively,
265 * a different PRNG, larger pools, or fast entropy sources which are
266 * not able to provide as much entropy as we request may result in the
267 * not being fully seeded (and thus remaining blocked) but in that
268 * case we will return here after 1 / RANDOM_KTHREAD_HZ seconds and
269 * try again for a large amount of entropy.
270 */
271 if (!p_random_alg_context->ra_seeded())
272 npools = howmany(p_random_alg_context->ra_poolcount *
273 RANDOM_FORTUNA_DEFPOOLSIZE, sizeof(entropy));
274
275 /*
276 * Step over all of live entropy sources, and feed their output
277 * to the system-wide RNG.
278 */
279 if (rse_warm)
280 epoch_enter_preempt(rs_epoch, &et);
281 CK_LIST_FOREACH(rrs, &source_list, rrs_entries) {
282 for (i = 0; i < npools; i++) {
283 if (rrs->rrs_source->rs_read == NULL) {
284 /* Source pushes entropy asynchronously. */
285 continue;
286 }
287 n = rrs->rrs_source->rs_read(entropy, sizeof(entropy));
288 KASSERT((n <= sizeof(entropy)),
289 ("%s: rs_read returned too much data (%u > %zu)",
290 __func__, n, sizeof(entropy)));
291
292 /*
293 * Sometimes the HW entropy source doesn't have anything
294 * ready for us. This isn't necessarily untrustworthy.
295 * We don't perform any other verification of an entropy
296 * source (i.e., length is allowed to be anywhere from 1
297 * to sizeof(entropy), quality is unchecked, etc), so
298 * don't balk verbosely at slow random sources either.
299 * There are reports that RDSEED on x86 metal falls
300 * behind the rate at which we query it, for example.
301 * But it's still a better entropy source than RDRAND.
302 */
303 if (n == 0)
304 continue;
305 random_harvest_direct(entropy, n, rrs->rrs_source->rs_source);
306 }
307 }
308 if (rse_warm)
309 epoch_exit_preempt(rs_epoch, &et);
310 explicit_bzero(entropy, sizeof(entropy));
311 }
312
313 /*
314 * State used for conducting NIST SP 800-90B health tests on entropy sources.
315 */
316 static struct health_test_softc {
317 uint32_t ht_rct_value[HARVESTSIZE + 1];
318 u_int ht_rct_count; /* number of samples with the same value */
319 u_int ht_rct_limit; /* constant after init */
320
321 uint32_t ht_apt_value[HARVESTSIZE + 1];
322 u_int ht_apt_count; /* number of samples with the same value */
323 u_int ht_apt_seq; /* sequence number of the last sample */
324 u_int ht_apt_cutoff; /* constant after init */
325
326 uint64_t ht_total_samples;
327 bool ondemand; /* Set to true to restart the state machine */
328 enum {
329 INIT = 0, /* initial state */
330 DISABLED, /* health checking is disabled */
331 STARTUP, /* doing startup tests, samples are discarded */
332 STEADY, /* steady-state operation */
333 FAILED, /* health check failed, discard samples */
334 } ht_state;
335 } healthtest[ENTROPYSOURCE];
336
337 #define RANDOM_SELFTEST_STARTUP_SAMPLES 1024 /* 4.3, requirement 4 */
338 #define RANDOM_SELFTEST_APT_WINDOW 512 /* 4.4.2 */
339
340 static void
copy_event(uint32_t dst[static HARVESTSIZE+1],const struct harvest_event * event)341 copy_event(uint32_t dst[static HARVESTSIZE + 1],
342 const struct harvest_event *event)
343 {
344 memset(dst, 0, sizeof(uint32_t) * (HARVESTSIZE + 1));
345 memcpy(dst, event->he_entropy, event->he_size);
346 if (event->he_source <= RANDOM_ENVIRONMENTAL_END) {
347 /*
348 * For pure entropy sources the timestamp counter is generally
349 * quite determinstic since samples are taken at regular
350 * intervals, so does not contribute much to the entropy. To
351 * make health tests more effective, exclude it from the sample,
352 * since it might otherwise defeat the health tests in a
353 * scenario where the source is stuck.
354 */
355 dst[HARVESTSIZE] = event->he_somecounter;
356 }
357 }
358
359 static void
random_healthtest_rct_init(struct health_test_softc * ht,const struct harvest_event * event)360 random_healthtest_rct_init(struct health_test_softc *ht,
361 const struct harvest_event *event)
362 {
363 ht->ht_rct_count = 1;
364 copy_event(ht->ht_rct_value, event);
365 }
366
367 /*
368 * Apply the repitition count test to a sample.
369 *
370 * Return false if the test failed, i.e., we observed >= C consecutive samples
371 * with the same value, and true otherwise.
372 */
373 static bool
random_healthtest_rct_next(struct health_test_softc * ht,const struct harvest_event * event)374 random_healthtest_rct_next(struct health_test_softc *ht,
375 const struct harvest_event *event)
376 {
377 uint32_t val[HARVESTSIZE + 1];
378
379 copy_event(val, event);
380 if (memcmp(val, ht->ht_rct_value, sizeof(ht->ht_rct_value)) != 0) {
381 ht->ht_rct_count = 1;
382 memcpy(ht->ht_rct_value, val, sizeof(ht->ht_rct_value));
383 return (true);
384 } else {
385 ht->ht_rct_count++;
386 return (ht->ht_rct_count < ht->ht_rct_limit);
387 }
388 }
389
390 static void
random_healthtest_apt_init(struct health_test_softc * ht,const struct harvest_event * event)391 random_healthtest_apt_init(struct health_test_softc *ht,
392 const struct harvest_event *event)
393 {
394 ht->ht_apt_count = 1;
395 ht->ht_apt_seq = 1;
396 copy_event(ht->ht_apt_value, event);
397 }
398
399 static bool
random_healthtest_apt_next(struct health_test_softc * ht,const struct harvest_event * event)400 random_healthtest_apt_next(struct health_test_softc *ht,
401 const struct harvest_event *event)
402 {
403 uint32_t val[HARVESTSIZE + 1];
404
405 if (ht->ht_apt_seq == 0) {
406 random_healthtest_apt_init(ht, event);
407 return (true);
408 }
409
410 copy_event(val, event);
411 if (memcmp(val, ht->ht_apt_value, sizeof(ht->ht_apt_value)) == 0) {
412 ht->ht_apt_count++;
413 if (ht->ht_apt_count >= ht->ht_apt_cutoff)
414 return (false);
415 }
416
417 ht->ht_apt_seq++;
418 if (ht->ht_apt_seq == RANDOM_SELFTEST_APT_WINDOW)
419 ht->ht_apt_seq = 0;
420
421 return (true);
422 }
423
424 /*
425 * Run the health tests for the given event. This is assumed to be called from
426 * a serialized context.
427 */
428 bool
random_harvest_healthtest(const struct harvest_event * event)429 random_harvest_healthtest(const struct harvest_event *event)
430 {
431 struct health_test_softc *ht;
432
433 ht = &healthtest[event->he_source];
434
435 /*
436 * Was on-demand testing requested? Restart the state machine if so,
437 * restarting the startup tests.
438 */
439 if (atomic_load_bool(&ht->ondemand)) {
440 atomic_store_bool(&ht->ondemand, false);
441 ht->ht_state = INIT;
442 }
443
444 switch (ht->ht_state) {
445 case __predict_false(INIT):
446 /* Store the first sample and initialize test state. */
447 random_healthtest_rct_init(ht, event);
448 random_healthtest_apt_init(ht, event);
449 ht->ht_total_samples = 0;
450 ht->ht_state = STARTUP;
451 return (false);
452 case DISABLED:
453 /* No health testing for this source. */
454 return (true);
455 case STEADY:
456 case STARTUP:
457 ht->ht_total_samples++;
458 if (random_healthtest_rct_next(ht, event) &&
459 random_healthtest_apt_next(ht, event)) {
460 if (ht->ht_state == STARTUP &&
461 ht->ht_total_samples >=
462 RANDOM_SELFTEST_STARTUP_SAMPLES) {
463 printf(
464 "random: health test passed for source %s\n",
465 random_source_descr[event->he_source]);
466 ht->ht_state = STEADY;
467 }
468 return (ht->ht_state == STEADY);
469 }
470 ht->ht_state = FAILED;
471 printf(
472 "random: health test failed for source %s, discarding samples\n",
473 random_source_descr[event->he_source]);
474 /* FALLTHROUGH */
475 case FAILED:
476 return (false);
477 }
478 }
479
480 static bool nist_healthtest_enabled = false;
481 SYSCTL_BOOL(_kern_random, OID_AUTO, nist_healthtest_enabled,
482 CTLFLAG_RDTUN, &nist_healthtest_enabled, 0,
483 "Enable NIST SP 800-90B health tests for noise sources");
484
485 static void
random_healthtest_init(enum random_entropy_source source,int min_entropy)486 random_healthtest_init(enum random_entropy_source source, int min_entropy)
487 {
488 struct health_test_softc *ht;
489
490 ht = &healthtest[source];
491 memset(ht, 0, sizeof(*ht));
492 KASSERT(ht->ht_state == INIT,
493 ("%s: health test state is %d for source %d",
494 __func__, ht->ht_state, source));
495
496 /*
497 * If health-testing is enabled, validate all sources except CACHED and
498 * VMGENID: they are deterministic sources used only a small, fixed
499 * number of times, so statistical testing is not applicable.
500 */
501 if (!nist_healthtest_enabled ||
502 source == RANDOM_CACHED || source == RANDOM_PURE_VMGENID) {
503 ht->ht_state = DISABLED;
504 return;
505 }
506
507 /*
508 * Set cutoff values for the two tests, given a min-entropy estimate for
509 * the source and allowing for an error rate of 1 in 2^{34}. With a
510 * min-entropy estimate of 1 bit and a sample rate of RANDOM_KTHREAD_HZ,
511 * we expect to see an false positive once in ~54.5 years.
512 *
513 * The RCT limit comes from the formula in section 4.4.1.
514 *
515 * The APT cutoffs are calculated using the formula in section 4.4.2
516 * footnote 10 with the number of Bernoulli trials changed from W to
517 * W-1, since the test as written counts the number of samples equal to
518 * the first sample in the window, and thus tests W-1 samples. We
519 * provide cutoffs for estimates up to sizeof(uint32_t)*HARVESTSIZE*8
520 * bits.
521 */
522 const int apt_cutoffs[] = {
523 [1] = 329,
524 [2] = 195,
525 [3] = 118,
526 [4] = 73,
527 [5] = 48,
528 [6] = 33,
529 [7] = 23,
530 [8] = 17,
531 [9] = 13,
532 [10] = 11,
533 [11] = 9,
534 [12] = 8,
535 [13] = 7,
536 [14] = 6,
537 [15] = 5,
538 [16] = 5,
539 [17 ... 19] = 4,
540 [20 ... 25] = 3,
541 [26 ... 42] = 2,
542 [43 ... 64] = 1,
543 };
544 const int error_rate = 34;
545
546 if (min_entropy == 0) {
547 /*
548 * For environmental sources, the main source of entropy is the
549 * associated timecounter value. Since these sources can be
550 * influenced by unprivileged users, we conservatively use a
551 * min-entropy estimate of 1 bit per sample. For "pure"
552 * sources, we assume 8 bits per sample, as such sources provide
553 * a variable amount of data per read and in particular might
554 * only provide a single byte at a time.
555 */
556 min_entropy = source >= RANDOM_PURE_START ? 8 : 1;
557 } else if (min_entropy < 0 || min_entropy >= nitems(apt_cutoffs)) {
558 panic("invalid min_entropy %d for %s", min_entropy,
559 random_source_descr[source]);
560 }
561
562 ht->ht_rct_limit = 1 + howmany(error_rate, min_entropy);
563 ht->ht_apt_cutoff = apt_cutoffs[min_entropy];
564 }
565
566 static int
random_healthtest_ondemand(SYSCTL_HANDLER_ARGS)567 random_healthtest_ondemand(SYSCTL_HANDLER_ARGS)
568 {
569 u_int mask, source;
570 int error;
571
572 mask = 0;
573 error = sysctl_handle_int(oidp, &mask, 0, req);
574 if (error != 0 || req->newptr == NULL)
575 return (error);
576
577 while (mask != 0) {
578 source = ffs(mask) - 1;
579 if (source < nitems(healthtest))
580 atomic_store_bool(&healthtest[source].ondemand, true);
581 mask &= ~(1u << source);
582 }
583 return (0);
584 }
585 SYSCTL_PROC(_kern_random, OID_AUTO, nist_healthtest_ondemand,
586 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0,
587 random_healthtest_ondemand, "I",
588 "Re-run NIST SP 800-90B startup health tests for a noise source");
589
590 static int
random_check_uint_harvestmask(SYSCTL_HANDLER_ARGS)591 random_check_uint_harvestmask(SYSCTL_HANDLER_ARGS)
592 {
593 static const u_int user_immutable_mask =
594 (((1 << ENTROPYSOURCE) - 1) & (-1UL << RANDOM_PURE_START)) |
595 _RANDOM_HARVEST_ETHER_OFF | _RANDOM_HARVEST_UMA_OFF;
596
597 int error;
598 u_int value;
599
600 value = atomic_load_int(&hc_source_mask);
601 error = sysctl_handle_int(oidp, &value, 0, req);
602 if (error != 0 || req->newptr == NULL)
603 return (error);
604
605 if (flsl(value) > ENTROPYSOURCE)
606 return (EINVAL);
607
608 /*
609 * Disallow userspace modification of pure entropy sources.
610 */
611 RANDOM_HARVEST_LOCK();
612 hc_source_mask = (value & ~user_immutable_mask) |
613 (hc_source_mask & user_immutable_mask);
614 RANDOM_HARVEST_UNLOCK();
615 return (0);
616 }
617 SYSCTL_PROC(_kern_random_harvest, OID_AUTO, mask,
618 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0,
619 random_check_uint_harvestmask, "IU",
620 "Entropy harvesting mask");
621
622 static int
random_print_harvestmask(SYSCTL_HANDLER_ARGS)623 random_print_harvestmask(SYSCTL_HANDLER_ARGS)
624 {
625 struct sbuf sbuf;
626 int error, i;
627
628 error = sysctl_wire_old_buffer(req, 0);
629 if (error == 0) {
630 u_int mask;
631
632 sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
633 mask = atomic_load_int(&hc_source_mask);
634 for (i = ENTROPYSOURCE - 1; i >= 0; i--) {
635 bool present;
636
637 present = (mask & (1u << i)) != 0;
638 sbuf_cat(&sbuf, present ? "1" : "0");
639 }
640 error = sbuf_finish(&sbuf);
641 sbuf_delete(&sbuf);
642 }
643 return (error);
644 }
645 SYSCTL_PROC(_kern_random_harvest, OID_AUTO, mask_bin,
646 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
647 random_print_harvestmask, "A",
648 "Entropy harvesting mask (printable)");
649
650 static const char *random_source_descr[ENTROPYSOURCE] = {
651 [RANDOM_CACHED] = "CACHED",
652 [RANDOM_ATTACH] = "ATTACH",
653 [RANDOM_KEYBOARD] = "KEYBOARD",
654 [RANDOM_MOUSE] = "MOUSE",
655 [RANDOM_NET_TUN] = "NET_TUN",
656 [RANDOM_NET_ETHER] = "NET_ETHER",
657 [RANDOM_NET_NG] = "NET_NG",
658 [RANDOM_INTERRUPT] = "INTERRUPT",
659 [RANDOM_SWI] = "SWI",
660 [RANDOM_FS_ATIME] = "FS_ATIME",
661 [RANDOM_UMA] = "UMA",
662 [RANDOM_CALLOUT] = "CALLOUT",
663 [RANDOM_RANDOMDEV] = "RANDOMDEV", /* ENVIRONMENTAL_END */
664 [RANDOM_PURE_OCTEON] = "PURE_OCTEON", /* PURE_START */
665 [RANDOM_PURE_SAFE] = "PURE_SAFE",
666 [RANDOM_PURE_GLXSB] = "PURE_GLXSB",
667 [RANDOM_PURE_HIFN] = "PURE_HIFN",
668 [RANDOM_PURE_RDRAND] = "PURE_RDRAND",
669 [RANDOM_PURE_NEHEMIAH] = "PURE_NEHEMIAH",
670 [RANDOM_PURE_RNDTEST] = "PURE_RNDTEST",
671 [RANDOM_PURE_VIRTIO] = "PURE_VIRTIO",
672 [RANDOM_PURE_BROADCOM] = "PURE_BROADCOM",
673 [RANDOM_PURE_CCP] = "PURE_CCP",
674 [RANDOM_PURE_DARN] = "PURE_DARN",
675 [RANDOM_PURE_TPM] = "PURE_TPM",
676 [RANDOM_PURE_VMGENID] = "PURE_VMGENID",
677 [RANDOM_PURE_QUALCOMM] = "PURE_QUALCOMM",
678 [RANDOM_PURE_ARMV8] = "PURE_ARMV8",
679 [RANDOM_PURE_ARM_TRNG] = "PURE_ARM_TRNG",
680 /* "ENTROPYSOURCE" */
681 };
682
683 static int
random_print_harvestmask_symbolic(SYSCTL_HANDLER_ARGS)684 random_print_harvestmask_symbolic(SYSCTL_HANDLER_ARGS)
685 {
686 struct sbuf sbuf;
687 int error, i;
688 bool first;
689
690 first = true;
691 error = sysctl_wire_old_buffer(req, 0);
692 if (error == 0) {
693 u_int mask;
694
695 sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
696 mask = atomic_load_int(&hc_source_mask);
697 for (i = ENTROPYSOURCE - 1; i >= 0; i--) {
698 bool present;
699
700 present = (mask & (1u << i)) != 0;
701 if (i >= RANDOM_PURE_START && !present)
702 continue;
703 if (!first)
704 sbuf_cat(&sbuf, ",");
705 sbuf_cat(&sbuf, !present ? "[" : "");
706 sbuf_cat(&sbuf, random_source_descr[i]);
707 sbuf_cat(&sbuf, !present ? "]" : "");
708 first = false;
709 }
710 error = sbuf_finish(&sbuf);
711 sbuf_delete(&sbuf);
712 }
713 return (error);
714 }
715 SYSCTL_PROC(_kern_random_harvest, OID_AUTO, mask_symbolic,
716 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
717 random_print_harvestmask_symbolic, "A",
718 "Entropy harvesting mask (symbolic)");
719
720 static void
random_harvestq_init(void * unused __unused)721 random_harvestq_init(void *unused __unused)
722 {
723 static const u_int almost_everything_mask =
724 (((1 << (RANDOM_ENVIRONMENTAL_END + 1)) - 1) &
725 ~_RANDOM_HARVEST_ETHER_OFF & ~_RANDOM_HARVEST_UMA_OFF);
726
727 hc_source_mask = almost_everything_mask;
728 RANDOM_HARVEST_INIT_LOCK();
729 harvest_context.hc_active_buf = 0;
730
731 for (int i = RANDOM_START; i <= RANDOM_ENVIRONMENTAL_END; i++)
732 random_healthtest_init(i, 0);
733 }
734 SYSINIT(random_device_h_init, SI_SUB_RANDOM, SI_ORDER_THIRD, random_harvestq_init, NULL);
735
736 /*
737 * Subroutine to slice up a contiguous chunk of 'entropy' and feed it into the
738 * underlying algorithm. Returns number of bytes actually fed into underlying
739 * algorithm.
740 */
741 static size_t
random_early_prime(char * entropy,size_t len)742 random_early_prime(char *entropy, size_t len)
743 {
744 struct harvest_event event;
745 size_t i;
746
747 len = rounddown(len, sizeof(event.he_entropy));
748 if (len == 0)
749 return (0);
750
751 for (i = 0; i < len; i += sizeof(event.he_entropy)) {
752 event.he_somecounter = random_get_cyclecount();
753 event.he_size = sizeof(event.he_entropy);
754 event.he_source = RANDOM_CACHED;
755 event.he_destination =
756 harvest_context.hc_destination[RANDOM_CACHED]++;
757 memcpy(event.he_entropy, entropy + i, sizeof(event.he_entropy));
758 random_harvestq_fast_process_event(&event);
759 }
760 explicit_bzero(entropy, len);
761 return (len);
762 }
763
764 /*
765 * Subroutine to search for known loader-loaded files in memory and feed them
766 * into the underlying algorithm early in boot. Returns the number of bytes
767 * loaded (zero if none were loaded).
768 */
769 static size_t
random_prime_loader_file(const char * type)770 random_prime_loader_file(const char *type)
771 {
772 uint8_t *keyfile, *data;
773 size_t size;
774
775 keyfile = preload_search_by_type(type);
776 if (keyfile == NULL)
777 return (0);
778
779 data = preload_fetch_addr(keyfile);
780 size = preload_fetch_size(keyfile);
781 if (data == NULL)
782 return (0);
783
784 return (random_early_prime(data, size));
785 }
786
787 /*
788 * This is used to prime the RNG by grabbing any early random stuff
789 * known to the kernel, and inserting it directly into the hashing
790 * module, currently Fortuna.
791 */
792 static void
random_harvestq_prime(void * unused __unused)793 random_harvestq_prime(void *unused __unused)
794 {
795 size_t size;
796
797 /*
798 * Get entropy that may have been preloaded by loader(8)
799 * and use it to pre-charge the entropy harvest queue.
800 */
801 size = random_prime_loader_file(RANDOM_CACHED_BOOT_ENTROPY_MODULE);
802 if (bootverbose) {
803 if (size > 0)
804 printf("random: read %zu bytes from preloaded cache\n",
805 size);
806 else
807 printf("random: no preloaded entropy cache\n");
808 }
809 size = random_prime_loader_file(RANDOM_PLATFORM_BOOT_ENTROPY_MODULE);
810 if (bootverbose) {
811 if (size > 0)
812 printf("random: read %zu bytes from platform bootloader\n",
813 size);
814 else
815 printf("random: no platform bootloader entropy\n");
816 }
817 }
818 SYSINIT(random_device_prime, SI_SUB_RANDOM, SI_ORDER_MIDDLE, random_harvestq_prime, NULL);
819
820 static void
random_harvestq_deinit(void * unused __unused)821 random_harvestq_deinit(void *unused __unused)
822 {
823
824 /* Command the hash/reseed thread to end and wait for it to finish */
825 random_kthread_control = 0;
826 while (random_kthread_control >= 0)
827 tsleep(&harvest_context.hc_kthread_proc, 0, "harvqterm", hz/5);
828 }
829 SYSUNINIT(random_device_h_init, SI_SUB_RANDOM, SI_ORDER_THIRD, random_harvestq_deinit, NULL);
830
831 /*-
832 * Entropy harvesting queue routine.
833 *
834 * This is supposed to be fast; do not do anything slow in here!
835 * It is also illegal (and morally reprehensible) to insert any
836 * high-rate data here. "High-rate" is defined as a data source
837 * that is likely to fill up the buffer in much less than 100ms.
838 * This includes the "always-on" sources like the Intel "rdrand"
839 * or the VIA Nehamiah "xstore" sources.
840 */
841 /* XXXRW: get_cyclecount() is cheap on most modern hardware, where cycle
842 * counters are built in, but on older hardware it will do a real time clock
843 * read which can be quite expensive.
844 */
845 void
random_harvest_queue_(const void * entropy,u_int size,enum random_entropy_source origin)846 random_harvest_queue_(const void *entropy, u_int size, enum random_entropy_source origin)
847 {
848 struct harvest_context *hc;
849 struct entropy_buffer *buf;
850 struct harvest_event *event;
851
852 KASSERT(origin >= RANDOM_START && origin < ENTROPYSOURCE,
853 ("%s: origin %d invalid", __func__, origin));
854
855 hc = &harvest_context;
856 RANDOM_HARVEST_LOCK();
857 buf = &hc->hc_entropy_buf[hc->hc_active_buf];
858 if (buf->pos < RANDOM_RING_MAX) {
859 event = &buf->ring[buf->pos++];
860 event->he_somecounter = random_get_cyclecount();
861 event->he_source = origin;
862 event->he_destination = hc->hc_destination[origin]++;
863 if (size <= sizeof(event->he_entropy)) {
864 event->he_size = size;
865 memcpy(event->he_entropy, entropy, size);
866 } else {
867 /* Big event, so squash it */
868 event->he_size = sizeof(event->he_entropy[0]);
869 event->he_entropy[0] = jenkins_hash(entropy, size, (uint32_t)(uintptr_t)event);
870 }
871 }
872 RANDOM_HARVEST_UNLOCK();
873 }
874
875 /*-
876 * Entropy harvesting fast routine.
877 *
878 * This is supposed to be very fast; do not do anything slow in here!
879 * This is the right place for high-rate harvested data.
880 */
881 void
random_harvest_fast_(const void * entropy,u_int size)882 random_harvest_fast_(const void *entropy, u_int size)
883 {
884 u_int pos;
885
886 pos = harvest_context.hc_entropy_fast_accumulator.pos;
887 harvest_context.hc_entropy_fast_accumulator.buf[pos] ^=
888 jenkins_hash(entropy, size, random_get_cyclecount());
889 harvest_context.hc_entropy_fast_accumulator.pos = (pos + 1)%RANDOM_ACCUM_MAX;
890 }
891
892 /*-
893 * Entropy harvesting direct routine.
894 *
895 * This is not supposed to be fast, but will only be used during
896 * (e.g.) booting when initial entropy is being gathered.
897 */
898 void
random_harvest_direct_(const void * entropy,u_int size,enum random_entropy_source origin)899 random_harvest_direct_(const void *entropy, u_int size, enum random_entropy_source origin)
900 {
901 struct harvest_event event;
902
903 KASSERT(origin >= RANDOM_START && origin < ENTROPYSOURCE, ("%s: origin %d invalid\n", __func__, origin));
904 size = MIN(size, sizeof(event.he_entropy));
905 event.he_somecounter = random_get_cyclecount();
906 event.he_size = size;
907 event.he_source = origin;
908 event.he_destination = harvest_context.hc_destination[origin]++;
909 memcpy(event.he_entropy, entropy, size);
910 random_harvestq_fast_process_event(&event);
911 }
912
913 void
random_source_register(const struct random_source * rsource)914 random_source_register(const struct random_source *rsource)
915 {
916 struct random_sources *rrs;
917
918 KASSERT(rsource != NULL, ("invalid input to %s", __func__));
919
920 rrs = malloc(sizeof(*rrs), M_ENTROPY, M_WAITOK);
921 rrs->rrs_source = rsource;
922
923 printf("random: registering fast source %s\n", rsource->rs_ident);
924
925 random_healthtest_init(rsource->rs_source, rsource->rs_min_entropy);
926
927 RANDOM_HARVEST_LOCK();
928 hc_source_mask |= (1 << rsource->rs_source);
929 CK_LIST_INSERT_HEAD(&source_list, rrs, rrs_entries);
930 RANDOM_HARVEST_UNLOCK();
931 }
932
933 void
random_source_deregister(const struct random_source * rsource)934 random_source_deregister(const struct random_source *rsource)
935 {
936 struct random_sources *rrs = NULL;
937
938 KASSERT(rsource != NULL, ("invalid input to %s", __func__));
939
940 RANDOM_HARVEST_LOCK();
941 hc_source_mask &= ~(1 << rsource->rs_source);
942 CK_LIST_FOREACH(rrs, &source_list, rrs_entries)
943 if (rrs->rrs_source == rsource) {
944 CK_LIST_REMOVE(rrs, rrs_entries);
945 break;
946 }
947 RANDOM_HARVEST_UNLOCK();
948
949 if (rrs != NULL && epoch_inited)
950 epoch_wait_preempt(rs_epoch);
951 free(rrs, M_ENTROPY);
952 }
953
954 static int
random_source_handler(SYSCTL_HANDLER_ARGS)955 random_source_handler(SYSCTL_HANDLER_ARGS)
956 {
957 struct epoch_tracker et;
958 struct random_sources *rrs;
959 struct sbuf sbuf;
960 int error, count;
961
962 error = sysctl_wire_old_buffer(req, 0);
963 if (error != 0)
964 return (error);
965
966 sbuf_new_for_sysctl(&sbuf, NULL, 64, req);
967 count = 0;
968 epoch_enter_preempt(rs_epoch, &et);
969 CK_LIST_FOREACH(rrs, &source_list, rrs_entries) {
970 sbuf_cat(&sbuf, (count++ ? ",'" : "'"));
971 sbuf_cat(&sbuf, rrs->rrs_source->rs_ident);
972 sbuf_cat(&sbuf, "'");
973 }
974 epoch_exit_preempt(rs_epoch, &et);
975 error = sbuf_finish(&sbuf);
976 sbuf_delete(&sbuf);
977 return (error);
978 }
979 SYSCTL_PROC(_kern_random, OID_AUTO, random_sources, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
980 NULL, 0, random_source_handler, "A",
981 "List of active fast entropy sources.");
982
983 MODULE_VERSION(random_harvestq, 1);
984