1 /*-
2 * Copyright (c) 2017 Oliver Pinter
3 * Copyright (c) 2017 W. Dean Freeman
4 * Copyright (c) 2000-2015 Mark R V Murray
5 * Copyright (c) 2013 Arthur Mesh
6 * Copyright (c) 2004 Robert N. M. Watson
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer
14 * in this position and unchanged.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 */
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/ck.h>
35 #include <sys/conf.h>
36 #include <sys/epoch.h>
37 #include <sys/eventhandler.h>
38 #include <sys/hash.h>
39 #include <sys/kernel.h>
40 #include <sys/kthread.h>
41 #include <sys/linker.h>
42 #include <sys/lock.h>
43 #include <sys/malloc.h>
44 #include <sys/module.h>
45 #include <sys/mutex.h>
46 #include <sys/random.h>
47 #include <sys/sbuf.h>
48 #include <sys/sysctl.h>
49 #include <sys/unistd.h>
50
51 #include <machine/atomic.h>
52 #include <machine/cpu.h>
53
54 #include <crypto/rijndael/rijndael-api-fst.h>
55 #include <crypto/sha2/sha256.h>
56
57 #include <dev/random/fortuna.h>
58 #include <dev/random/hash.h>
59 #include <dev/random/randomdev.h>
60 #include <dev/random/random_harvestq.h>
61
62 #if defined(RANDOM_ENABLE_ETHER)
63 #define _RANDOM_HARVEST_ETHER_OFF 0
64 #else
65 #define _RANDOM_HARVEST_ETHER_OFF (1u << RANDOM_NET_ETHER)
66 #endif
67 #if defined(RANDOM_ENABLE_UMA)
68 #define _RANDOM_HARVEST_UMA_OFF 0
69 #else
70 #define _RANDOM_HARVEST_UMA_OFF (1u << RANDOM_UMA)
71 #endif
72
73 /*
74 * Note that random_sources_feed() will also use this to try and split up
75 * entropy into a subset of pools per iteration with the goal of feeding
76 * HARVESTSIZE into every pool at least once per second.
77 */
78 #define RANDOM_KTHREAD_HZ 10
79
80 static void random_kthread(void);
81 static void random_sources_feed(void);
82
83 /*
84 * Random must initialize much earlier than epoch, but we can initialize the
85 * epoch code before SMP starts. Prior to SMP, we can safely bypass
86 * concurrency primitives.
87 */
88 static __read_mostly bool epoch_inited;
89 static __read_mostly epoch_t rs_epoch;
90
91 static const char *random_source_descr[];
92
93 /*
94 * How many events to queue up. We create this many items in
95 * an 'empty' queue, then transfer them to the 'harvest' queue with
96 * supplied junk. When used, they are transferred back to the
97 * 'empty' queue.
98 */
99 #define RANDOM_RING_MAX 1024
100 #define RANDOM_ACCUM_MAX 8
101
102 /* 1 to let the kernel thread run, 0 to terminate, -1 to mark completion */
103 volatile int random_kthread_control;
104
105
106 /*
107 * Allow the sysadmin to select the broad category of entropy types to harvest.
108 *
109 * Updates are synchronized by the harvest mutex.
110 */
111 __read_frequently u_int hc_source_mask;
112 CTASSERT(ENTROPYSOURCE <= sizeof(hc_source_mask) * NBBY);
113
114 struct random_sources {
115 CK_LIST_ENTRY(random_sources) rrs_entries;
116 const struct random_source *rrs_source;
117 };
118
119 static CK_LIST_HEAD(sources_head, random_sources) source_list =
120 CK_LIST_HEAD_INITIALIZER(source_list);
121
122 SYSCTL_NODE(_kern_random, OID_AUTO, harvest, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
123 "Entropy Device Parameters");
124
125 /*
126 * Put all the harvest queue context stuff in one place.
127 * this make is a bit easier to lock and protect.
128 */
129 static struct harvest_context {
130 /* The harvest mutex protects all of harvest_context and
131 * the related data.
132 */
133 struct mtx hc_mtx;
134 /* Round-robin destination cache. */
135 u_int hc_destination[ENTROPYSOURCE];
136 /* The context of the kernel thread processing harvested entropy */
137 struct proc *hc_kthread_proc;
138 /*
139 * A pair of buffers for queued events. New events are added to the
140 * active queue while the kthread processes the other one in parallel.
141 */
142 struct entropy_buffer {
143 struct harvest_event ring[RANDOM_RING_MAX];
144 u_int pos;
145 } hc_entropy_buf[2];
146 u_int hc_active_buf;
147 struct fast_entropy_accumulator {
148 volatile u_int pos;
149 uint32_t buf[RANDOM_ACCUM_MAX];
150 } hc_entropy_fast_accumulator;
151 } harvest_context;
152
153 #define RANDOM_HARVEST_INIT_LOCK() mtx_init(&harvest_context.hc_mtx, \
154 "entropy harvest mutex", NULL, MTX_SPIN)
155 #define RANDOM_HARVEST_LOCK() mtx_lock_spin(&harvest_context.hc_mtx)
156 #define RANDOM_HARVEST_UNLOCK() mtx_unlock_spin(&harvest_context.hc_mtx)
157
158 static struct kproc_desc random_proc_kp = {
159 "rand_harvestq",
160 random_kthread,
161 &harvest_context.hc_kthread_proc,
162 };
163
164 /* Pass the given event straight through to Fortuna/Whatever. */
165 static __inline void
random_harvestq_fast_process_event(struct harvest_event * event)166 random_harvestq_fast_process_event(struct harvest_event *event)
167 {
168 p_random_alg_context->ra_event_processor(event);
169 explicit_bzero(event, sizeof(*event));
170 }
171
172 static void
random_kthread(void)173 random_kthread(void)
174 {
175 struct harvest_context *hc;
176
177 hc = &harvest_context;
178 for (random_kthread_control = 1; random_kthread_control;) {
179 struct entropy_buffer *buf;
180 u_int entries;
181
182 /* Deal with queued events. */
183 RANDOM_HARVEST_LOCK();
184 buf = &hc->hc_entropy_buf[hc->hc_active_buf];
185 entries = buf->pos;
186 buf->pos = 0;
187 hc->hc_active_buf = (hc->hc_active_buf + 1) %
188 nitems(hc->hc_entropy_buf);
189 RANDOM_HARVEST_UNLOCK();
190 for (u_int i = 0; i < entries; i++)
191 random_harvestq_fast_process_event(&buf->ring[i]);
192
193 /* Poll sources of noise. */
194 random_sources_feed();
195
196 /* XXX: FIX!! Increase the high-performance data rate? Need some measurements first. */
197 for (u_int i = 0; i < RANDOM_ACCUM_MAX; i++) {
198 if (hc->hc_entropy_fast_accumulator.buf[i]) {
199 random_harvest_direct(&hc->hc_entropy_fast_accumulator.buf[i],
200 sizeof(hc->hc_entropy_fast_accumulator.buf[0]), RANDOM_UMA);
201 hc->hc_entropy_fast_accumulator.buf[i] = 0;
202 }
203 }
204 /* XXX: FIX!! This is a *great* place to pass hardware/live entropy to random(9) */
205 tsleep_sbt(&hc->hc_kthread_proc, 0, "-",
206 SBT_1S/RANDOM_KTHREAD_HZ, 0, C_PREL(1));
207 }
208 random_kthread_control = -1;
209 wakeup(&hc->hc_kthread_proc);
210 kproc_exit(0);
211 /* NOTREACHED */
212 }
213 SYSINIT(random_device_h_proc, SI_SUB_KICK_SCHEDULER, SI_ORDER_ANY, kproc_start,
214 &random_proc_kp);
215 _Static_assert(SI_SUB_KICK_SCHEDULER > SI_SUB_RANDOM,
216 "random kthread starting before subsystem initialization");
217
218 static void
rs_epoch_init(void * dummy __unused)219 rs_epoch_init(void *dummy __unused)
220 {
221 rs_epoch = epoch_alloc("Random Sources", EPOCH_PREEMPT);
222 epoch_inited = true;
223 }
224 SYSINIT(rs_epoch_init, SI_SUB_EPOCH, SI_ORDER_ANY, rs_epoch_init, NULL);
225
226 /*
227 * Run through all fast sources reading entropy for the given
228 * number of rounds, which should be a multiple of the number
229 * of entropy accumulation pools in use; it is 32 for Fortuna.
230 */
231 static void
random_sources_feed(void)232 random_sources_feed(void)
233 {
234 uint32_t entropy[HARVESTSIZE];
235 struct epoch_tracker et;
236 struct random_sources *rrs;
237 u_int i, n, npools;
238 bool rse_warm;
239
240 rse_warm = epoch_inited;
241
242 /*
243 * Evenly-ish distribute pool population across the second based on how
244 * frequently random_kthread iterates.
245 *
246 * For Fortuna, the math currently works out as such:
247 *
248 * 64 bits * 4 pools = 256 bits per iteration
249 * 256 bits * 10 Hz = 2560 bits per second, 320 B/s
250 *
251 */
252 npools = howmany(p_random_alg_context->ra_poolcount, RANDOM_KTHREAD_HZ);
253
254 /*-
255 * If we're not seeded yet, attempt to perform a "full seed", filling
256 * all of the PRNG's pools with entropy; if there is enough entropy
257 * available from "fast" entropy sources this will allow us to finish
258 * seeding and unblock the boot process immediately rather than being
259 * stuck for a few seconds with random_kthread gradually collecting a
260 * small chunk of entropy every 1 / RANDOM_KTHREAD_HZ seconds.
261 *
262 * We collect RANDOM_FORTUNA_DEFPOOLSIZE bytes per pool, i.e. enough
263 * to fill Fortuna's pools in the default configuration. With another
264 * PRNG or smaller pools for Fortuna, we might collect more entropy
265 * than needed to fill the pools, but this is harmless; alternatively,
266 * a different PRNG, larger pools, or fast entropy sources which are
267 * not able to provide as much entropy as we request may result in the
268 * not being fully seeded (and thus remaining blocked) but in that
269 * case we will return here after 1 / RANDOM_KTHREAD_HZ seconds and
270 * try again for a large amount of entropy.
271 */
272 if (!p_random_alg_context->ra_seeded())
273 npools = howmany(p_random_alg_context->ra_poolcount *
274 RANDOM_FORTUNA_DEFPOOLSIZE, sizeof(entropy));
275
276 /*
277 * Step over all of live entropy sources, and feed their output
278 * to the system-wide RNG.
279 */
280 if (rse_warm)
281 epoch_enter_preempt(rs_epoch, &et);
282 CK_LIST_FOREACH(rrs, &source_list, rrs_entries) {
283 for (i = 0; i < npools; i++) {
284 if (rrs->rrs_source->rs_read == NULL) {
285 /* Source pushes entropy asynchronously. */
286 continue;
287 }
288 n = rrs->rrs_source->rs_read(entropy, sizeof(entropy));
289 KASSERT((n <= sizeof(entropy)),
290 ("%s: rs_read returned too much data (%u > %zu)",
291 __func__, n, sizeof(entropy)));
292
293 /*
294 * Sometimes the HW entropy source doesn't have anything
295 * ready for us. This isn't necessarily untrustworthy.
296 * We don't perform any other verification of an entropy
297 * source (i.e., length is allowed to be anywhere from 1
298 * to sizeof(entropy), quality is unchecked, etc), so
299 * don't balk verbosely at slow random sources either.
300 * There are reports that RDSEED on x86 metal falls
301 * behind the rate at which we query it, for example.
302 * But it's still a better entropy source than RDRAND.
303 */
304 if (n == 0)
305 continue;
306 random_harvest_direct(entropy, n, rrs->rrs_source->rs_source);
307 }
308 }
309 if (rse_warm)
310 epoch_exit_preempt(rs_epoch, &et);
311 explicit_bzero(entropy, sizeof(entropy));
312 }
313
314 /*
315 * State used for conducting NIST SP 800-90B health tests on entropy sources.
316 */
317 static struct health_test_softc {
318 uint32_t ht_rct_value[HARVESTSIZE + 1];
319 u_int ht_rct_count; /* number of samples with the same value */
320 u_int ht_rct_limit; /* constant after init */
321
322 uint32_t ht_apt_value[HARVESTSIZE + 1];
323 u_int ht_apt_count; /* number of samples with the same value */
324 u_int ht_apt_seq; /* sequence number of the last sample */
325 u_int ht_apt_cutoff; /* constant after init */
326
327 uint64_t ht_total_samples;
328 bool ondemand; /* Set to true to restart the state machine */
329 enum {
330 INIT = 0, /* initial state */
331 DISABLED, /* health checking is disabled */
332 STARTUP, /* doing startup tests, samples are discarded */
333 STEADY, /* steady-state operation */
334 FAILED, /* health check failed, discard samples */
335 } ht_state;
336 } healthtest[ENTROPYSOURCE];
337
338 #define RANDOM_SELFTEST_STARTUP_SAMPLES 1024 /* 4.3, requirement 4 */
339 #define RANDOM_SELFTEST_APT_WINDOW 512 /* 4.4.2 */
340
341 static void
copy_event(uint32_t dst[static HARVESTSIZE+1],const struct harvest_event * event)342 copy_event(uint32_t dst[static HARVESTSIZE + 1],
343 const struct harvest_event *event)
344 {
345 memset(dst, 0, sizeof(uint32_t) * (HARVESTSIZE + 1));
346 memcpy(dst, event->he_entropy, event->he_size);
347 if (event->he_source <= RANDOM_ENVIRONMENTAL_END) {
348 /*
349 * For pure entropy sources the timestamp counter is generally
350 * quite determinstic since samples are taken at regular
351 * intervals, so does not contribute much to the entropy. To
352 * make health tests more effective, exclude it from the sample,
353 * since it might otherwise defeat the health tests in a
354 * scenario where the source is stuck.
355 */
356 dst[HARVESTSIZE] = event->he_somecounter;
357 }
358 }
359
360 static void
random_healthtest_rct_init(struct health_test_softc * ht,const struct harvest_event * event)361 random_healthtest_rct_init(struct health_test_softc *ht,
362 const struct harvest_event *event)
363 {
364 ht->ht_rct_count = 1;
365 copy_event(ht->ht_rct_value, event);
366 }
367
368 /*
369 * Apply the repitition count test to a sample.
370 *
371 * Return false if the test failed, i.e., we observed >= C consecutive samples
372 * with the same value, and true otherwise.
373 */
374 static bool
random_healthtest_rct_next(struct health_test_softc * ht,const struct harvest_event * event)375 random_healthtest_rct_next(struct health_test_softc *ht,
376 const struct harvest_event *event)
377 {
378 uint32_t val[HARVESTSIZE + 1];
379
380 copy_event(val, event);
381 if (memcmp(val, ht->ht_rct_value, sizeof(ht->ht_rct_value)) != 0) {
382 ht->ht_rct_count = 1;
383 memcpy(ht->ht_rct_value, val, sizeof(ht->ht_rct_value));
384 return (true);
385 } else {
386 ht->ht_rct_count++;
387 return (ht->ht_rct_count < ht->ht_rct_limit);
388 }
389 }
390
391 static void
random_healthtest_apt_init(struct health_test_softc * ht,const struct harvest_event * event)392 random_healthtest_apt_init(struct health_test_softc *ht,
393 const struct harvest_event *event)
394 {
395 ht->ht_apt_count = 1;
396 ht->ht_apt_seq = 1;
397 copy_event(ht->ht_apt_value, event);
398 }
399
400 static bool
random_healthtest_apt_next(struct health_test_softc * ht,const struct harvest_event * event)401 random_healthtest_apt_next(struct health_test_softc *ht,
402 const struct harvest_event *event)
403 {
404 uint32_t val[HARVESTSIZE + 1];
405
406 if (ht->ht_apt_seq == 0) {
407 random_healthtest_apt_init(ht, event);
408 return (true);
409 }
410
411 copy_event(val, event);
412 if (memcmp(val, ht->ht_apt_value, sizeof(ht->ht_apt_value)) == 0) {
413 ht->ht_apt_count++;
414 if (ht->ht_apt_count >= ht->ht_apt_cutoff)
415 return (false);
416 }
417
418 ht->ht_apt_seq++;
419 if (ht->ht_apt_seq == RANDOM_SELFTEST_APT_WINDOW)
420 ht->ht_apt_seq = 0;
421
422 return (true);
423 }
424
425 /*
426 * Run the health tests for the given event. This is assumed to be called from
427 * a serialized context.
428 */
429 bool
random_harvest_healthtest(const struct harvest_event * event)430 random_harvest_healthtest(const struct harvest_event *event)
431 {
432 struct health_test_softc *ht;
433
434 ht = &healthtest[event->he_source];
435
436 /*
437 * Was on-demand testing requested? Restart the state machine if so,
438 * restarting the startup tests.
439 */
440 if (atomic_load_bool(&ht->ondemand)) {
441 atomic_store_bool(&ht->ondemand, false);
442 ht->ht_state = INIT;
443 }
444
445 switch (ht->ht_state) {
446 case __predict_false(INIT):
447 /* Store the first sample and initialize test state. */
448 random_healthtest_rct_init(ht, event);
449 random_healthtest_apt_init(ht, event);
450 ht->ht_total_samples = 0;
451 ht->ht_state = STARTUP;
452 return (false);
453 case DISABLED:
454 /* No health testing for this source. */
455 return (true);
456 case STEADY:
457 case STARTUP:
458 ht->ht_total_samples++;
459 if (random_healthtest_rct_next(ht, event) &&
460 random_healthtest_apt_next(ht, event)) {
461 if (ht->ht_state == STARTUP &&
462 ht->ht_total_samples >=
463 RANDOM_SELFTEST_STARTUP_SAMPLES) {
464 printf(
465 "random: health test passed for source %s\n",
466 random_source_descr[event->he_source]);
467 ht->ht_state = STEADY;
468 }
469 return (ht->ht_state == STEADY);
470 }
471 ht->ht_state = FAILED;
472 printf(
473 "random: health test failed for source %s, discarding samples\n",
474 random_source_descr[event->he_source]);
475 /* FALLTHROUGH */
476 case FAILED:
477 return (false);
478 }
479 }
480
481 static bool nist_healthtest_enabled = false;
482 SYSCTL_BOOL(_kern_random, OID_AUTO, nist_healthtest_enabled,
483 CTLFLAG_RDTUN, &nist_healthtest_enabled, 0,
484 "Enable NIST SP 800-90B health tests for noise sources");
485
486 static void
random_healthtest_init(enum random_entropy_source source,int min_entropy)487 random_healthtest_init(enum random_entropy_source source, int min_entropy)
488 {
489 struct health_test_softc *ht;
490
491 ht = &healthtest[source];
492 memset(ht, 0, sizeof(*ht));
493 KASSERT(ht->ht_state == INIT,
494 ("%s: health test state is %d for source %d",
495 __func__, ht->ht_state, source));
496
497 /*
498 * If health-testing is enabled, validate all sources except CACHED and
499 * VMGENID: they are deterministic sources used only a small, fixed
500 * number of times, so statistical testing is not applicable.
501 */
502 if (!nist_healthtest_enabled ||
503 source == RANDOM_CACHED || source == RANDOM_PURE_VMGENID) {
504 ht->ht_state = DISABLED;
505 return;
506 }
507
508 /*
509 * Set cutoff values for the two tests, given a min-entropy estimate for
510 * the source and allowing for an error rate of 1 in 2^{34}. With a
511 * min-entropy estimate of 1 bit and a sample rate of RANDOM_KTHREAD_HZ,
512 * we expect to see an false positive once in ~54.5 years.
513 *
514 * The RCT limit comes from the formula in section 4.4.1.
515 *
516 * The APT cutoffs are calculated using the formula in section 4.4.2
517 * footnote 10 with the number of Bernoulli trials changed from W to
518 * W-1, since the test as written counts the number of samples equal to
519 * the first sample in the window, and thus tests W-1 samples. We
520 * provide cutoffs for estimates up to sizeof(uint32_t)*HARVESTSIZE*8
521 * bits.
522 */
523 const int apt_cutoffs[] = {
524 [1] = 329,
525 [2] = 195,
526 [3] = 118,
527 [4] = 73,
528 [5] = 48,
529 [6] = 33,
530 [7] = 23,
531 [8] = 17,
532 [9] = 13,
533 [10] = 11,
534 [11] = 9,
535 [12] = 8,
536 [13] = 7,
537 [14] = 6,
538 [15] = 5,
539 [16] = 5,
540 [17 ... 19] = 4,
541 [20 ... 25] = 3,
542 [26 ... 42] = 2,
543 [43 ... 64] = 1,
544 };
545 const int error_rate = 34;
546
547 if (min_entropy == 0) {
548 /*
549 * For environmental sources, the main source of entropy is the
550 * associated timecounter value. Since these sources can be
551 * influenced by unprivileged users, we conservatively use a
552 * min-entropy estimate of 1 bit per sample. For "pure"
553 * sources, we assume 8 bits per sample, as such sources provide
554 * a variable amount of data per read and in particular might
555 * only provide a single byte at a time.
556 */
557 min_entropy = source >= RANDOM_PURE_START ? 8 : 1;
558 } else if (min_entropy < 0 || min_entropy >= nitems(apt_cutoffs)) {
559 panic("invalid min_entropy %d for %s", min_entropy,
560 random_source_descr[source]);
561 }
562
563 ht->ht_rct_limit = 1 + howmany(error_rate, min_entropy);
564 ht->ht_apt_cutoff = apt_cutoffs[min_entropy];
565 }
566
567 static int
random_healthtest_ondemand(SYSCTL_HANDLER_ARGS)568 random_healthtest_ondemand(SYSCTL_HANDLER_ARGS)
569 {
570 u_int mask, source;
571 int error;
572
573 mask = 0;
574 error = sysctl_handle_int(oidp, &mask, 0, req);
575 if (error != 0 || req->newptr == NULL)
576 return (error);
577
578 while (mask != 0) {
579 source = ffs(mask) - 1;
580 if (source < nitems(healthtest))
581 atomic_store_bool(&healthtest[source].ondemand, true);
582 mask &= ~(1u << source);
583 }
584 return (0);
585 }
586 SYSCTL_PROC(_kern_random, OID_AUTO, nist_healthtest_ondemand,
587 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0,
588 random_healthtest_ondemand, "I",
589 "Re-run NIST SP 800-90B startup health tests for a noise source");
590
591 static int
random_check_uint_harvestmask(SYSCTL_HANDLER_ARGS)592 random_check_uint_harvestmask(SYSCTL_HANDLER_ARGS)
593 {
594 static const u_int user_immutable_mask =
595 (((1 << ENTROPYSOURCE) - 1) & (-1UL << RANDOM_PURE_START)) |
596 _RANDOM_HARVEST_ETHER_OFF | _RANDOM_HARVEST_UMA_OFF;
597
598 int error;
599 u_int value;
600
601 value = atomic_load_int(&hc_source_mask);
602 error = sysctl_handle_int(oidp, &value, 0, req);
603 if (error != 0 || req->newptr == NULL)
604 return (error);
605
606 if (flsl(value) > ENTROPYSOURCE)
607 return (EINVAL);
608
609 /*
610 * Disallow userspace modification of pure entropy sources.
611 */
612 RANDOM_HARVEST_LOCK();
613 hc_source_mask = (value & ~user_immutable_mask) |
614 (hc_source_mask & user_immutable_mask);
615 RANDOM_HARVEST_UNLOCK();
616 return (0);
617 }
618 SYSCTL_PROC(_kern_random_harvest, OID_AUTO, mask,
619 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0,
620 random_check_uint_harvestmask, "IU",
621 "Entropy harvesting mask");
622
623 static int
random_print_harvestmask(SYSCTL_HANDLER_ARGS)624 random_print_harvestmask(SYSCTL_HANDLER_ARGS)
625 {
626 struct sbuf sbuf;
627 int error, i;
628
629 error = sysctl_wire_old_buffer(req, 0);
630 if (error == 0) {
631 u_int mask;
632
633 sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
634 mask = atomic_load_int(&hc_source_mask);
635 for (i = ENTROPYSOURCE - 1; i >= 0; i--) {
636 bool present;
637
638 present = (mask & (1u << i)) != 0;
639 sbuf_cat(&sbuf, present ? "1" : "0");
640 }
641 error = sbuf_finish(&sbuf);
642 sbuf_delete(&sbuf);
643 }
644 return (error);
645 }
646 SYSCTL_PROC(_kern_random_harvest, OID_AUTO, mask_bin,
647 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
648 random_print_harvestmask, "A",
649 "Entropy harvesting mask (printable)");
650
651 static const char *random_source_descr[/*ENTROPYSOURCE*/] = {
652 [RANDOM_CACHED] = "CACHED",
653 [RANDOM_ATTACH] = "ATTACH",
654 [RANDOM_KEYBOARD] = "KEYBOARD",
655 [RANDOM_MOUSE] = "MOUSE",
656 [RANDOM_NET_TUN] = "NET_TUN",
657 [RANDOM_NET_ETHER] = "NET_ETHER",
658 [RANDOM_NET_NG] = "NET_NG",
659 [RANDOM_INTERRUPT] = "INTERRUPT",
660 [RANDOM_SWI] = "SWI",
661 [RANDOM_FS_ATIME] = "FS_ATIME",
662 [RANDOM_UMA] = "UMA",
663 [RANDOM_CALLOUT] = "CALLOUT",
664 [RANDOM_RANDOMDEV] = "RANDOMDEV", /* ENVIRONMENTAL_END */
665 [RANDOM_PURE_SAFE] = "PURE_SAFE", /* PURE_START */
666 [RANDOM_PURE_GLXSB] = "PURE_GLXSB",
667 [RANDOM_PURE_RDRAND] = "PURE_RDRAND",
668 [RANDOM_PURE_RDSEED] = "PURE_RDSEED",
669 [RANDOM_PURE_NEHEMIAH] = "PURE_NEHEMIAH",
670 [RANDOM_PURE_RNDTEST] = "PURE_RNDTEST",
671 [RANDOM_PURE_VIRTIO] = "PURE_VIRTIO",
672 [RANDOM_PURE_BROADCOM] = "PURE_BROADCOM",
673 [RANDOM_PURE_CCP] = "PURE_CCP",
674 [RANDOM_PURE_DARN] = "PURE_DARN",
675 [RANDOM_PURE_TPM] = "PURE_TPM",
676 [RANDOM_PURE_VMGENID] = "PURE_VMGENID",
677 [RANDOM_PURE_QUALCOMM] = "PURE_QUALCOMM",
678 [RANDOM_PURE_ARMV8] = "PURE_ARMV8",
679 [RANDOM_PURE_ARM_TRNG] = "PURE_ARM_TRNG",
680 /* "ENTROPYSOURCE" */
681 };
682 CTASSERT(nitems(random_source_descr) == ENTROPYSOURCE);
683
684 static int
random_print_harvestmask_symbolic(SYSCTL_HANDLER_ARGS)685 random_print_harvestmask_symbolic(SYSCTL_HANDLER_ARGS)
686 {
687 struct sbuf sbuf;
688 int error, i;
689 bool first;
690
691 first = true;
692 error = sysctl_wire_old_buffer(req, 0);
693 if (error == 0) {
694 u_int mask;
695
696 sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
697 mask = atomic_load_int(&hc_source_mask);
698 for (i = ENTROPYSOURCE - 1; i >= 0; i--) {
699 bool present;
700
701 present = (mask & (1u << i)) != 0;
702 if (i >= RANDOM_PURE_START && !present)
703 continue;
704 if (!first)
705 sbuf_cat(&sbuf, ",");
706 sbuf_cat(&sbuf, !present ? "[" : "");
707 sbuf_cat(&sbuf, random_source_descr[i]);
708 sbuf_cat(&sbuf, !present ? "]" : "");
709 first = false;
710 }
711 error = sbuf_finish(&sbuf);
712 sbuf_delete(&sbuf);
713 }
714 return (error);
715 }
716 SYSCTL_PROC(_kern_random_harvest, OID_AUTO, mask_symbolic,
717 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
718 random_print_harvestmask_symbolic, "A",
719 "Entropy harvesting mask (symbolic)");
720
721 static void
random_harvestq_init(void * unused __unused)722 random_harvestq_init(void *unused __unused)
723 {
724 static const u_int almost_everything_mask =
725 (((1 << (RANDOM_ENVIRONMENTAL_END + 1)) - 1) &
726 ~_RANDOM_HARVEST_ETHER_OFF & ~_RANDOM_HARVEST_UMA_OFF);
727
728 hc_source_mask = almost_everything_mask;
729 RANDOM_HARVEST_INIT_LOCK();
730 harvest_context.hc_active_buf = 0;
731
732 for (int i = RANDOM_START; i <= RANDOM_ENVIRONMENTAL_END; i++)
733 random_healthtest_init(i, 0);
734 }
735 SYSINIT(random_device_h_init, SI_SUB_RANDOM, SI_ORDER_THIRD, random_harvestq_init, NULL);
736
737 /*
738 * Subroutine to slice up a contiguous chunk of 'entropy' and feed it into the
739 * underlying algorithm. Returns number of bytes actually fed into underlying
740 * algorithm.
741 */
742 static size_t
random_early_prime(char * entropy,size_t len)743 random_early_prime(char *entropy, size_t len)
744 {
745 struct harvest_event event;
746 size_t i;
747
748 len = rounddown(len, sizeof(event.he_entropy));
749 if (len == 0)
750 return (0);
751
752 for (i = 0; i < len; i += sizeof(event.he_entropy)) {
753 event.he_somecounter = random_get_cyclecount();
754 event.he_size = sizeof(event.he_entropy);
755 event.he_source = RANDOM_CACHED;
756 event.he_destination =
757 harvest_context.hc_destination[RANDOM_CACHED]++;
758 memcpy(event.he_entropy, entropy + i, sizeof(event.he_entropy));
759 random_harvestq_fast_process_event(&event);
760 }
761 explicit_bzero(entropy, len);
762 return (len);
763 }
764
765 /*
766 * Subroutine to search for known loader-loaded files in memory and feed them
767 * into the underlying algorithm early in boot. Returns the number of bytes
768 * loaded (zero if none were loaded).
769 */
770 static size_t
random_prime_loader_file(const char * type)771 random_prime_loader_file(const char *type)
772 {
773 uint8_t *keyfile, *data;
774 size_t size;
775
776 keyfile = preload_search_by_type(type);
777 if (keyfile == NULL)
778 return (0);
779
780 data = preload_fetch_addr(keyfile);
781 size = preload_fetch_size(keyfile);
782 if (data == NULL)
783 return (0);
784
785 return (random_early_prime(data, size));
786 }
787
788 /*
789 * This is used to prime the RNG by grabbing any early random stuff
790 * known to the kernel, and inserting it directly into the hashing
791 * module, currently Fortuna.
792 */
793 static void
random_harvestq_prime(void * unused __unused)794 random_harvestq_prime(void *unused __unused)
795 {
796 size_t size;
797
798 /*
799 * Get entropy that may have been preloaded by loader(8)
800 * and use it to pre-charge the entropy harvest queue.
801 */
802 size = random_prime_loader_file(RANDOM_CACHED_BOOT_ENTROPY_MODULE);
803 if (bootverbose) {
804 if (size > 0)
805 printf("random: read %zu bytes from preloaded cache\n",
806 size);
807 else
808 printf("random: no preloaded entropy cache\n");
809 }
810 size = random_prime_loader_file(RANDOM_PLATFORM_BOOT_ENTROPY_MODULE);
811 if (bootverbose) {
812 if (size > 0)
813 printf("random: read %zu bytes from platform bootloader\n",
814 size);
815 else
816 printf("random: no platform bootloader entropy\n");
817 }
818 }
819 SYSINIT(random_device_prime, SI_SUB_RANDOM, SI_ORDER_MIDDLE, random_harvestq_prime, NULL);
820
821 static void
random_harvestq_deinit(void * unused __unused)822 random_harvestq_deinit(void *unused __unused)
823 {
824
825 /* Command the hash/reseed thread to end and wait for it to finish */
826 random_kthread_control = 0;
827 while (random_kthread_control >= 0)
828 tsleep(&harvest_context.hc_kthread_proc, 0, "harvqterm", hz/5);
829 }
830 SYSUNINIT(random_device_h_init, SI_SUB_RANDOM, SI_ORDER_THIRD, random_harvestq_deinit, NULL);
831
832 /*-
833 * Entropy harvesting queue routine.
834 *
835 * This is supposed to be fast; do not do anything slow in here!
836 * It is also illegal (and morally reprehensible) to insert any
837 * high-rate data here. "High-rate" is defined as a data source
838 * that is likely to fill up the buffer in much less than 100ms.
839 * This includes the "always-on" sources like the Intel "rdrand"
840 * or the VIA Nehamiah "xstore" sources.
841 */
842 /* XXXRW: get_cyclecount() is cheap on most modern hardware, where cycle
843 * counters are built in, but on older hardware it will do a real time clock
844 * read which can be quite expensive.
845 */
846 void
random_harvest_queue_(const void * entropy,u_int size,enum random_entropy_source origin)847 random_harvest_queue_(const void *entropy, u_int size, enum random_entropy_source origin)
848 {
849 struct harvest_context *hc;
850 struct entropy_buffer *buf;
851 struct harvest_event *event;
852
853 KASSERT(origin >= RANDOM_START && origin < ENTROPYSOURCE,
854 ("%s: origin %d invalid", __func__, origin));
855
856 hc = &harvest_context;
857 RANDOM_HARVEST_LOCK();
858 buf = &hc->hc_entropy_buf[hc->hc_active_buf];
859 if (buf->pos < RANDOM_RING_MAX) {
860 event = &buf->ring[buf->pos++];
861 event->he_somecounter = random_get_cyclecount();
862 event->he_source = origin;
863 event->he_destination = hc->hc_destination[origin]++;
864 if (size <= sizeof(event->he_entropy)) {
865 event->he_size = size;
866 memcpy(event->he_entropy, entropy, size);
867 } else {
868 /* Big event, so squash it */
869 event->he_size = sizeof(event->he_entropy[0]);
870 event->he_entropy[0] = jenkins_hash(entropy, size, (uint32_t)(uintptr_t)event);
871 }
872 }
873 RANDOM_HARVEST_UNLOCK();
874 }
875
876 /*-
877 * Entropy harvesting fast routine.
878 *
879 * This is supposed to be very fast; do not do anything slow in here!
880 * This is the right place for high-rate harvested data.
881 */
882 void
random_harvest_fast_(const void * entropy,u_int size)883 random_harvest_fast_(const void *entropy, u_int size)
884 {
885 u_int pos;
886
887 pos = harvest_context.hc_entropy_fast_accumulator.pos;
888 harvest_context.hc_entropy_fast_accumulator.buf[pos] ^=
889 jenkins_hash(entropy, size, random_get_cyclecount());
890 harvest_context.hc_entropy_fast_accumulator.pos = (pos + 1)%RANDOM_ACCUM_MAX;
891 }
892
893 /*-
894 * Entropy harvesting direct routine.
895 *
896 * This is not supposed to be fast, but will only be used during
897 * (e.g.) booting when initial entropy is being gathered.
898 */
899 void
random_harvest_direct_(const void * entropy,u_int size,enum random_entropy_source origin)900 random_harvest_direct_(const void *entropy, u_int size, enum random_entropy_source origin)
901 {
902 struct harvest_event event;
903
904 KASSERT(origin >= RANDOM_START && origin < ENTROPYSOURCE, ("%s: origin %d invalid\n", __func__, origin));
905 size = MIN(size, sizeof(event.he_entropy));
906 event.he_somecounter = random_get_cyclecount();
907 event.he_size = size;
908 event.he_source = origin;
909 event.he_destination = harvest_context.hc_destination[origin]++;
910 memcpy(event.he_entropy, entropy, size);
911 random_harvestq_fast_process_event(&event);
912 }
913
914 void
random_source_register(const struct random_source * rsource)915 random_source_register(const struct random_source *rsource)
916 {
917 struct random_sources *rrs;
918
919 KASSERT(rsource != NULL, ("invalid input to %s", __func__));
920
921 rrs = malloc(sizeof(*rrs), M_ENTROPY, M_WAITOK);
922 rrs->rrs_source = rsource;
923
924 printf("random: registering fast source %s\n", rsource->rs_ident);
925
926 random_healthtest_init(rsource->rs_source, rsource->rs_min_entropy);
927
928 RANDOM_HARVEST_LOCK();
929 hc_source_mask |= (1 << rsource->rs_source);
930 CK_LIST_INSERT_HEAD(&source_list, rrs, rrs_entries);
931 RANDOM_HARVEST_UNLOCK();
932 }
933
934 void
random_source_deregister(const struct random_source * rsource)935 random_source_deregister(const struct random_source *rsource)
936 {
937 struct random_sources *rrs = NULL;
938
939 KASSERT(rsource != NULL, ("invalid input to %s", __func__));
940
941 RANDOM_HARVEST_LOCK();
942 hc_source_mask &= ~(1 << rsource->rs_source);
943 CK_LIST_FOREACH(rrs, &source_list, rrs_entries)
944 if (rrs->rrs_source == rsource) {
945 CK_LIST_REMOVE(rrs, rrs_entries);
946 break;
947 }
948 RANDOM_HARVEST_UNLOCK();
949
950 if (rrs != NULL && epoch_inited)
951 epoch_wait_preempt(rs_epoch);
952 free(rrs, M_ENTROPY);
953 }
954
955 static int
random_source_handler(SYSCTL_HANDLER_ARGS)956 random_source_handler(SYSCTL_HANDLER_ARGS)
957 {
958 struct epoch_tracker et;
959 struct random_sources *rrs;
960 struct sbuf sbuf;
961 int error, count;
962
963 error = sysctl_wire_old_buffer(req, 0);
964 if (error != 0)
965 return (error);
966
967 sbuf_new_for_sysctl(&sbuf, NULL, 64, req);
968 count = 0;
969 epoch_enter_preempt(rs_epoch, &et);
970 CK_LIST_FOREACH(rrs, &source_list, rrs_entries) {
971 sbuf_cat(&sbuf, (count++ ? ",'" : "'"));
972 sbuf_cat(&sbuf, rrs->rrs_source->rs_ident);
973 sbuf_cat(&sbuf, "'");
974 }
975 epoch_exit_preempt(rs_epoch, &et);
976 error = sbuf_finish(&sbuf);
977 sbuf_delete(&sbuf);
978 return (error);
979 }
980 SYSCTL_PROC(_kern_random, OID_AUTO, random_sources, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
981 NULL, 0, random_source_handler, "A",
982 "List of active fast entropy sources.");
983
984 MODULE_VERSION(random_harvestq, 1);
985