1 /*-
2 * Copyright (c) 2017 Oliver Pinter
3 * Copyright (c) 2017 W. Dean Freeman
4 * Copyright (c) 2000-2015 Mark R V Murray
5 * Copyright (c) 2013 Arthur Mesh
6 * Copyright (c) 2004 Robert N. M. Watson
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer
14 * in this position and unchanged.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 *
30 */
31
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/ck.h>
35 #include <sys/conf.h>
36 #include <sys/epoch.h>
37 #include <sys/eventhandler.h>
38 #include <sys/hash.h>
39 #include <sys/kernel.h>
40 #include <sys/kthread.h>
41 #include <sys/linker.h>
42 #include <sys/lock.h>
43 #include <sys/malloc.h>
44 #include <sys/module.h>
45 #include <sys/mutex.h>
46 #include <sys/random.h>
47 #include <sys/sbuf.h>
48 #include <sys/sysctl.h>
49 #include <sys/unistd.h>
50
51 #include <machine/atomic.h>
52 #include <machine/cpu.h>
53
54 #include <crypto/rijndael/rijndael-api-fst.h>
55 #include <crypto/sha2/sha256.h>
56
57 #include <dev/random/fortuna.h>
58 #include <dev/random/hash.h>
59 #include <dev/random/randomdev.h>
60 #include <dev/random/random_harvestq.h>
61
62 #if defined(RANDOM_ENABLE_ETHER)
63 #define _RANDOM_HARVEST_ETHER_OFF 0
64 #else
65 #define _RANDOM_HARVEST_ETHER_OFF (1u << RANDOM_NET_ETHER)
66 #endif
67 #if defined(RANDOM_ENABLE_UMA)
68 #define _RANDOM_HARVEST_UMA_OFF 0
69 #else
70 #define _RANDOM_HARVEST_UMA_OFF (1u << RANDOM_UMA)
71 #endif
72
73 /*
74 * Note that random_sources_feed() will also use this to try and split up
75 * entropy into a subset of pools per iteration with the goal of feeding
76 * HARVESTSIZE into every pool at least once per second.
77 */
78 #define RANDOM_KTHREAD_HZ 10
79
80 static void random_kthread(void);
81 static void random_sources_feed(void);
82
83 /*
84 * Random must initialize much earlier than epoch, but we can initialize the
85 * epoch code before SMP starts. Prior to SMP, we can safely bypass
86 * concurrency primitives.
87 */
88 static __read_mostly bool epoch_inited;
89 static __read_mostly epoch_t rs_epoch;
90
91 static const char *random_source_descr[ENTROPYSOURCE];
92
93 /*
94 * How many events to queue up. We create this many items in
95 * an 'empty' queue, then transfer them to the 'harvest' queue with
96 * supplied junk. When used, they are transferred back to the
97 * 'empty' queue.
98 */
99 #define RANDOM_RING_MAX 1024
100 #define RANDOM_ACCUM_MAX 8
101
102 /* 1 to let the kernel thread run, 0 to terminate, -1 to mark completion */
103 volatile int random_kthread_control;
104
105
106 /* Allow the sysadmin to select the broad category of
107 * entropy types to harvest.
108 */
109 __read_frequently u_int hc_source_mask;
110
111 struct random_sources {
112 CK_LIST_ENTRY(random_sources) rrs_entries;
113 const struct random_source *rrs_source;
114 };
115
116 static CK_LIST_HEAD(sources_head, random_sources) source_list =
117 CK_LIST_HEAD_INITIALIZER(source_list);
118
119 SYSCTL_NODE(_kern_random, OID_AUTO, harvest, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
120 "Entropy Device Parameters");
121
122 /*
123 * Put all the harvest queue context stuff in one place.
124 * this make is a bit easier to lock and protect.
125 */
126 static struct harvest_context {
127 /* The harvest mutex protects all of harvest_context and
128 * the related data.
129 */
130 struct mtx hc_mtx;
131 /* Round-robin destination cache. */
132 u_int hc_destination[ENTROPYSOURCE];
133 /* The context of the kernel thread processing harvested entropy */
134 struct proc *hc_kthread_proc;
135 /*
136 * A pair of buffers for queued events. New events are added to the
137 * active queue while the kthread processes the other one in parallel.
138 */
139 struct entropy_buffer {
140 struct harvest_event ring[RANDOM_RING_MAX];
141 u_int pos;
142 } hc_entropy_buf[2];
143 u_int hc_active_buf;
144 struct fast_entropy_accumulator {
145 volatile u_int pos;
146 uint32_t buf[RANDOM_ACCUM_MAX];
147 } hc_entropy_fast_accumulator;
148 } harvest_context;
149
150 #define RANDOM_HARVEST_INIT_LOCK() mtx_init(&harvest_context.hc_mtx, \
151 "entropy harvest mutex", NULL, MTX_SPIN)
152 #define RANDOM_HARVEST_LOCK() mtx_lock_spin(&harvest_context.hc_mtx)
153 #define RANDOM_HARVEST_UNLOCK() mtx_unlock_spin(&harvest_context.hc_mtx)
154
155 static struct kproc_desc random_proc_kp = {
156 "rand_harvestq",
157 random_kthread,
158 &harvest_context.hc_kthread_proc,
159 };
160
161 /* Pass the given event straight through to Fortuna/Whatever. */
162 static __inline void
random_harvestq_fast_process_event(struct harvest_event * event)163 random_harvestq_fast_process_event(struct harvest_event *event)
164 {
165 p_random_alg_context->ra_event_processor(event);
166 explicit_bzero(event, sizeof(*event));
167 }
168
169 static void
random_kthread(void)170 random_kthread(void)
171 {
172 struct harvest_context *hc;
173
174 hc = &harvest_context;
175 for (random_kthread_control = 1; random_kthread_control;) {
176 struct entropy_buffer *buf;
177 u_int entries;
178
179 /* Deal with queued events. */
180 RANDOM_HARVEST_LOCK();
181 buf = &hc->hc_entropy_buf[hc->hc_active_buf];
182 entries = buf->pos;
183 buf->pos = 0;
184 hc->hc_active_buf = (hc->hc_active_buf + 1) %
185 nitems(hc->hc_entropy_buf);
186 RANDOM_HARVEST_UNLOCK();
187 for (u_int i = 0; i < entries; i++)
188 random_harvestq_fast_process_event(&buf->ring[i]);
189
190 /* Poll sources of noise. */
191 random_sources_feed();
192
193 /* XXX: FIX!! Increase the high-performance data rate? Need some measurements first. */
194 for (u_int i = 0; i < RANDOM_ACCUM_MAX; i++) {
195 if (hc->hc_entropy_fast_accumulator.buf[i]) {
196 random_harvest_direct(&hc->hc_entropy_fast_accumulator.buf[i],
197 sizeof(hc->hc_entropy_fast_accumulator.buf[0]), RANDOM_UMA);
198 hc->hc_entropy_fast_accumulator.buf[i] = 0;
199 }
200 }
201 /* XXX: FIX!! This is a *great* place to pass hardware/live entropy to random(9) */
202 tsleep_sbt(&hc->hc_kthread_proc, 0, "-",
203 SBT_1S/RANDOM_KTHREAD_HZ, 0, C_PREL(1));
204 }
205 random_kthread_control = -1;
206 wakeup(&hc->hc_kthread_proc);
207 kproc_exit(0);
208 /* NOTREACHED */
209 }
210 SYSINIT(random_device_h_proc, SI_SUB_KICK_SCHEDULER, SI_ORDER_ANY, kproc_start,
211 &random_proc_kp);
212 _Static_assert(SI_SUB_KICK_SCHEDULER > SI_SUB_RANDOM,
213 "random kthread starting before subsystem initialization");
214
215 static void
rs_epoch_init(void * dummy __unused)216 rs_epoch_init(void *dummy __unused)
217 {
218 rs_epoch = epoch_alloc("Random Sources", EPOCH_PREEMPT);
219 epoch_inited = true;
220 }
221 SYSINIT(rs_epoch_init, SI_SUB_EPOCH, SI_ORDER_ANY, rs_epoch_init, NULL);
222
223 /*
224 * Run through all fast sources reading entropy for the given
225 * number of rounds, which should be a multiple of the number
226 * of entropy accumulation pools in use; it is 32 for Fortuna.
227 */
228 static void
random_sources_feed(void)229 random_sources_feed(void)
230 {
231 uint32_t entropy[HARVESTSIZE];
232 struct epoch_tracker et;
233 struct random_sources *rrs;
234 u_int i, n, npools;
235 bool rse_warm;
236
237 rse_warm = epoch_inited;
238
239 /*
240 * Evenly-ish distribute pool population across the second based on how
241 * frequently random_kthread iterates.
242 *
243 * For Fortuna, the math currently works out as such:
244 *
245 * 64 bits * 4 pools = 256 bits per iteration
246 * 256 bits * 10 Hz = 2560 bits per second, 320 B/s
247 *
248 */
249 npools = howmany(p_random_alg_context->ra_poolcount, RANDOM_KTHREAD_HZ);
250
251 /*-
252 * If we're not seeded yet, attempt to perform a "full seed", filling
253 * all of the PRNG's pools with entropy; if there is enough entropy
254 * available from "fast" entropy sources this will allow us to finish
255 * seeding and unblock the boot process immediately rather than being
256 * stuck for a few seconds with random_kthread gradually collecting a
257 * small chunk of entropy every 1 / RANDOM_KTHREAD_HZ seconds.
258 *
259 * We collect RANDOM_FORTUNA_DEFPOOLSIZE bytes per pool, i.e. enough
260 * to fill Fortuna's pools in the default configuration. With another
261 * PRNG or smaller pools for Fortuna, we might collect more entropy
262 * than needed to fill the pools, but this is harmless; alternatively,
263 * a different PRNG, larger pools, or fast entropy sources which are
264 * not able to provide as much entropy as we request may result in the
265 * not being fully seeded (and thus remaining blocked) but in that
266 * case we will return here after 1 / RANDOM_KTHREAD_HZ seconds and
267 * try again for a large amount of entropy.
268 */
269 if (!p_random_alg_context->ra_seeded())
270 npools = howmany(p_random_alg_context->ra_poolcount *
271 RANDOM_FORTUNA_DEFPOOLSIZE, sizeof(entropy));
272
273 /*
274 * Step over all of live entropy sources, and feed their output
275 * to the system-wide RNG.
276 */
277 if (rse_warm)
278 epoch_enter_preempt(rs_epoch, &et);
279 CK_LIST_FOREACH(rrs, &source_list, rrs_entries) {
280 for (i = 0; i < npools; i++) {
281 n = rrs->rrs_source->rs_read(entropy, sizeof(entropy));
282 KASSERT((n <= sizeof(entropy)), ("%s: rs_read returned too much data (%u > %zu)", __func__, n, sizeof(entropy)));
283 /*
284 * Sometimes the HW entropy source doesn't have anything
285 * ready for us. This isn't necessarily untrustworthy.
286 * We don't perform any other verification of an entropy
287 * source (i.e., length is allowed to be anywhere from 1
288 * to sizeof(entropy), quality is unchecked, etc), so
289 * don't balk verbosely at slow random sources either.
290 * There are reports that RDSEED on x86 metal falls
291 * behind the rate at which we query it, for example.
292 * But it's still a better entropy source than RDRAND.
293 */
294 if (n == 0)
295 continue;
296 random_harvest_direct(entropy, n, rrs->rrs_source->rs_source);
297 }
298 }
299 if (rse_warm)
300 epoch_exit_preempt(rs_epoch, &et);
301 explicit_bzero(entropy, sizeof(entropy));
302 }
303
304 /*
305 * State used for conducting NIST SP 800-90B health tests on entropy sources.
306 */
307 static struct health_test_softc {
308 uint32_t ht_rct_value[HARVESTSIZE + 1];
309 u_int ht_rct_count; /* number of samples with the same value */
310 u_int ht_rct_limit; /* constant after init */
311
312 uint32_t ht_apt_value[HARVESTSIZE + 1];
313 u_int ht_apt_count; /* number of samples with the same value */
314 u_int ht_apt_seq; /* sequence number of the last sample */
315 u_int ht_apt_cutoff; /* constant after init */
316
317 uint64_t ht_total_samples;
318 bool ondemand; /* Set to true to restart the state machine */
319 enum {
320 INIT = 0, /* initial state */
321 DISABLED, /* health checking is disabled */
322 STARTUP, /* doing startup tests, samples are discarded */
323 STEADY, /* steady-state operation */
324 FAILED, /* health check failed, discard samples */
325 } ht_state;
326 } healthtest[ENTROPYSOURCE];
327
328 #define RANDOM_SELFTEST_STARTUP_SAMPLES 1024 /* 4.3, requirement 4 */
329 #define RANDOM_SELFTEST_APT_WINDOW 512 /* 4.4.2 */
330
331 static void
copy_event(uint32_t dst[static HARVESTSIZE+1],const struct harvest_event * event)332 copy_event(uint32_t dst[static HARVESTSIZE + 1],
333 const struct harvest_event *event)
334 {
335 memset(dst, 0, sizeof(uint32_t) * (HARVESTSIZE + 1));
336 memcpy(dst, event->he_entropy, event->he_size);
337 dst[HARVESTSIZE] = event->he_somecounter;
338 }
339
340 static void
random_healthtest_rct_init(struct health_test_softc * ht,const struct harvest_event * event)341 random_healthtest_rct_init(struct health_test_softc *ht,
342 const struct harvest_event *event)
343 {
344 ht->ht_rct_count = 1;
345 copy_event(ht->ht_rct_value, event);
346 }
347
348 /*
349 * Apply the repitition count test to a sample.
350 *
351 * Return false if the test failed, i.e., we observed >= C consecutive samples
352 * with the same value, and true otherwise.
353 */
354 static bool
random_healthtest_rct_next(struct health_test_softc * ht,const struct harvest_event * event)355 random_healthtest_rct_next(struct health_test_softc *ht,
356 const struct harvest_event *event)
357 {
358 uint32_t val[HARVESTSIZE + 1];
359
360 copy_event(val, event);
361 if (memcmp(val, ht->ht_rct_value, sizeof(ht->ht_rct_value)) != 0) {
362 ht->ht_rct_count = 1;
363 memcpy(ht->ht_rct_value, val, sizeof(ht->ht_rct_value));
364 return (true);
365 } else {
366 ht->ht_rct_count++;
367 return (ht->ht_rct_count < ht->ht_rct_limit);
368 }
369 }
370
371 static void
random_healthtest_apt_init(struct health_test_softc * ht,const struct harvest_event * event)372 random_healthtest_apt_init(struct health_test_softc *ht,
373 const struct harvest_event *event)
374 {
375 ht->ht_apt_count = 1;
376 ht->ht_apt_seq = 1;
377 copy_event(ht->ht_apt_value, event);
378 }
379
380 static bool
random_healthtest_apt_next(struct health_test_softc * ht,const struct harvest_event * event)381 random_healthtest_apt_next(struct health_test_softc *ht,
382 const struct harvest_event *event)
383 {
384 uint32_t val[HARVESTSIZE + 1];
385
386 if (ht->ht_apt_seq == 0) {
387 random_healthtest_apt_init(ht, event);
388 return (true);
389 }
390
391 copy_event(val, event);
392 if (memcmp(val, ht->ht_apt_value, sizeof(ht->ht_apt_value)) == 0) {
393 ht->ht_apt_count++;
394 if (ht->ht_apt_count >= ht->ht_apt_cutoff)
395 return (false);
396 }
397
398 ht->ht_apt_seq++;
399 if (ht->ht_apt_seq == RANDOM_SELFTEST_APT_WINDOW)
400 ht->ht_apt_seq = 0;
401
402 return (true);
403 }
404
405 /*
406 * Run the health tests for the given event. This is assumed to be called from
407 * a serialized context.
408 */
409 bool
random_harvest_healthtest(const struct harvest_event * event)410 random_harvest_healthtest(const struct harvest_event *event)
411 {
412 struct health_test_softc *ht;
413
414 ht = &healthtest[event->he_source];
415
416 /*
417 * Was on-demand testing requested? Restart the state machine if so,
418 * restarting the startup tests.
419 */
420 if (atomic_load_bool(&ht->ondemand)) {
421 atomic_store_bool(&ht->ondemand, false);
422 ht->ht_state = INIT;
423 }
424
425 switch (ht->ht_state) {
426 case __predict_false(INIT):
427 /* Store the first sample and initialize test state. */
428 random_healthtest_rct_init(ht, event);
429 random_healthtest_apt_init(ht, event);
430 ht->ht_total_samples = 0;
431 ht->ht_state = STARTUP;
432 return (false);
433 case DISABLED:
434 /* No health testing for this source. */
435 return (true);
436 case STEADY:
437 case STARTUP:
438 ht->ht_total_samples++;
439 if (random_healthtest_rct_next(ht, event) &&
440 random_healthtest_apt_next(ht, event)) {
441 if (ht->ht_state == STARTUP &&
442 ht->ht_total_samples >=
443 RANDOM_SELFTEST_STARTUP_SAMPLES) {
444 printf(
445 "random: health test passed for source %s\n",
446 random_source_descr[event->he_source]);
447 ht->ht_state = STEADY;
448 }
449 return (ht->ht_state == STEADY);
450 }
451 ht->ht_state = FAILED;
452 printf(
453 "random: health test failed for source %s, discarding samples\n",
454 random_source_descr[event->he_source]);
455 /* FALLTHROUGH */
456 case FAILED:
457 return (false);
458 }
459 }
460
461 static bool nist_healthtest_enabled = false;
462 SYSCTL_BOOL(_kern_random, OID_AUTO, nist_healthtest_enabled,
463 CTLFLAG_RDTUN, &nist_healthtest_enabled, 0,
464 "Enable NIST SP 800-90B health tests for noise sources");
465
466 static void
random_healthtest_init(enum random_entropy_source source)467 random_healthtest_init(enum random_entropy_source source)
468 {
469 struct health_test_softc *ht;
470
471 ht = &healthtest[source];
472 KASSERT(ht->ht_state == INIT,
473 ("%s: health test state is %d for source %d",
474 __func__, ht->ht_state, source));
475
476 /*
477 * If health-testing is enabled, validate all sources except CACHED and
478 * VMGENID: they are deterministic sources used only a small, fixed
479 * number of times, so statistical testing is not applicable.
480 */
481 if (!nist_healthtest_enabled ||
482 source == RANDOM_CACHED || source == RANDOM_PURE_VMGENID) {
483 ht->ht_state = DISABLED;
484 return;
485 }
486
487 /*
488 * Set cutoff values for the two tests, assuming that each sample has
489 * min-entropy of 1 bit and allowing for an error rate of 1 in 2^{34}.
490 * With a sample rate of RANDOM_KTHREAD_HZ, we expect to see an false
491 * positive once in ~54.5 years.
492 *
493 * The RCT limit comes from the formula in section 4.4.1.
494 *
495 * The APT cutoff is calculated using the formula in section 4.4.2
496 * footnote 10 with the number of Bernoulli trials changed from W to
497 * W-1, since the test as written counts the number of samples equal to
498 * the first sample in the window, and thus tests W-1 samples.
499 */
500 ht->ht_rct_limit = 35;
501 ht->ht_apt_cutoff = 330;
502 }
503
504 static int
random_healthtest_ondemand(SYSCTL_HANDLER_ARGS)505 random_healthtest_ondemand(SYSCTL_HANDLER_ARGS)
506 {
507 u_int mask, source;
508 int error;
509
510 mask = 0;
511 error = sysctl_handle_int(oidp, &mask, 0, req);
512 if (error != 0 || req->newptr == NULL)
513 return (error);
514
515 while (mask != 0) {
516 source = ffs(mask) - 1;
517 if (source < nitems(healthtest))
518 atomic_store_bool(&healthtest[source].ondemand, true);
519 mask &= ~(1u << source);
520 }
521 return (0);
522 }
523 SYSCTL_PROC(_kern_random, OID_AUTO, nist_healthtest_ondemand,
524 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0,
525 random_healthtest_ondemand, "I",
526 "Re-run NIST SP 800-90B startup health tests for a noise source");
527
528 static int
random_check_uint_harvestmask(SYSCTL_HANDLER_ARGS)529 random_check_uint_harvestmask(SYSCTL_HANDLER_ARGS)
530 {
531 static const u_int user_immutable_mask =
532 (((1 << ENTROPYSOURCE) - 1) & (-1UL << RANDOM_PURE_START)) |
533 _RANDOM_HARVEST_ETHER_OFF | _RANDOM_HARVEST_UMA_OFF;
534
535 int error;
536 u_int value, orig_value;
537
538 orig_value = value = hc_source_mask;
539 error = sysctl_handle_int(oidp, &value, 0, req);
540 if (error != 0 || req->newptr == NULL)
541 return (error);
542
543 if (flsl(value) > ENTROPYSOURCE)
544 return (EINVAL);
545
546 /*
547 * Disallow userspace modification of pure entropy sources.
548 */
549 hc_source_mask = (value & ~user_immutable_mask) |
550 (orig_value & user_immutable_mask);
551 return (0);
552 }
553 SYSCTL_PROC(_kern_random_harvest, OID_AUTO, mask,
554 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, NULL, 0,
555 random_check_uint_harvestmask, "IU",
556 "Entropy harvesting mask");
557
558 static int
random_print_harvestmask(SYSCTL_HANDLER_ARGS)559 random_print_harvestmask(SYSCTL_HANDLER_ARGS)
560 {
561 struct sbuf sbuf;
562 int error, i;
563
564 error = sysctl_wire_old_buffer(req, 0);
565 if (error == 0) {
566 sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
567 for (i = ENTROPYSOURCE - 1; i >= 0; i--)
568 sbuf_cat(&sbuf, (hc_source_mask & (1 << i)) ? "1" : "0");
569 error = sbuf_finish(&sbuf);
570 sbuf_delete(&sbuf);
571 }
572 return (error);
573 }
574 SYSCTL_PROC(_kern_random_harvest, OID_AUTO, mask_bin,
575 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
576 random_print_harvestmask, "A",
577 "Entropy harvesting mask (printable)");
578
579 static const char *random_source_descr[ENTROPYSOURCE] = {
580 [RANDOM_CACHED] = "CACHED",
581 [RANDOM_ATTACH] = "ATTACH",
582 [RANDOM_KEYBOARD] = "KEYBOARD",
583 [RANDOM_MOUSE] = "MOUSE",
584 [RANDOM_NET_TUN] = "NET_TUN",
585 [RANDOM_NET_ETHER] = "NET_ETHER",
586 [RANDOM_NET_NG] = "NET_NG",
587 [RANDOM_INTERRUPT] = "INTERRUPT",
588 [RANDOM_SWI] = "SWI",
589 [RANDOM_FS_ATIME] = "FS_ATIME",
590 [RANDOM_UMA] = "UMA",
591 [RANDOM_CALLOUT] = "CALLOUT",
592 [RANDOM_RANDOMDEV] = "RANDOMDEV", /* ENVIRONMENTAL_END */
593 [RANDOM_PURE_OCTEON] = "PURE_OCTEON", /* PURE_START */
594 [RANDOM_PURE_SAFE] = "PURE_SAFE",
595 [RANDOM_PURE_GLXSB] = "PURE_GLXSB",
596 [RANDOM_PURE_HIFN] = "PURE_HIFN",
597 [RANDOM_PURE_RDRAND] = "PURE_RDRAND",
598 [RANDOM_PURE_NEHEMIAH] = "PURE_NEHEMIAH",
599 [RANDOM_PURE_RNDTEST] = "PURE_RNDTEST",
600 [RANDOM_PURE_VIRTIO] = "PURE_VIRTIO",
601 [RANDOM_PURE_BROADCOM] = "PURE_BROADCOM",
602 [RANDOM_PURE_CCP] = "PURE_CCP",
603 [RANDOM_PURE_DARN] = "PURE_DARN",
604 [RANDOM_PURE_TPM] = "PURE_TPM",
605 [RANDOM_PURE_VMGENID] = "PURE_VMGENID",
606 [RANDOM_PURE_QUALCOMM] = "PURE_QUALCOMM",
607 [RANDOM_PURE_ARMV8] = "PURE_ARMV8",
608 [RANDOM_PURE_ARM_TRNG] = "PURE_ARM_TRNG",
609 /* "ENTROPYSOURCE" */
610 };
611
612 static int
random_print_harvestmask_symbolic(SYSCTL_HANDLER_ARGS)613 random_print_harvestmask_symbolic(SYSCTL_HANDLER_ARGS)
614 {
615 struct sbuf sbuf;
616 int error, i;
617 bool first;
618
619 first = true;
620 error = sysctl_wire_old_buffer(req, 0);
621 if (error == 0) {
622 sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
623 for (i = ENTROPYSOURCE - 1; i >= 0; i--) {
624 if (i >= RANDOM_PURE_START &&
625 (hc_source_mask & (1 << i)) == 0)
626 continue;
627 if (!first)
628 sbuf_cat(&sbuf, ",");
629 sbuf_cat(&sbuf, !(hc_source_mask & (1 << i)) ? "[" : "");
630 sbuf_cat(&sbuf, random_source_descr[i]);
631 sbuf_cat(&sbuf, !(hc_source_mask & (1 << i)) ? "]" : "");
632 first = false;
633 }
634 error = sbuf_finish(&sbuf);
635 sbuf_delete(&sbuf);
636 }
637 return (error);
638 }
639 SYSCTL_PROC(_kern_random_harvest, OID_AUTO, mask_symbolic,
640 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
641 random_print_harvestmask_symbolic, "A",
642 "Entropy harvesting mask (symbolic)");
643
644 static void
random_harvestq_init(void * unused __unused)645 random_harvestq_init(void *unused __unused)
646 {
647 static const u_int almost_everything_mask =
648 (((1 << (RANDOM_ENVIRONMENTAL_END + 1)) - 1) &
649 ~_RANDOM_HARVEST_ETHER_OFF & ~_RANDOM_HARVEST_UMA_OFF);
650
651 hc_source_mask = almost_everything_mask;
652 RANDOM_HARVEST_INIT_LOCK();
653 harvest_context.hc_active_buf = 0;
654
655 for (int i = 0; i < ENTROPYSOURCE; i++)
656 random_healthtest_init(i);
657 }
658 SYSINIT(random_device_h_init, SI_SUB_RANDOM, SI_ORDER_THIRD, random_harvestq_init, NULL);
659
660 /*
661 * Subroutine to slice up a contiguous chunk of 'entropy' and feed it into the
662 * underlying algorithm. Returns number of bytes actually fed into underlying
663 * algorithm.
664 */
665 static size_t
random_early_prime(char * entropy,size_t len)666 random_early_prime(char *entropy, size_t len)
667 {
668 struct harvest_event event;
669 size_t i;
670
671 len = rounddown(len, sizeof(event.he_entropy));
672 if (len == 0)
673 return (0);
674
675 for (i = 0; i < len; i += sizeof(event.he_entropy)) {
676 event.he_somecounter = random_get_cyclecount();
677 event.he_size = sizeof(event.he_entropy);
678 event.he_source = RANDOM_CACHED;
679 event.he_destination =
680 harvest_context.hc_destination[RANDOM_CACHED]++;
681 memcpy(event.he_entropy, entropy + i, sizeof(event.he_entropy));
682 random_harvestq_fast_process_event(&event);
683 }
684 explicit_bzero(entropy, len);
685 return (len);
686 }
687
688 /*
689 * Subroutine to search for known loader-loaded files in memory and feed them
690 * into the underlying algorithm early in boot. Returns the number of bytes
691 * loaded (zero if none were loaded).
692 */
693 static size_t
random_prime_loader_file(const char * type)694 random_prime_loader_file(const char *type)
695 {
696 uint8_t *keyfile, *data;
697 size_t size;
698
699 keyfile = preload_search_by_type(type);
700 if (keyfile == NULL)
701 return (0);
702
703 data = preload_fetch_addr(keyfile);
704 size = preload_fetch_size(keyfile);
705 if (data == NULL)
706 return (0);
707
708 return (random_early_prime(data, size));
709 }
710
711 /*
712 * This is used to prime the RNG by grabbing any early random stuff
713 * known to the kernel, and inserting it directly into the hashing
714 * module, currently Fortuna.
715 */
716 static void
random_harvestq_prime(void * unused __unused)717 random_harvestq_prime(void *unused __unused)
718 {
719 size_t size;
720
721 /*
722 * Get entropy that may have been preloaded by loader(8)
723 * and use it to pre-charge the entropy harvest queue.
724 */
725 size = random_prime_loader_file(RANDOM_CACHED_BOOT_ENTROPY_MODULE);
726 if (bootverbose) {
727 if (size > 0)
728 printf("random: read %zu bytes from preloaded cache\n",
729 size);
730 else
731 printf("random: no preloaded entropy cache\n");
732 }
733 size = random_prime_loader_file(RANDOM_PLATFORM_BOOT_ENTROPY_MODULE);
734 if (bootverbose) {
735 if (size > 0)
736 printf("random: read %zu bytes from platform bootloader\n",
737 size);
738 else
739 printf("random: no platform bootloader entropy\n");
740 }
741 }
742 SYSINIT(random_device_prime, SI_SUB_RANDOM, SI_ORDER_MIDDLE, random_harvestq_prime, NULL);
743
744 static void
random_harvestq_deinit(void * unused __unused)745 random_harvestq_deinit(void *unused __unused)
746 {
747
748 /* Command the hash/reseed thread to end and wait for it to finish */
749 random_kthread_control = 0;
750 while (random_kthread_control >= 0)
751 tsleep(&harvest_context.hc_kthread_proc, 0, "harvqterm", hz/5);
752 }
753 SYSUNINIT(random_device_h_init, SI_SUB_RANDOM, SI_ORDER_THIRD, random_harvestq_deinit, NULL);
754
755 /*-
756 * Entropy harvesting queue routine.
757 *
758 * This is supposed to be fast; do not do anything slow in here!
759 * It is also illegal (and morally reprehensible) to insert any
760 * high-rate data here. "High-rate" is defined as a data source
761 * that is likely to fill up the buffer in much less than 100ms.
762 * This includes the "always-on" sources like the Intel "rdrand"
763 * or the VIA Nehamiah "xstore" sources.
764 */
765 /* XXXRW: get_cyclecount() is cheap on most modern hardware, where cycle
766 * counters are built in, but on older hardware it will do a real time clock
767 * read which can be quite expensive.
768 */
769 void
random_harvest_queue_(const void * entropy,u_int size,enum random_entropy_source origin)770 random_harvest_queue_(const void *entropy, u_int size, enum random_entropy_source origin)
771 {
772 struct harvest_context *hc;
773 struct entropy_buffer *buf;
774 struct harvest_event *event;
775
776 KASSERT(origin >= RANDOM_START && origin < ENTROPYSOURCE,
777 ("%s: origin %d invalid", __func__, origin));
778
779 hc = &harvest_context;
780 RANDOM_HARVEST_LOCK();
781 buf = &hc->hc_entropy_buf[hc->hc_active_buf];
782 if (buf->pos < RANDOM_RING_MAX) {
783 event = &buf->ring[buf->pos++];
784 event->he_somecounter = random_get_cyclecount();
785 event->he_source = origin;
786 event->he_destination = hc->hc_destination[origin]++;
787 if (size <= sizeof(event->he_entropy)) {
788 event->he_size = size;
789 memcpy(event->he_entropy, entropy, size);
790 } else {
791 /* Big event, so squash it */
792 event->he_size = sizeof(event->he_entropy[0]);
793 event->he_entropy[0] = jenkins_hash(entropy, size, (uint32_t)(uintptr_t)event);
794 }
795 }
796 RANDOM_HARVEST_UNLOCK();
797 }
798
799 /*-
800 * Entropy harvesting fast routine.
801 *
802 * This is supposed to be very fast; do not do anything slow in here!
803 * This is the right place for high-rate harvested data.
804 */
805 void
random_harvest_fast_(const void * entropy,u_int size)806 random_harvest_fast_(const void *entropy, u_int size)
807 {
808 u_int pos;
809
810 pos = harvest_context.hc_entropy_fast_accumulator.pos;
811 harvest_context.hc_entropy_fast_accumulator.buf[pos] ^=
812 jenkins_hash(entropy, size, random_get_cyclecount());
813 harvest_context.hc_entropy_fast_accumulator.pos = (pos + 1)%RANDOM_ACCUM_MAX;
814 }
815
816 /*-
817 * Entropy harvesting direct routine.
818 *
819 * This is not supposed to be fast, but will only be used during
820 * (e.g.) booting when initial entropy is being gathered.
821 */
822 void
random_harvest_direct_(const void * entropy,u_int size,enum random_entropy_source origin)823 random_harvest_direct_(const void *entropy, u_int size, enum random_entropy_source origin)
824 {
825 struct harvest_event event;
826
827 KASSERT(origin >= RANDOM_START && origin < ENTROPYSOURCE, ("%s: origin %d invalid\n", __func__, origin));
828 size = MIN(size, sizeof(event.he_entropy));
829 event.he_somecounter = random_get_cyclecount();
830 event.he_size = size;
831 event.he_source = origin;
832 event.he_destination = harvest_context.hc_destination[origin]++;
833 memcpy(event.he_entropy, entropy, size);
834 random_harvestq_fast_process_event(&event);
835 }
836
837 void
random_harvest_register_source(enum random_entropy_source source)838 random_harvest_register_source(enum random_entropy_source source)
839 {
840
841 hc_source_mask |= (1 << source);
842 }
843
844 void
random_harvest_deregister_source(enum random_entropy_source source)845 random_harvest_deregister_source(enum random_entropy_source source)
846 {
847
848 hc_source_mask &= ~(1 << source);
849 }
850
851 void
random_source_register(const struct random_source * rsource)852 random_source_register(const struct random_source *rsource)
853 {
854 struct random_sources *rrs;
855
856 KASSERT(rsource != NULL, ("invalid input to %s", __func__));
857
858 rrs = malloc(sizeof(*rrs), M_ENTROPY, M_WAITOK);
859 rrs->rrs_source = rsource;
860
861 random_harvest_register_source(rsource->rs_source);
862
863 printf("random: registering fast source %s\n", rsource->rs_ident);
864
865 RANDOM_HARVEST_LOCK();
866 CK_LIST_INSERT_HEAD(&source_list, rrs, rrs_entries);
867 RANDOM_HARVEST_UNLOCK();
868 }
869
870 void
random_source_deregister(const struct random_source * rsource)871 random_source_deregister(const struct random_source *rsource)
872 {
873 struct random_sources *rrs = NULL;
874
875 KASSERT(rsource != NULL, ("invalid input to %s", __func__));
876
877 random_harvest_deregister_source(rsource->rs_source);
878
879 RANDOM_HARVEST_LOCK();
880 CK_LIST_FOREACH(rrs, &source_list, rrs_entries)
881 if (rrs->rrs_source == rsource) {
882 CK_LIST_REMOVE(rrs, rrs_entries);
883 break;
884 }
885 RANDOM_HARVEST_UNLOCK();
886
887 if (rrs != NULL && epoch_inited)
888 epoch_wait_preempt(rs_epoch);
889 free(rrs, M_ENTROPY);
890 }
891
892 static int
random_source_handler(SYSCTL_HANDLER_ARGS)893 random_source_handler(SYSCTL_HANDLER_ARGS)
894 {
895 struct epoch_tracker et;
896 struct random_sources *rrs;
897 struct sbuf sbuf;
898 int error, count;
899
900 error = sysctl_wire_old_buffer(req, 0);
901 if (error != 0)
902 return (error);
903
904 sbuf_new_for_sysctl(&sbuf, NULL, 64, req);
905 count = 0;
906 epoch_enter_preempt(rs_epoch, &et);
907 CK_LIST_FOREACH(rrs, &source_list, rrs_entries) {
908 sbuf_cat(&sbuf, (count++ ? ",'" : "'"));
909 sbuf_cat(&sbuf, rrs->rrs_source->rs_ident);
910 sbuf_cat(&sbuf, "'");
911 }
912 epoch_exit_preempt(rs_epoch, &et);
913 error = sbuf_finish(&sbuf);
914 sbuf_delete(&sbuf);
915 return (error);
916 }
917 SYSCTL_PROC(_kern_random, OID_AUTO, random_sources, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
918 NULL, 0, random_source_handler, "A",
919 "List of active fast entropy sources.");
920
921 MODULE_VERSION(random_harvestq, 1);
922