1 /* 2 * random.c -- A strong random number generator 3 * 4 * Copyright (C) 2017 Jason A. Donenfeld <Jason@zx2c4.com>. All 5 * Rights Reserved. 6 * 7 * Copyright Matt Mackall <mpm@selenic.com>, 2003, 2004, 2005 8 * 9 * Copyright Theodore Ts'o, 1994, 1995, 1996, 1997, 1998, 1999. All 10 * rights reserved. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, and the entire permission notice in its entirety, 17 * including the disclaimer of warranties. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. The name of the author may not be used to endorse or promote 22 * products derived from this software without specific prior 23 * written permission. 24 * 25 * ALTERNATIVELY, this product may be distributed under the terms of 26 * the GNU General Public License, in which case the provisions of the GPL are 27 * required INSTEAD OF the above restrictions. (This clause is 28 * necessary due to a potential bad interaction between the GPL and 29 * the restrictions contained in a BSD-style copyright.) 30 * 31 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED 32 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 33 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF 34 * WHICH ARE HEREBY DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE 35 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 36 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 37 * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 38 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 39 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 41 * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH 42 * DAMAGE. 43 */ 44 45 /* 46 * (now, with legal B.S. out of the way.....) 47 * 48 * This routine gathers environmental noise from device drivers, etc., 49 * and returns good random numbers, suitable for cryptographic use. 50 * Besides the obvious cryptographic uses, these numbers are also good 51 * for seeding TCP sequence numbers, and other places where it is 52 * desirable to have numbers which are not only random, but hard to 53 * predict by an attacker. 54 * 55 * Theory of operation 56 * =================== 57 * 58 * Computers are very predictable devices. Hence it is extremely hard 59 * to produce truly random numbers on a computer --- as opposed to 60 * pseudo-random numbers, which can easily generated by using a 61 * algorithm. Unfortunately, it is very easy for attackers to guess 62 * the sequence of pseudo-random number generators, and for some 63 * applications this is not acceptable. So instead, we must try to 64 * gather "environmental noise" from the computer's environment, which 65 * must be hard for outside attackers to observe, and use that to 66 * generate random numbers. In a Unix environment, this is best done 67 * from inside the kernel. 68 * 69 * Sources of randomness from the environment include inter-keyboard 70 * timings, inter-interrupt timings from some interrupts, and other 71 * events which are both (a) non-deterministic and (b) hard for an 72 * outside observer to measure. Randomness from these sources are 73 * added to an "entropy pool", which is mixed using a CRC-like function. 74 * This is not cryptographically strong, but it is adequate assuming 75 * the randomness is not chosen maliciously, and it is fast enough that 76 * the overhead of doing it on every interrupt is very reasonable. 77 * As random bytes are mixed into the entropy pool, the routines keep 78 * an *estimate* of how many bits of randomness have been stored into 79 * the random number generator's internal state. 80 * 81 * When random bytes are desired, they are obtained by taking the SHA 82 * hash of the contents of the "entropy pool". The SHA hash avoids 83 * exposing the internal state of the entropy pool. It is believed to 84 * be computationally infeasible to derive any useful information 85 * about the input of SHA from its output. Even if it is possible to 86 * analyze SHA in some clever way, as long as the amount of data 87 * returned from the generator is less than the inherent entropy in 88 * the pool, the output data is totally unpredictable. For this 89 * reason, the routine decreases its internal estimate of how many 90 * bits of "true randomness" are contained in the entropy pool as it 91 * outputs random numbers. 92 * 93 * If this estimate goes to zero, the routine can still generate 94 * random numbers; however, an attacker may (at least in theory) be 95 * able to infer the future output of the generator from prior 96 * outputs. This requires successful cryptanalysis of SHA, which is 97 * not believed to be feasible, but there is a remote possibility. 98 * Nonetheless, these numbers should be useful for the vast majority 99 * of purposes. 100 * 101 * Exported interfaces ---- output 102 * =============================== 103 * 104 * There are four exported interfaces; two for use within the kernel, 105 * and two or use from userspace. 106 * 107 * Exported interfaces ---- userspace output 108 * ----------------------------------------- 109 * 110 * The userspace interfaces are two character devices /dev/random and 111 * /dev/urandom. /dev/random is suitable for use when very high 112 * quality randomness is desired (for example, for key generation or 113 * one-time pads), as it will only return a maximum of the number of 114 * bits of randomness (as estimated by the random number generator) 115 * contained in the entropy pool. 116 * 117 * The /dev/urandom device does not have this limit, and will return 118 * as many bytes as are requested. As more and more random bytes are 119 * requested without giving time for the entropy pool to recharge, 120 * this will result in random numbers that are merely cryptographically 121 * strong. For many applications, however, this is acceptable. 122 * 123 * Exported interfaces ---- kernel output 124 * -------------------------------------- 125 * 126 * The primary kernel interface is 127 * 128 * void get_random_bytes(void *buf, int nbytes); 129 * 130 * This interface will return the requested number of random bytes, 131 * and place it in the requested buffer. This is equivalent to a 132 * read from /dev/urandom. 133 * 134 * For less critical applications, there are the functions: 135 * 136 * u32 get_random_u32() 137 * u64 get_random_u64() 138 * unsigned int get_random_int() 139 * unsigned long get_random_long() 140 * 141 * These are produced by a cryptographic RNG seeded from get_random_bytes, 142 * and so do not deplete the entropy pool as much. These are recommended 143 * for most in-kernel operations *if the result is going to be stored in 144 * the kernel*. 145 * 146 * Specifically, the get_random_int() family do not attempt to do 147 * "anti-backtracking". If you capture the state of the kernel (e.g. 148 * by snapshotting the VM), you can figure out previous get_random_int() 149 * return values. But if the value is stored in the kernel anyway, 150 * this is not a problem. 151 * 152 * It *is* safe to expose get_random_int() output to attackers (e.g. as 153 * network cookies); given outputs 1..n, it's not feasible to predict 154 * outputs 0 or n+1. The only concern is an attacker who breaks into 155 * the kernel later; the get_random_int() engine is not reseeded as 156 * often as the get_random_bytes() one. 157 * 158 * get_random_bytes() is needed for keys that need to stay secret after 159 * they are erased from the kernel. For example, any key that will 160 * be wrapped and stored encrypted. And session encryption keys: we'd 161 * like to know that after the session is closed and the keys erased, 162 * the plaintext is unrecoverable to someone who recorded the ciphertext. 163 * 164 * But for network ports/cookies, stack canaries, PRNG seeds, address 165 * space layout randomization, session *authentication* keys, or other 166 * applications where the sensitive data is stored in the kernel in 167 * plaintext for as long as it's sensitive, the get_random_int() family 168 * is just fine. 169 * 170 * Consider ASLR. We want to keep the address space secret from an 171 * outside attacker while the process is running, but once the address 172 * space is torn down, it's of no use to an attacker any more. And it's 173 * stored in kernel data structures as long as it's alive, so worrying 174 * about an attacker's ability to extrapolate it from the get_random_int() 175 * CRNG is silly. 176 * 177 * Even some cryptographic keys are safe to generate with get_random_int(). 178 * In particular, keys for SipHash are generally fine. Here, knowledge 179 * of the key authorizes you to do something to a kernel object (inject 180 * packets to a network connection, or flood a hash table), and the 181 * key is stored with the object being protected. Once it goes away, 182 * we no longer care if anyone knows the key. 183 * 184 * prandom_u32() 185 * ------------- 186 * 187 * For even weaker applications, see the pseudorandom generator 188 * prandom_u32(), prandom_max(), and prandom_bytes(). If the random 189 * numbers aren't security-critical at all, these are *far* cheaper. 190 * Useful for self-tests, random error simulation, randomized backoffs, 191 * and any other application where you trust that nobody is trying to 192 * maliciously mess with you by guessing the "random" numbers. 193 * 194 * Exported interfaces ---- input 195 * ============================== 196 * 197 * The current exported interfaces for gathering environmental noise 198 * from the devices are: 199 * 200 * void add_device_randomness(const void *buf, unsigned int size); 201 * void add_input_randomness(unsigned int type, unsigned int code, 202 * unsigned int value); 203 * void add_interrupt_randomness(int irq, int irq_flags); 204 * void add_disk_randomness(struct gendisk *disk); 205 * 206 * add_device_randomness() is for adding data to the random pool that 207 * is likely to differ between two devices (or possibly even per boot). 208 * This would be things like MAC addresses or serial numbers, or the 209 * read-out of the RTC. This does *not* add any actual entropy to the 210 * pool, but it initializes the pool to different values for devices 211 * that might otherwise be identical and have very little entropy 212 * available to them (particularly common in the embedded world). 213 * 214 * add_input_randomness() uses the input layer interrupt timing, as well as 215 * the event type information from the hardware. 216 * 217 * add_interrupt_randomness() uses the interrupt timing as random 218 * inputs to the entropy pool. Using the cycle counters and the irq source 219 * as inputs, it feeds the randomness roughly once a second. 220 * 221 * add_disk_randomness() uses what amounts to the seek time of block 222 * layer request events, on a per-disk_devt basis, as input to the 223 * entropy pool. Note that high-speed solid state drives with very low 224 * seek times do not make for good sources of entropy, as their seek 225 * times are usually fairly consistent. 226 * 227 * All of these routines try to estimate how many bits of randomness a 228 * particular randomness source. They do this by keeping track of the 229 * first and second order deltas of the event timings. 230 * 231 * Ensuring unpredictability at system startup 232 * ============================================ 233 * 234 * When any operating system starts up, it will go through a sequence 235 * of actions that are fairly predictable by an adversary, especially 236 * if the start-up does not involve interaction with a human operator. 237 * This reduces the actual number of bits of unpredictability in the 238 * entropy pool below the value in entropy_count. In order to 239 * counteract this effect, it helps to carry information in the 240 * entropy pool across shut-downs and start-ups. To do this, put the 241 * following lines an appropriate script which is run during the boot 242 * sequence: 243 * 244 * echo "Initializing random number generator..." 245 * random_seed=/var/run/random-seed 246 * # Carry a random seed from start-up to start-up 247 * # Load and then save the whole entropy pool 248 * if [ -f $random_seed ]; then 249 * cat $random_seed >/dev/urandom 250 * else 251 * touch $random_seed 252 * fi 253 * chmod 600 $random_seed 254 * dd if=/dev/urandom of=$random_seed count=1 bs=512 255 * 256 * and the following lines in an appropriate script which is run as 257 * the system is shutdown: 258 * 259 * # Carry a random seed from shut-down to start-up 260 * # Save the whole entropy pool 261 * echo "Saving random seed..." 262 * random_seed=/var/run/random-seed 263 * touch $random_seed 264 * chmod 600 $random_seed 265 * dd if=/dev/urandom of=$random_seed count=1 bs=512 266 * 267 * For example, on most modern systems using the System V init 268 * scripts, such code fragments would be found in 269 * /etc/rc.d/init.d/random. On older Linux systems, the correct script 270 * location might be in /etc/rcb.d/rc.local or /etc/rc.d/rc.0. 271 * 272 * Effectively, these commands cause the contents of the entropy pool 273 * to be saved at shut-down time and reloaded into the entropy pool at 274 * start-up. (The 'dd' in the addition to the bootup script is to 275 * make sure that /etc/random-seed is different for every start-up, 276 * even if the system crashes without executing rc.0.) Even with 277 * complete knowledge of the start-up activities, predicting the state 278 * of the entropy pool requires knowledge of the previous history of 279 * the system. 280 * 281 * Configuring the /dev/random driver under Linux 282 * ============================================== 283 * 284 * The /dev/random driver under Linux uses minor numbers 8 and 9 of 285 * the /dev/mem major number (#1). So if your system does not have 286 * /dev/random and /dev/urandom created already, they can be created 287 * by using the commands: 288 * 289 * mknod /dev/random c 1 8 290 * mknod /dev/urandom c 1 9 291 * 292 * Acknowledgements: 293 * ================= 294 * 295 * Ideas for constructing this random number generator were derived 296 * from Pretty Good Privacy's random number generator, and from private 297 * discussions with Phil Karn. Colin Plumb provided a faster random 298 * number generator, which speed up the mixing function of the entropy 299 * pool, taken from PGPfone. Dale Worley has also contributed many 300 * useful ideas and suggestions to improve this driver. 301 * 302 * Any flaws in the design are solely my responsibility, and should 303 * not be attributed to the Phil, Colin, or any of authors of PGP. 304 * 305 * Further background information on this topic may be obtained from 306 * RFC 1750, "Randomness Recommendations for Security", by Donald 307 * Eastlake, Steve Crocker, and Jeff Schiller. 308 */ 309 310 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 311 312 #include <linux/utsname.h> 313 #include <linux/module.h> 314 #include <linux/kernel.h> 315 #include <linux/major.h> 316 #include <linux/string.h> 317 #include <linux/fcntl.h> 318 #include <linux/slab.h> 319 #include <linux/random.h> 320 #include <linux/poll.h> 321 #include <linux/init.h> 322 #include <linux/fs.h> 323 #include <linux/genhd.h> 324 #include <linux/interrupt.h> 325 #include <linux/mm.h> 326 #include <linux/nodemask.h> 327 #include <linux/spinlock.h> 328 #include <linux/kthread.h> 329 #include <linux/percpu.h> 330 #include <linux/cryptohash.h> 331 #include <linux/fips.h> 332 #include <linux/ptrace.h> 333 #include <linux/workqueue.h> 334 #include <linux/irq.h> 335 #include <linux/ratelimit.h> 336 #include <linux/syscalls.h> 337 #include <linux/completion.h> 338 #include <linux/uuid.h> 339 #include <crypto/chacha.h> 340 341 #include <asm/processor.h> 342 #include <linux/uaccess.h> 343 #include <asm/irq.h> 344 #include <asm/irq_regs.h> 345 #include <asm/io.h> 346 347 #define CREATE_TRACE_POINTS 348 #include <trace/events/random.h> 349 350 /* #define ADD_INTERRUPT_BENCH */ 351 352 /* 353 * Configuration information 354 */ 355 #define INPUT_POOL_SHIFT 12 356 #define INPUT_POOL_WORDS (1 << (INPUT_POOL_SHIFT-5)) 357 #define OUTPUT_POOL_SHIFT 10 358 #define OUTPUT_POOL_WORDS (1 << (OUTPUT_POOL_SHIFT-5)) 359 #define EXTRACT_SIZE 10 360 361 362 #define LONGS(x) (((x) + sizeof(unsigned long) - 1)/sizeof(unsigned long)) 363 364 /* 365 * To allow fractional bits to be tracked, the entropy_count field is 366 * denominated in units of 1/8th bits. 367 * 368 * 2*(ENTROPY_SHIFT + poolbitshift) must <= 31, or the multiply in 369 * credit_entropy_bits() needs to be 64 bits wide. 370 */ 371 #define ENTROPY_SHIFT 3 372 #define ENTROPY_BITS(r) ((r)->entropy_count >> ENTROPY_SHIFT) 373 374 /* 375 * If the entropy count falls under this number of bits, then we 376 * should wake up processes which are selecting or polling on write 377 * access to /dev/random. 378 */ 379 static int random_write_wakeup_bits = 28 * OUTPUT_POOL_WORDS; 380 381 /* 382 * Originally, we used a primitive polynomial of degree .poolwords 383 * over GF(2). The taps for various sizes are defined below. They 384 * were chosen to be evenly spaced except for the last tap, which is 1 385 * to get the twisting happening as fast as possible. 386 * 387 * For the purposes of better mixing, we use the CRC-32 polynomial as 388 * well to make a (modified) twisted Generalized Feedback Shift 389 * Register. (See M. Matsumoto & Y. Kurita, 1992. Twisted GFSR 390 * generators. ACM Transactions on Modeling and Computer Simulation 391 * 2(3):179-194. Also see M. Matsumoto & Y. Kurita, 1994. Twisted 392 * GFSR generators II. ACM Transactions on Modeling and Computer 393 * Simulation 4:254-266) 394 * 395 * Thanks to Colin Plumb for suggesting this. 396 * 397 * The mixing operation is much less sensitive than the output hash, 398 * where we use SHA-1. All that we want of mixing operation is that 399 * it be a good non-cryptographic hash; i.e. it not produce collisions 400 * when fed "random" data of the sort we expect to see. As long as 401 * the pool state differs for different inputs, we have preserved the 402 * input entropy and done a good job. The fact that an intelligent 403 * attacker can construct inputs that will produce controlled 404 * alterations to the pool's state is not important because we don't 405 * consider such inputs to contribute any randomness. The only 406 * property we need with respect to them is that the attacker can't 407 * increase his/her knowledge of the pool's state. Since all 408 * additions are reversible (knowing the final state and the input, 409 * you can reconstruct the initial state), if an attacker has any 410 * uncertainty about the initial state, he/she can only shuffle that 411 * uncertainty about, but never cause any collisions (which would 412 * decrease the uncertainty). 413 * 414 * Our mixing functions were analyzed by Lacharme, Roeck, Strubel, and 415 * Videau in their paper, "The Linux Pseudorandom Number Generator 416 * Revisited" (see: http://eprint.iacr.org/2012/251.pdf). In their 417 * paper, they point out that we are not using a true Twisted GFSR, 418 * since Matsumoto & Kurita used a trinomial feedback polynomial (that 419 * is, with only three taps, instead of the six that we are using). 420 * As a result, the resulting polynomial is neither primitive nor 421 * irreducible, and hence does not have a maximal period over 422 * GF(2**32). They suggest a slight change to the generator 423 * polynomial which improves the resulting TGFSR polynomial to be 424 * irreducible, which we have made here. 425 */ 426 static const struct poolinfo { 427 int poolbitshift, poolwords, poolbytes, poolfracbits; 428 #define S(x) ilog2(x)+5, (x), (x)*4, (x) << (ENTROPY_SHIFT+5) 429 int tap1, tap2, tap3, tap4, tap5; 430 } poolinfo_table[] = { 431 /* was: x^128 + x^103 + x^76 + x^51 +x^25 + x + 1 */ 432 /* x^128 + x^104 + x^76 + x^51 +x^25 + x + 1 */ 433 { S(128), 104, 76, 51, 25, 1 }, 434 }; 435 436 /* 437 * Static global variables 438 */ 439 static DECLARE_WAIT_QUEUE_HEAD(random_write_wait); 440 static struct fasync_struct *fasync; 441 442 static DEFINE_SPINLOCK(random_ready_list_lock); 443 static LIST_HEAD(random_ready_list); 444 445 struct crng_state { 446 __u32 state[16]; 447 unsigned long init_time; 448 spinlock_t lock; 449 }; 450 451 static struct crng_state primary_crng = { 452 .lock = __SPIN_LOCK_UNLOCKED(primary_crng.lock), 453 }; 454 455 /* 456 * crng_init = 0 --> Uninitialized 457 * 1 --> Initialized 458 * 2 --> Initialized from input_pool 459 * 460 * crng_init is protected by primary_crng->lock, and only increases 461 * its value (from 0->1->2). 462 */ 463 static int crng_init = 0; 464 #define crng_ready() (likely(crng_init > 1)) 465 static int crng_init_cnt = 0; 466 static unsigned long crng_global_init_time = 0; 467 #define CRNG_INIT_CNT_THRESH (2*CHACHA_KEY_SIZE) 468 static void _extract_crng(struct crng_state *crng, __u8 out[CHACHA_BLOCK_SIZE]); 469 static void _crng_backtrack_protect(struct crng_state *crng, 470 __u8 tmp[CHACHA_BLOCK_SIZE], int used); 471 static void process_random_ready_list(void); 472 static void _get_random_bytes(void *buf, int nbytes); 473 474 static struct ratelimit_state unseeded_warning = 475 RATELIMIT_STATE_INIT("warn_unseeded_randomness", HZ, 3); 476 static struct ratelimit_state urandom_warning = 477 RATELIMIT_STATE_INIT("warn_urandom_randomness", HZ, 3); 478 479 static int ratelimit_disable __read_mostly; 480 481 module_param_named(ratelimit_disable, ratelimit_disable, int, 0644); 482 MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression"); 483 484 /********************************************************************** 485 * 486 * OS independent entropy store. Here are the functions which handle 487 * storing entropy in an entropy pool. 488 * 489 **********************************************************************/ 490 491 struct entropy_store; 492 struct entropy_store { 493 /* read-only data: */ 494 const struct poolinfo *poolinfo; 495 __u32 *pool; 496 const char *name; 497 498 /* read-write data: */ 499 spinlock_t lock; 500 unsigned short add_ptr; 501 unsigned short input_rotate; 502 int entropy_count; 503 unsigned int initialized:1; 504 unsigned int last_data_init:1; 505 __u8 last_data[EXTRACT_SIZE]; 506 }; 507 508 static ssize_t extract_entropy(struct entropy_store *r, void *buf, 509 size_t nbytes, int min, int rsvd); 510 static ssize_t _extract_entropy(struct entropy_store *r, void *buf, 511 size_t nbytes, int fips); 512 513 static void crng_reseed(struct crng_state *crng, struct entropy_store *r); 514 static __u32 input_pool_data[INPUT_POOL_WORDS] __latent_entropy; 515 516 static struct entropy_store input_pool = { 517 .poolinfo = &poolinfo_table[0], 518 .name = "input", 519 .lock = __SPIN_LOCK_UNLOCKED(input_pool.lock), 520 .pool = input_pool_data 521 }; 522 523 static __u32 const twist_table[8] = { 524 0x00000000, 0x3b6e20c8, 0x76dc4190, 0x4db26158, 525 0xedb88320, 0xd6d6a3e8, 0x9b64c2b0, 0xa00ae278 }; 526 527 /* 528 * This function adds bytes into the entropy "pool". It does not 529 * update the entropy estimate. The caller should call 530 * credit_entropy_bits if this is appropriate. 531 * 532 * The pool is stirred with a primitive polynomial of the appropriate 533 * degree, and then twisted. We twist by three bits at a time because 534 * it's cheap to do so and helps slightly in the expected case where 535 * the entropy is concentrated in the low-order bits. 536 */ 537 static void _mix_pool_bytes(struct entropy_store *r, const void *in, 538 int nbytes) 539 { 540 unsigned long i, tap1, tap2, tap3, tap4, tap5; 541 int input_rotate; 542 int wordmask = r->poolinfo->poolwords - 1; 543 const char *bytes = in; 544 __u32 w; 545 546 tap1 = r->poolinfo->tap1; 547 tap2 = r->poolinfo->tap2; 548 tap3 = r->poolinfo->tap3; 549 tap4 = r->poolinfo->tap4; 550 tap5 = r->poolinfo->tap5; 551 552 input_rotate = r->input_rotate; 553 i = r->add_ptr; 554 555 /* mix one byte at a time to simplify size handling and churn faster */ 556 while (nbytes--) { 557 w = rol32(*bytes++, input_rotate); 558 i = (i - 1) & wordmask; 559 560 /* XOR in the various taps */ 561 w ^= r->pool[i]; 562 w ^= r->pool[(i + tap1) & wordmask]; 563 w ^= r->pool[(i + tap2) & wordmask]; 564 w ^= r->pool[(i + tap3) & wordmask]; 565 w ^= r->pool[(i + tap4) & wordmask]; 566 w ^= r->pool[(i + tap5) & wordmask]; 567 568 /* Mix the result back in with a twist */ 569 r->pool[i] = (w >> 3) ^ twist_table[w & 7]; 570 571 /* 572 * Normally, we add 7 bits of rotation to the pool. 573 * At the beginning of the pool, add an extra 7 bits 574 * rotation, so that successive passes spread the 575 * input bits across the pool evenly. 576 */ 577 input_rotate = (input_rotate + (i ? 7 : 14)) & 31; 578 } 579 580 r->input_rotate = input_rotate; 581 r->add_ptr = i; 582 } 583 584 static void __mix_pool_bytes(struct entropy_store *r, const void *in, 585 int nbytes) 586 { 587 trace_mix_pool_bytes_nolock(r->name, nbytes, _RET_IP_); 588 _mix_pool_bytes(r, in, nbytes); 589 } 590 591 static void mix_pool_bytes(struct entropy_store *r, const void *in, 592 int nbytes) 593 { 594 unsigned long flags; 595 596 trace_mix_pool_bytes(r->name, nbytes, _RET_IP_); 597 spin_lock_irqsave(&r->lock, flags); 598 _mix_pool_bytes(r, in, nbytes); 599 spin_unlock_irqrestore(&r->lock, flags); 600 } 601 602 struct fast_pool { 603 __u32 pool[4]; 604 unsigned long last; 605 unsigned short reg_idx; 606 unsigned char count; 607 }; 608 609 /* 610 * This is a fast mixing routine used by the interrupt randomness 611 * collector. It's hardcoded for an 128 bit pool and assumes that any 612 * locks that might be needed are taken by the caller. 613 */ 614 static void fast_mix(struct fast_pool *f) 615 { 616 __u32 a = f->pool[0], b = f->pool[1]; 617 __u32 c = f->pool[2], d = f->pool[3]; 618 619 a += b; c += d; 620 b = rol32(b, 6); d = rol32(d, 27); 621 d ^= a; b ^= c; 622 623 a += b; c += d; 624 b = rol32(b, 16); d = rol32(d, 14); 625 d ^= a; b ^= c; 626 627 a += b; c += d; 628 b = rol32(b, 6); d = rol32(d, 27); 629 d ^= a; b ^= c; 630 631 a += b; c += d; 632 b = rol32(b, 16); d = rol32(d, 14); 633 d ^= a; b ^= c; 634 635 f->pool[0] = a; f->pool[1] = b; 636 f->pool[2] = c; f->pool[3] = d; 637 f->count++; 638 } 639 640 static void process_random_ready_list(void) 641 { 642 unsigned long flags; 643 struct random_ready_callback *rdy, *tmp; 644 645 spin_lock_irqsave(&random_ready_list_lock, flags); 646 list_for_each_entry_safe(rdy, tmp, &random_ready_list, list) { 647 struct module *owner = rdy->owner; 648 649 list_del_init(&rdy->list); 650 rdy->func(rdy); 651 module_put(owner); 652 } 653 spin_unlock_irqrestore(&random_ready_list_lock, flags); 654 } 655 656 /* 657 * Credit (or debit) the entropy store with n bits of entropy. 658 * Use credit_entropy_bits_safe() if the value comes from userspace 659 * or otherwise should be checked for extreme values. 660 */ 661 static void credit_entropy_bits(struct entropy_store *r, int nbits) 662 { 663 int entropy_count, orig, has_initialized = 0; 664 const int pool_size = r->poolinfo->poolfracbits; 665 int nfrac = nbits << ENTROPY_SHIFT; 666 667 if (!nbits) 668 return; 669 670 retry: 671 entropy_count = orig = READ_ONCE(r->entropy_count); 672 if (nfrac < 0) { 673 /* Debit */ 674 entropy_count += nfrac; 675 } else { 676 /* 677 * Credit: we have to account for the possibility of 678 * overwriting already present entropy. Even in the 679 * ideal case of pure Shannon entropy, new contributions 680 * approach the full value asymptotically: 681 * 682 * entropy <- entropy + (pool_size - entropy) * 683 * (1 - exp(-add_entropy/pool_size)) 684 * 685 * For add_entropy <= pool_size/2 then 686 * (1 - exp(-add_entropy/pool_size)) >= 687 * (add_entropy/pool_size)*0.7869... 688 * so we can approximate the exponential with 689 * 3/4*add_entropy/pool_size and still be on the 690 * safe side by adding at most pool_size/2 at a time. 691 * 692 * The use of pool_size-2 in the while statement is to 693 * prevent rounding artifacts from making the loop 694 * arbitrarily long; this limits the loop to log2(pool_size)*2 695 * turns no matter how large nbits is. 696 */ 697 int pnfrac = nfrac; 698 const int s = r->poolinfo->poolbitshift + ENTROPY_SHIFT + 2; 699 /* The +2 corresponds to the /4 in the denominator */ 700 701 do { 702 unsigned int anfrac = min(pnfrac, pool_size/2); 703 unsigned int add = 704 ((pool_size - entropy_count)*anfrac*3) >> s; 705 706 entropy_count += add; 707 pnfrac -= anfrac; 708 } while (unlikely(entropy_count < pool_size-2 && pnfrac)); 709 } 710 711 if (WARN_ON(entropy_count < 0)) { 712 pr_warn("negative entropy/overflow: pool %s count %d\n", 713 r->name, entropy_count); 714 entropy_count = 0; 715 } else if (entropy_count > pool_size) 716 entropy_count = pool_size; 717 if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig) 718 goto retry; 719 720 if (has_initialized) { 721 r->initialized = 1; 722 kill_fasync(&fasync, SIGIO, POLL_IN); 723 } 724 725 trace_credit_entropy_bits(r->name, nbits, 726 entropy_count >> ENTROPY_SHIFT, _RET_IP_); 727 728 if (r == &input_pool) { 729 int entropy_bits = entropy_count >> ENTROPY_SHIFT; 730 731 if (crng_init < 2) { 732 if (entropy_bits < 128) 733 return; 734 crng_reseed(&primary_crng, r); 735 entropy_bits = ENTROPY_BITS(r); 736 } 737 } 738 } 739 740 static int credit_entropy_bits_safe(struct entropy_store *r, int nbits) 741 { 742 const int nbits_max = r->poolinfo->poolwords * 32; 743 744 if (nbits < 0) 745 return -EINVAL; 746 747 /* Cap the value to avoid overflows */ 748 nbits = min(nbits, nbits_max); 749 750 credit_entropy_bits(r, nbits); 751 return 0; 752 } 753 754 /********************************************************************* 755 * 756 * CRNG using CHACHA20 757 * 758 *********************************************************************/ 759 760 #define CRNG_RESEED_INTERVAL (300*HZ) 761 762 static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait); 763 764 #ifdef CONFIG_NUMA 765 /* 766 * Hack to deal with crazy userspace progams when they are all trying 767 * to access /dev/urandom in parallel. The programs are almost 768 * certainly doing something terribly wrong, but we'll work around 769 * their brain damage. 770 */ 771 static struct crng_state **crng_node_pool __read_mostly; 772 #endif 773 774 static void invalidate_batched_entropy(void); 775 static void numa_crng_init(void); 776 777 static bool trust_cpu __ro_after_init = IS_ENABLED(CONFIG_RANDOM_TRUST_CPU); 778 static int __init parse_trust_cpu(char *arg) 779 { 780 return kstrtobool(arg, &trust_cpu); 781 } 782 early_param("random.trust_cpu", parse_trust_cpu); 783 784 static bool crng_init_try_arch(struct crng_state *crng) 785 { 786 int i; 787 bool arch_init = true; 788 unsigned long rv; 789 790 for (i = 4; i < 16; i++) { 791 if (!arch_get_random_seed_long(&rv) && 792 !arch_get_random_long(&rv)) { 793 rv = random_get_entropy(); 794 arch_init = false; 795 } 796 crng->state[i] ^= rv; 797 } 798 799 return arch_init; 800 } 801 802 static bool __init crng_init_try_arch_early(struct crng_state *crng) 803 { 804 int i; 805 bool arch_init = true; 806 unsigned long rv; 807 808 for (i = 4; i < 16; i++) { 809 if (!arch_get_random_seed_long_early(&rv) && 810 !arch_get_random_long_early(&rv)) { 811 rv = random_get_entropy(); 812 arch_init = false; 813 } 814 crng->state[i] ^= rv; 815 } 816 817 return arch_init; 818 } 819 820 static void __maybe_unused crng_initialize_secondary(struct crng_state *crng) 821 { 822 memcpy(&crng->state[0], "expand 32-byte k", 16); 823 _get_random_bytes(&crng->state[4], sizeof(__u32) * 12); 824 crng_init_try_arch(crng); 825 crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1; 826 } 827 828 static void __init crng_initialize_primary(struct crng_state *crng) 829 { 830 memcpy(&crng->state[0], "expand 32-byte k", 16); 831 _extract_entropy(&input_pool, &crng->state[4], sizeof(__u32) * 12, 0); 832 if (crng_init_try_arch_early(crng) && trust_cpu) { 833 invalidate_batched_entropy(); 834 numa_crng_init(); 835 crng_init = 2; 836 pr_notice("crng done (trusting CPU's manufacturer)\n"); 837 } 838 crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1; 839 } 840 841 #ifdef CONFIG_NUMA 842 static void do_numa_crng_init(struct work_struct *work) 843 { 844 int i; 845 struct crng_state *crng; 846 struct crng_state **pool; 847 848 pool = kcalloc(nr_node_ids, sizeof(*pool), GFP_KERNEL|__GFP_NOFAIL); 849 for_each_online_node(i) { 850 crng = kmalloc_node(sizeof(struct crng_state), 851 GFP_KERNEL | __GFP_NOFAIL, i); 852 spin_lock_init(&crng->lock); 853 crng_initialize_secondary(crng); 854 pool[i] = crng; 855 } 856 mb(); 857 if (cmpxchg(&crng_node_pool, NULL, pool)) { 858 for_each_node(i) 859 kfree(pool[i]); 860 kfree(pool); 861 } 862 } 863 864 static DECLARE_WORK(numa_crng_init_work, do_numa_crng_init); 865 866 static void numa_crng_init(void) 867 { 868 schedule_work(&numa_crng_init_work); 869 } 870 #else 871 static void numa_crng_init(void) {} 872 #endif 873 874 /* 875 * crng_fast_load() can be called by code in the interrupt service 876 * path. So we can't afford to dilly-dally. 877 */ 878 static int crng_fast_load(const char *cp, size_t len) 879 { 880 unsigned long flags; 881 char *p; 882 883 if (!spin_trylock_irqsave(&primary_crng.lock, flags)) 884 return 0; 885 if (crng_init != 0) { 886 spin_unlock_irqrestore(&primary_crng.lock, flags); 887 return 0; 888 } 889 p = (unsigned char *) &primary_crng.state[4]; 890 while (len > 0 && crng_init_cnt < CRNG_INIT_CNT_THRESH) { 891 p[crng_init_cnt % CHACHA_KEY_SIZE] ^= *cp; 892 cp++; crng_init_cnt++; len--; 893 } 894 spin_unlock_irqrestore(&primary_crng.lock, flags); 895 if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) { 896 invalidate_batched_entropy(); 897 crng_init = 1; 898 pr_notice("fast init done\n"); 899 } 900 return 1; 901 } 902 903 /* 904 * crng_slow_load() is called by add_device_randomness, which has two 905 * attributes. (1) We can't trust the buffer passed to it is 906 * guaranteed to be unpredictable (so it might not have any entropy at 907 * all), and (2) it doesn't have the performance constraints of 908 * crng_fast_load(). 909 * 910 * So we do something more comprehensive which is guaranteed to touch 911 * all of the primary_crng's state, and which uses a LFSR with a 912 * period of 255 as part of the mixing algorithm. Finally, we do 913 * *not* advance crng_init_cnt since buffer we may get may be something 914 * like a fixed DMI table (for example), which might very well be 915 * unique to the machine, but is otherwise unvarying. 916 */ 917 static int crng_slow_load(const char *cp, size_t len) 918 { 919 unsigned long flags; 920 static unsigned char lfsr = 1; 921 unsigned char tmp; 922 unsigned i, max = CHACHA_KEY_SIZE; 923 const char * src_buf = cp; 924 char * dest_buf = (char *) &primary_crng.state[4]; 925 926 if (!spin_trylock_irqsave(&primary_crng.lock, flags)) 927 return 0; 928 if (crng_init != 0) { 929 spin_unlock_irqrestore(&primary_crng.lock, flags); 930 return 0; 931 } 932 if (len > max) 933 max = len; 934 935 for (i = 0; i < max ; i++) { 936 tmp = lfsr; 937 lfsr >>= 1; 938 if (tmp & 1) 939 lfsr ^= 0xE1; 940 tmp = dest_buf[i % CHACHA_KEY_SIZE]; 941 dest_buf[i % CHACHA_KEY_SIZE] ^= src_buf[i % len] ^ lfsr; 942 lfsr += (tmp << 3) | (tmp >> 5); 943 } 944 spin_unlock_irqrestore(&primary_crng.lock, flags); 945 return 1; 946 } 947 948 static void crng_reseed(struct crng_state *crng, struct entropy_store *r) 949 { 950 unsigned long flags; 951 int i, num; 952 union { 953 __u8 block[CHACHA_BLOCK_SIZE]; 954 __u32 key[8]; 955 } buf; 956 957 if (r) { 958 num = extract_entropy(r, &buf, 32, 16, 0); 959 if (num == 0) 960 return; 961 } else { 962 _extract_crng(&primary_crng, buf.block); 963 _crng_backtrack_protect(&primary_crng, buf.block, 964 CHACHA_KEY_SIZE); 965 } 966 spin_lock_irqsave(&crng->lock, flags); 967 for (i = 0; i < 8; i++) { 968 unsigned long rv; 969 if (!arch_get_random_seed_long(&rv) && 970 !arch_get_random_long(&rv)) 971 rv = random_get_entropy(); 972 crng->state[i+4] ^= buf.key[i] ^ rv; 973 } 974 memzero_explicit(&buf, sizeof(buf)); 975 crng->init_time = jiffies; 976 spin_unlock_irqrestore(&crng->lock, flags); 977 if (crng == &primary_crng && crng_init < 2) { 978 invalidate_batched_entropy(); 979 numa_crng_init(); 980 crng_init = 2; 981 process_random_ready_list(); 982 wake_up_interruptible(&crng_init_wait); 983 kill_fasync(&fasync, SIGIO, POLL_IN); 984 pr_notice("crng init done\n"); 985 if (unseeded_warning.missed) { 986 pr_notice("%d get_random_xx warning(s) missed due to ratelimiting\n", 987 unseeded_warning.missed); 988 unseeded_warning.missed = 0; 989 } 990 if (urandom_warning.missed) { 991 pr_notice("%d urandom warning(s) missed due to ratelimiting\n", 992 urandom_warning.missed); 993 urandom_warning.missed = 0; 994 } 995 } 996 } 997 998 static void _extract_crng(struct crng_state *crng, 999 __u8 out[CHACHA_BLOCK_SIZE]) 1000 { 1001 unsigned long v, flags; 1002 1003 if (crng_ready() && 1004 (time_after(crng_global_init_time, crng->init_time) || 1005 time_after(jiffies, crng->init_time + CRNG_RESEED_INTERVAL))) 1006 crng_reseed(crng, crng == &primary_crng ? &input_pool : NULL); 1007 spin_lock_irqsave(&crng->lock, flags); 1008 if (arch_get_random_long(&v)) 1009 crng->state[14] ^= v; 1010 chacha20_block(&crng->state[0], out); 1011 if (crng->state[12] == 0) 1012 crng->state[13]++; 1013 spin_unlock_irqrestore(&crng->lock, flags); 1014 } 1015 1016 static void extract_crng(__u8 out[CHACHA_BLOCK_SIZE]) 1017 { 1018 struct crng_state *crng = NULL; 1019 1020 #ifdef CONFIG_NUMA 1021 if (crng_node_pool) 1022 crng = crng_node_pool[numa_node_id()]; 1023 if (crng == NULL) 1024 #endif 1025 crng = &primary_crng; 1026 _extract_crng(crng, out); 1027 } 1028 1029 /* 1030 * Use the leftover bytes from the CRNG block output (if there is 1031 * enough) to mutate the CRNG key to provide backtracking protection. 1032 */ 1033 static void _crng_backtrack_protect(struct crng_state *crng, 1034 __u8 tmp[CHACHA_BLOCK_SIZE], int used) 1035 { 1036 unsigned long flags; 1037 __u32 *s, *d; 1038 int i; 1039 1040 used = round_up(used, sizeof(__u32)); 1041 if (used + CHACHA_KEY_SIZE > CHACHA_BLOCK_SIZE) { 1042 extract_crng(tmp); 1043 used = 0; 1044 } 1045 spin_lock_irqsave(&crng->lock, flags); 1046 s = (__u32 *) &tmp[used]; 1047 d = &crng->state[4]; 1048 for (i=0; i < 8; i++) 1049 *d++ ^= *s++; 1050 spin_unlock_irqrestore(&crng->lock, flags); 1051 } 1052 1053 static void crng_backtrack_protect(__u8 tmp[CHACHA_BLOCK_SIZE], int used) 1054 { 1055 struct crng_state *crng = NULL; 1056 1057 #ifdef CONFIG_NUMA 1058 if (crng_node_pool) 1059 crng = crng_node_pool[numa_node_id()]; 1060 if (crng == NULL) 1061 #endif 1062 crng = &primary_crng; 1063 _crng_backtrack_protect(crng, tmp, used); 1064 } 1065 1066 static ssize_t extract_crng_user(void __user *buf, size_t nbytes) 1067 { 1068 ssize_t ret = 0, i = CHACHA_BLOCK_SIZE; 1069 __u8 tmp[CHACHA_BLOCK_SIZE] __aligned(4); 1070 int large_request = (nbytes > 256); 1071 1072 while (nbytes) { 1073 if (large_request && need_resched()) { 1074 if (signal_pending(current)) { 1075 if (ret == 0) 1076 ret = -ERESTARTSYS; 1077 break; 1078 } 1079 schedule(); 1080 } 1081 1082 extract_crng(tmp); 1083 i = min_t(int, nbytes, CHACHA_BLOCK_SIZE); 1084 if (copy_to_user(buf, tmp, i)) { 1085 ret = -EFAULT; 1086 break; 1087 } 1088 1089 nbytes -= i; 1090 buf += i; 1091 ret += i; 1092 } 1093 crng_backtrack_protect(tmp, i); 1094 1095 /* Wipe data just written to memory */ 1096 memzero_explicit(tmp, sizeof(tmp)); 1097 1098 return ret; 1099 } 1100 1101 1102 /********************************************************************* 1103 * 1104 * Entropy input management 1105 * 1106 *********************************************************************/ 1107 1108 /* There is one of these per entropy source */ 1109 struct timer_rand_state { 1110 cycles_t last_time; 1111 long last_delta, last_delta2; 1112 }; 1113 1114 #define INIT_TIMER_RAND_STATE { INITIAL_JIFFIES, }; 1115 1116 /* 1117 * Add device- or boot-specific data to the input pool to help 1118 * initialize it. 1119 * 1120 * None of this adds any entropy; it is meant to avoid the problem of 1121 * the entropy pool having similar initial state across largely 1122 * identical devices. 1123 */ 1124 void add_device_randomness(const void *buf, unsigned int size) 1125 { 1126 unsigned long time = random_get_entropy() ^ jiffies; 1127 unsigned long flags; 1128 1129 if (!crng_ready() && size) 1130 crng_slow_load(buf, size); 1131 1132 trace_add_device_randomness(size, _RET_IP_); 1133 spin_lock_irqsave(&input_pool.lock, flags); 1134 _mix_pool_bytes(&input_pool, buf, size); 1135 _mix_pool_bytes(&input_pool, &time, sizeof(time)); 1136 spin_unlock_irqrestore(&input_pool.lock, flags); 1137 } 1138 EXPORT_SYMBOL(add_device_randomness); 1139 1140 static struct timer_rand_state input_timer_state = INIT_TIMER_RAND_STATE; 1141 1142 /* 1143 * This function adds entropy to the entropy "pool" by using timing 1144 * delays. It uses the timer_rand_state structure to make an estimate 1145 * of how many bits of entropy this call has added to the pool. 1146 * 1147 * The number "num" is also added to the pool - it should somehow describe 1148 * the type of event which just happened. This is currently 0-255 for 1149 * keyboard scan codes, and 256 upwards for interrupts. 1150 * 1151 */ 1152 static void add_timer_randomness(struct timer_rand_state *state, unsigned num) 1153 { 1154 struct entropy_store *r; 1155 struct { 1156 long jiffies; 1157 unsigned cycles; 1158 unsigned num; 1159 } sample; 1160 long delta, delta2, delta3; 1161 1162 sample.jiffies = jiffies; 1163 sample.cycles = random_get_entropy(); 1164 sample.num = num; 1165 r = &input_pool; 1166 mix_pool_bytes(r, &sample, sizeof(sample)); 1167 1168 /* 1169 * Calculate number of bits of randomness we probably added. 1170 * We take into account the first, second and third-order deltas 1171 * in order to make our estimate. 1172 */ 1173 delta = sample.jiffies - READ_ONCE(state->last_time); 1174 WRITE_ONCE(state->last_time, sample.jiffies); 1175 1176 delta2 = delta - READ_ONCE(state->last_delta); 1177 WRITE_ONCE(state->last_delta, delta); 1178 1179 delta3 = delta2 - READ_ONCE(state->last_delta2); 1180 WRITE_ONCE(state->last_delta2, delta2); 1181 1182 if (delta < 0) 1183 delta = -delta; 1184 if (delta2 < 0) 1185 delta2 = -delta2; 1186 if (delta3 < 0) 1187 delta3 = -delta3; 1188 if (delta > delta2) 1189 delta = delta2; 1190 if (delta > delta3) 1191 delta = delta3; 1192 1193 /* 1194 * delta is now minimum absolute delta. 1195 * Round down by 1 bit on general principles, 1196 * and limit entropy estimate to 12 bits. 1197 */ 1198 credit_entropy_bits(r, min_t(int, fls(delta>>1), 11)); 1199 } 1200 1201 void add_input_randomness(unsigned int type, unsigned int code, 1202 unsigned int value) 1203 { 1204 static unsigned char last_value; 1205 1206 /* ignore autorepeat and the like */ 1207 if (value == last_value) 1208 return; 1209 1210 last_value = value; 1211 add_timer_randomness(&input_timer_state, 1212 (type << 4) ^ code ^ (code >> 4) ^ value); 1213 trace_add_input_randomness(ENTROPY_BITS(&input_pool)); 1214 } 1215 EXPORT_SYMBOL_GPL(add_input_randomness); 1216 1217 static DEFINE_PER_CPU(struct fast_pool, irq_randomness); 1218 1219 #ifdef ADD_INTERRUPT_BENCH 1220 static unsigned long avg_cycles, avg_deviation; 1221 1222 #define AVG_SHIFT 8 /* Exponential average factor k=1/256 */ 1223 #define FIXED_1_2 (1 << (AVG_SHIFT-1)) 1224 1225 static void add_interrupt_bench(cycles_t start) 1226 { 1227 long delta = random_get_entropy() - start; 1228 1229 /* Use a weighted moving average */ 1230 delta = delta - ((avg_cycles + FIXED_1_2) >> AVG_SHIFT); 1231 avg_cycles += delta; 1232 /* And average deviation */ 1233 delta = abs(delta) - ((avg_deviation + FIXED_1_2) >> AVG_SHIFT); 1234 avg_deviation += delta; 1235 } 1236 #else 1237 #define add_interrupt_bench(x) 1238 #endif 1239 1240 static __u32 get_reg(struct fast_pool *f, struct pt_regs *regs) 1241 { 1242 __u32 *ptr = (__u32 *) regs; 1243 unsigned int idx; 1244 1245 if (regs == NULL) 1246 return 0; 1247 idx = READ_ONCE(f->reg_idx); 1248 if (idx >= sizeof(struct pt_regs) / sizeof(__u32)) 1249 idx = 0; 1250 ptr += idx++; 1251 WRITE_ONCE(f->reg_idx, idx); 1252 return *ptr; 1253 } 1254 1255 void add_interrupt_randomness(int irq, int irq_flags) 1256 { 1257 struct entropy_store *r; 1258 struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness); 1259 struct pt_regs *regs = get_irq_regs(); 1260 unsigned long now = jiffies; 1261 cycles_t cycles = random_get_entropy(); 1262 __u32 c_high, j_high; 1263 __u64 ip; 1264 unsigned long seed; 1265 int credit = 0; 1266 1267 if (cycles == 0) 1268 cycles = get_reg(fast_pool, regs); 1269 c_high = (sizeof(cycles) > 4) ? cycles >> 32 : 0; 1270 j_high = (sizeof(now) > 4) ? now >> 32 : 0; 1271 fast_pool->pool[0] ^= cycles ^ j_high ^ irq; 1272 fast_pool->pool[1] ^= now ^ c_high; 1273 ip = regs ? instruction_pointer(regs) : _RET_IP_; 1274 fast_pool->pool[2] ^= ip; 1275 fast_pool->pool[3] ^= (sizeof(ip) > 4) ? ip >> 32 : 1276 get_reg(fast_pool, regs); 1277 1278 fast_mix(fast_pool); 1279 add_interrupt_bench(cycles); 1280 1281 if (unlikely(crng_init == 0)) { 1282 if ((fast_pool->count >= 64) && 1283 crng_fast_load((char *) fast_pool->pool, 1284 sizeof(fast_pool->pool))) { 1285 fast_pool->count = 0; 1286 fast_pool->last = now; 1287 } 1288 return; 1289 } 1290 1291 if ((fast_pool->count < 64) && 1292 !time_after(now, fast_pool->last + HZ)) 1293 return; 1294 1295 r = &input_pool; 1296 if (!spin_trylock(&r->lock)) 1297 return; 1298 1299 fast_pool->last = now; 1300 __mix_pool_bytes(r, &fast_pool->pool, sizeof(fast_pool->pool)); 1301 1302 /* 1303 * If we have architectural seed generator, produce a seed and 1304 * add it to the pool. For the sake of paranoia don't let the 1305 * architectural seed generator dominate the input from the 1306 * interrupt noise. 1307 */ 1308 if (arch_get_random_seed_long(&seed)) { 1309 __mix_pool_bytes(r, &seed, sizeof(seed)); 1310 credit = 1; 1311 } 1312 spin_unlock(&r->lock); 1313 1314 fast_pool->count = 0; 1315 1316 /* award one bit for the contents of the fast pool */ 1317 credit_entropy_bits(r, credit + 1); 1318 } 1319 EXPORT_SYMBOL_GPL(add_interrupt_randomness); 1320 1321 #ifdef CONFIG_BLOCK 1322 void add_disk_randomness(struct gendisk *disk) 1323 { 1324 if (!disk || !disk->random) 1325 return; 1326 /* first major is 1, so we get >= 0x200 here */ 1327 add_timer_randomness(disk->random, 0x100 + disk_devt(disk)); 1328 trace_add_disk_randomness(disk_devt(disk), ENTROPY_BITS(&input_pool)); 1329 } 1330 EXPORT_SYMBOL_GPL(add_disk_randomness); 1331 #endif 1332 1333 /********************************************************************* 1334 * 1335 * Entropy extraction routines 1336 * 1337 *********************************************************************/ 1338 1339 /* 1340 * This function decides how many bytes to actually take from the 1341 * given pool, and also debits the entropy count accordingly. 1342 */ 1343 static size_t account(struct entropy_store *r, size_t nbytes, int min, 1344 int reserved) 1345 { 1346 int entropy_count, orig, have_bytes; 1347 size_t ibytes, nfrac; 1348 1349 BUG_ON(r->entropy_count > r->poolinfo->poolfracbits); 1350 1351 /* Can we pull enough? */ 1352 retry: 1353 entropy_count = orig = READ_ONCE(r->entropy_count); 1354 ibytes = nbytes; 1355 /* never pull more than available */ 1356 have_bytes = entropy_count >> (ENTROPY_SHIFT + 3); 1357 1358 if ((have_bytes -= reserved) < 0) 1359 have_bytes = 0; 1360 ibytes = min_t(size_t, ibytes, have_bytes); 1361 if (ibytes < min) 1362 ibytes = 0; 1363 1364 if (WARN_ON(entropy_count < 0)) { 1365 pr_warn("negative entropy count: pool %s count %d\n", 1366 r->name, entropy_count); 1367 entropy_count = 0; 1368 } 1369 nfrac = ibytes << (ENTROPY_SHIFT + 3); 1370 if ((size_t) entropy_count > nfrac) 1371 entropy_count -= nfrac; 1372 else 1373 entropy_count = 0; 1374 1375 if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig) 1376 goto retry; 1377 1378 trace_debit_entropy(r->name, 8 * ibytes); 1379 if (ibytes && ENTROPY_BITS(r) < random_write_wakeup_bits) { 1380 wake_up_interruptible(&random_write_wait); 1381 kill_fasync(&fasync, SIGIO, POLL_OUT); 1382 } 1383 1384 return ibytes; 1385 } 1386 1387 /* 1388 * This function does the actual extraction for extract_entropy and 1389 * extract_entropy_user. 1390 * 1391 * Note: we assume that .poolwords is a multiple of 16 words. 1392 */ 1393 static void extract_buf(struct entropy_store *r, __u8 *out) 1394 { 1395 int i; 1396 union { 1397 __u32 w[5]; 1398 unsigned long l[LONGS(20)]; 1399 } hash; 1400 __u32 workspace[SHA_WORKSPACE_WORDS]; 1401 unsigned long flags; 1402 1403 /* 1404 * If we have an architectural hardware random number 1405 * generator, use it for SHA's initial vector 1406 */ 1407 sha_init(hash.w); 1408 for (i = 0; i < LONGS(20); i++) { 1409 unsigned long v; 1410 if (!arch_get_random_long(&v)) 1411 break; 1412 hash.l[i] = v; 1413 } 1414 1415 /* Generate a hash across the pool, 16 words (512 bits) at a time */ 1416 spin_lock_irqsave(&r->lock, flags); 1417 for (i = 0; i < r->poolinfo->poolwords; i += 16) 1418 sha_transform(hash.w, (__u8 *)(r->pool + i), workspace); 1419 1420 /* 1421 * We mix the hash back into the pool to prevent backtracking 1422 * attacks (where the attacker knows the state of the pool 1423 * plus the current outputs, and attempts to find previous 1424 * ouputs), unless the hash function can be inverted. By 1425 * mixing at least a SHA1 worth of hash data back, we make 1426 * brute-forcing the feedback as hard as brute-forcing the 1427 * hash. 1428 */ 1429 __mix_pool_bytes(r, hash.w, sizeof(hash.w)); 1430 spin_unlock_irqrestore(&r->lock, flags); 1431 1432 memzero_explicit(workspace, sizeof(workspace)); 1433 1434 /* 1435 * In case the hash function has some recognizable output 1436 * pattern, we fold it in half. Thus, we always feed back 1437 * twice as much data as we output. 1438 */ 1439 hash.w[0] ^= hash.w[3]; 1440 hash.w[1] ^= hash.w[4]; 1441 hash.w[2] ^= rol32(hash.w[2], 16); 1442 1443 memcpy(out, &hash, EXTRACT_SIZE); 1444 memzero_explicit(&hash, sizeof(hash)); 1445 } 1446 1447 static ssize_t _extract_entropy(struct entropy_store *r, void *buf, 1448 size_t nbytes, int fips) 1449 { 1450 ssize_t ret = 0, i; 1451 __u8 tmp[EXTRACT_SIZE]; 1452 unsigned long flags; 1453 1454 while (nbytes) { 1455 extract_buf(r, tmp); 1456 1457 if (fips) { 1458 spin_lock_irqsave(&r->lock, flags); 1459 if (!memcmp(tmp, r->last_data, EXTRACT_SIZE)) 1460 panic("Hardware RNG duplicated output!\n"); 1461 memcpy(r->last_data, tmp, EXTRACT_SIZE); 1462 spin_unlock_irqrestore(&r->lock, flags); 1463 } 1464 i = min_t(int, nbytes, EXTRACT_SIZE); 1465 memcpy(buf, tmp, i); 1466 nbytes -= i; 1467 buf += i; 1468 ret += i; 1469 } 1470 1471 /* Wipe data just returned from memory */ 1472 memzero_explicit(tmp, sizeof(tmp)); 1473 1474 return ret; 1475 } 1476 1477 /* 1478 * This function extracts randomness from the "entropy pool", and 1479 * returns it in a buffer. 1480 * 1481 * The min parameter specifies the minimum amount we can pull before 1482 * failing to avoid races that defeat catastrophic reseeding while the 1483 * reserved parameter indicates how much entropy we must leave in the 1484 * pool after each pull to avoid starving other readers. 1485 */ 1486 static ssize_t extract_entropy(struct entropy_store *r, void *buf, 1487 size_t nbytes, int min, int reserved) 1488 { 1489 __u8 tmp[EXTRACT_SIZE]; 1490 unsigned long flags; 1491 1492 /* if last_data isn't primed, we need EXTRACT_SIZE extra bytes */ 1493 if (fips_enabled) { 1494 spin_lock_irqsave(&r->lock, flags); 1495 if (!r->last_data_init) { 1496 r->last_data_init = 1; 1497 spin_unlock_irqrestore(&r->lock, flags); 1498 trace_extract_entropy(r->name, EXTRACT_SIZE, 1499 ENTROPY_BITS(r), _RET_IP_); 1500 extract_buf(r, tmp); 1501 spin_lock_irqsave(&r->lock, flags); 1502 memcpy(r->last_data, tmp, EXTRACT_SIZE); 1503 } 1504 spin_unlock_irqrestore(&r->lock, flags); 1505 } 1506 1507 trace_extract_entropy(r->name, nbytes, ENTROPY_BITS(r), _RET_IP_); 1508 nbytes = account(r, nbytes, min, reserved); 1509 1510 return _extract_entropy(r, buf, nbytes, fips_enabled); 1511 } 1512 1513 #define warn_unseeded_randomness(previous) \ 1514 _warn_unseeded_randomness(__func__, (void *) _RET_IP_, (previous)) 1515 1516 static void _warn_unseeded_randomness(const char *func_name, void *caller, 1517 void **previous) 1518 { 1519 #ifdef CONFIG_WARN_ALL_UNSEEDED_RANDOM 1520 const bool print_once = false; 1521 #else 1522 static bool print_once __read_mostly; 1523 #endif 1524 1525 if (print_once || 1526 crng_ready() || 1527 (previous && (caller == READ_ONCE(*previous)))) 1528 return; 1529 WRITE_ONCE(*previous, caller); 1530 #ifndef CONFIG_WARN_ALL_UNSEEDED_RANDOM 1531 print_once = true; 1532 #endif 1533 if (__ratelimit(&unseeded_warning)) 1534 printk_deferred(KERN_NOTICE "random: %s called from %pS " 1535 "with crng_init=%d\n", func_name, caller, 1536 crng_init); 1537 } 1538 1539 /* 1540 * This function is the exported kernel interface. It returns some 1541 * number of good random numbers, suitable for key generation, seeding 1542 * TCP sequence numbers, etc. It does not rely on the hardware random 1543 * number generator. For random bytes direct from the hardware RNG 1544 * (when available), use get_random_bytes_arch(). In order to ensure 1545 * that the randomness provided by this function is okay, the function 1546 * wait_for_random_bytes() should be called and return 0 at least once 1547 * at any point prior. 1548 */ 1549 static void _get_random_bytes(void *buf, int nbytes) 1550 { 1551 __u8 tmp[CHACHA_BLOCK_SIZE] __aligned(4); 1552 1553 trace_get_random_bytes(nbytes, _RET_IP_); 1554 1555 while (nbytes >= CHACHA_BLOCK_SIZE) { 1556 extract_crng(buf); 1557 buf += CHACHA_BLOCK_SIZE; 1558 nbytes -= CHACHA_BLOCK_SIZE; 1559 } 1560 1561 if (nbytes > 0) { 1562 extract_crng(tmp); 1563 memcpy(buf, tmp, nbytes); 1564 crng_backtrack_protect(tmp, nbytes); 1565 } else 1566 crng_backtrack_protect(tmp, CHACHA_BLOCK_SIZE); 1567 memzero_explicit(tmp, sizeof(tmp)); 1568 } 1569 1570 void get_random_bytes(void *buf, int nbytes) 1571 { 1572 static void *previous; 1573 1574 warn_unseeded_randomness(&previous); 1575 _get_random_bytes(buf, nbytes); 1576 } 1577 EXPORT_SYMBOL(get_random_bytes); 1578 1579 1580 /* 1581 * Each time the timer fires, we expect that we got an unpredictable 1582 * jump in the cycle counter. Even if the timer is running on another 1583 * CPU, the timer activity will be touching the stack of the CPU that is 1584 * generating entropy.. 1585 * 1586 * Note that we don't re-arm the timer in the timer itself - we are 1587 * happy to be scheduled away, since that just makes the load more 1588 * complex, but we do not want the timer to keep ticking unless the 1589 * entropy loop is running. 1590 * 1591 * So the re-arming always happens in the entropy loop itself. 1592 */ 1593 static void entropy_timer(struct timer_list *t) 1594 { 1595 credit_entropy_bits(&input_pool, 1); 1596 } 1597 1598 /* 1599 * If we have an actual cycle counter, see if we can 1600 * generate enough entropy with timing noise 1601 */ 1602 static void try_to_generate_entropy(void) 1603 { 1604 struct { 1605 unsigned long now; 1606 struct timer_list timer; 1607 } stack; 1608 1609 stack.now = random_get_entropy(); 1610 1611 /* Slow counter - or none. Don't even bother */ 1612 if (stack.now == random_get_entropy()) 1613 return; 1614 1615 timer_setup_on_stack(&stack.timer, entropy_timer, 0); 1616 while (!crng_ready()) { 1617 if (!timer_pending(&stack.timer)) 1618 mod_timer(&stack.timer, jiffies+1); 1619 mix_pool_bytes(&input_pool, &stack.now, sizeof(stack.now)); 1620 schedule(); 1621 stack.now = random_get_entropy(); 1622 } 1623 1624 del_timer_sync(&stack.timer); 1625 destroy_timer_on_stack(&stack.timer); 1626 mix_pool_bytes(&input_pool, &stack.now, sizeof(stack.now)); 1627 } 1628 1629 /* 1630 * Wait for the urandom pool to be seeded and thus guaranteed to supply 1631 * cryptographically secure random numbers. This applies to: the /dev/urandom 1632 * device, the get_random_bytes function, and the get_random_{u32,u64,int,long} 1633 * family of functions. Using any of these functions without first calling 1634 * this function forfeits the guarantee of security. 1635 * 1636 * Returns: 0 if the urandom pool has been seeded. 1637 * -ERESTARTSYS if the function was interrupted by a signal. 1638 */ 1639 int wait_for_random_bytes(void) 1640 { 1641 if (likely(crng_ready())) 1642 return 0; 1643 1644 do { 1645 int ret; 1646 ret = wait_event_interruptible_timeout(crng_init_wait, crng_ready(), HZ); 1647 if (ret) 1648 return ret > 0 ? 0 : ret; 1649 1650 try_to_generate_entropy(); 1651 } while (!crng_ready()); 1652 1653 return 0; 1654 } 1655 EXPORT_SYMBOL(wait_for_random_bytes); 1656 1657 /* 1658 * Returns whether or not the urandom pool has been seeded and thus guaranteed 1659 * to supply cryptographically secure random numbers. This applies to: the 1660 * /dev/urandom device, the get_random_bytes function, and the get_random_{u32, 1661 * ,u64,int,long} family of functions. 1662 * 1663 * Returns: true if the urandom pool has been seeded. 1664 * false if the urandom pool has not been seeded. 1665 */ 1666 bool rng_is_initialized(void) 1667 { 1668 return crng_ready(); 1669 } 1670 EXPORT_SYMBOL(rng_is_initialized); 1671 1672 /* 1673 * Add a callback function that will be invoked when the nonblocking 1674 * pool is initialised. 1675 * 1676 * returns: 0 if callback is successfully added 1677 * -EALREADY if pool is already initialised (callback not called) 1678 * -ENOENT if module for callback is not alive 1679 */ 1680 int add_random_ready_callback(struct random_ready_callback *rdy) 1681 { 1682 struct module *owner; 1683 unsigned long flags; 1684 int err = -EALREADY; 1685 1686 if (crng_ready()) 1687 return err; 1688 1689 owner = rdy->owner; 1690 if (!try_module_get(owner)) 1691 return -ENOENT; 1692 1693 spin_lock_irqsave(&random_ready_list_lock, flags); 1694 if (crng_ready()) 1695 goto out; 1696 1697 owner = NULL; 1698 1699 list_add(&rdy->list, &random_ready_list); 1700 err = 0; 1701 1702 out: 1703 spin_unlock_irqrestore(&random_ready_list_lock, flags); 1704 1705 module_put(owner); 1706 1707 return err; 1708 } 1709 EXPORT_SYMBOL(add_random_ready_callback); 1710 1711 /* 1712 * Delete a previously registered readiness callback function. 1713 */ 1714 void del_random_ready_callback(struct random_ready_callback *rdy) 1715 { 1716 unsigned long flags; 1717 struct module *owner = NULL; 1718 1719 spin_lock_irqsave(&random_ready_list_lock, flags); 1720 if (!list_empty(&rdy->list)) { 1721 list_del_init(&rdy->list); 1722 owner = rdy->owner; 1723 } 1724 spin_unlock_irqrestore(&random_ready_list_lock, flags); 1725 1726 module_put(owner); 1727 } 1728 EXPORT_SYMBOL(del_random_ready_callback); 1729 1730 /* 1731 * This function will use the architecture-specific hardware random 1732 * number generator if it is available. The arch-specific hw RNG will 1733 * almost certainly be faster than what we can do in software, but it 1734 * is impossible to verify that it is implemented securely (as 1735 * opposed, to, say, the AES encryption of a sequence number using a 1736 * key known by the NSA). So it's useful if we need the speed, but 1737 * only if we're willing to trust the hardware manufacturer not to 1738 * have put in a back door. 1739 * 1740 * Return number of bytes filled in. 1741 */ 1742 int __must_check get_random_bytes_arch(void *buf, int nbytes) 1743 { 1744 int left = nbytes; 1745 char *p = buf; 1746 1747 trace_get_random_bytes_arch(left, _RET_IP_); 1748 while (left) { 1749 unsigned long v; 1750 int chunk = min_t(int, left, sizeof(unsigned long)); 1751 1752 if (!arch_get_random_long(&v)) 1753 break; 1754 1755 memcpy(p, &v, chunk); 1756 p += chunk; 1757 left -= chunk; 1758 } 1759 1760 return nbytes - left; 1761 } 1762 EXPORT_SYMBOL(get_random_bytes_arch); 1763 1764 /* 1765 * init_std_data - initialize pool with system data 1766 * 1767 * @r: pool to initialize 1768 * 1769 * This function clears the pool's entropy count and mixes some system 1770 * data into the pool to prepare it for use. The pool is not cleared 1771 * as that can only decrease the entropy in the pool. 1772 */ 1773 static void __init init_std_data(struct entropy_store *r) 1774 { 1775 int i; 1776 ktime_t now = ktime_get_real(); 1777 unsigned long rv; 1778 1779 mix_pool_bytes(r, &now, sizeof(now)); 1780 for (i = r->poolinfo->poolbytes; i > 0; i -= sizeof(rv)) { 1781 if (!arch_get_random_seed_long(&rv) && 1782 !arch_get_random_long(&rv)) 1783 rv = random_get_entropy(); 1784 mix_pool_bytes(r, &rv, sizeof(rv)); 1785 } 1786 mix_pool_bytes(r, utsname(), sizeof(*(utsname()))); 1787 } 1788 1789 /* 1790 * Note that setup_arch() may call add_device_randomness() 1791 * long before we get here. This allows seeding of the pools 1792 * with some platform dependent data very early in the boot 1793 * process. But it limits our options here. We must use 1794 * statically allocated structures that already have all 1795 * initializations complete at compile time. We should also 1796 * take care not to overwrite the precious per platform data 1797 * we were given. 1798 */ 1799 int __init rand_initialize(void) 1800 { 1801 init_std_data(&input_pool); 1802 crng_initialize_primary(&primary_crng); 1803 crng_global_init_time = jiffies; 1804 if (ratelimit_disable) { 1805 urandom_warning.interval = 0; 1806 unseeded_warning.interval = 0; 1807 } 1808 return 0; 1809 } 1810 1811 #ifdef CONFIG_BLOCK 1812 void rand_initialize_disk(struct gendisk *disk) 1813 { 1814 struct timer_rand_state *state; 1815 1816 /* 1817 * If kzalloc returns null, we just won't use that entropy 1818 * source. 1819 */ 1820 state = kzalloc(sizeof(struct timer_rand_state), GFP_KERNEL); 1821 if (state) { 1822 state->last_time = INITIAL_JIFFIES; 1823 disk->random = state; 1824 } 1825 } 1826 #endif 1827 1828 static ssize_t 1829 urandom_read_nowarn(struct file *file, char __user *buf, size_t nbytes, 1830 loff_t *ppos) 1831 { 1832 int ret; 1833 1834 nbytes = min_t(size_t, nbytes, INT_MAX >> (ENTROPY_SHIFT + 3)); 1835 ret = extract_crng_user(buf, nbytes); 1836 trace_urandom_read(8 * nbytes, 0, ENTROPY_BITS(&input_pool)); 1837 return ret; 1838 } 1839 1840 static ssize_t 1841 urandom_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) 1842 { 1843 unsigned long flags; 1844 static int maxwarn = 10; 1845 1846 if (!crng_ready() && maxwarn > 0) { 1847 maxwarn--; 1848 if (__ratelimit(&urandom_warning)) 1849 pr_notice("%s: uninitialized urandom read (%zd bytes read)\n", 1850 current->comm, nbytes); 1851 spin_lock_irqsave(&primary_crng.lock, flags); 1852 crng_init_cnt = 0; 1853 spin_unlock_irqrestore(&primary_crng.lock, flags); 1854 } 1855 1856 return urandom_read_nowarn(file, buf, nbytes, ppos); 1857 } 1858 1859 static ssize_t 1860 random_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) 1861 { 1862 int ret; 1863 1864 ret = wait_for_random_bytes(); 1865 if (ret != 0) 1866 return ret; 1867 return urandom_read_nowarn(file, buf, nbytes, ppos); 1868 } 1869 1870 static __poll_t 1871 random_poll(struct file *file, poll_table * wait) 1872 { 1873 __poll_t mask; 1874 1875 poll_wait(file, &crng_init_wait, wait); 1876 poll_wait(file, &random_write_wait, wait); 1877 mask = 0; 1878 if (crng_ready()) 1879 mask |= EPOLLIN | EPOLLRDNORM; 1880 if (ENTROPY_BITS(&input_pool) < random_write_wakeup_bits) 1881 mask |= EPOLLOUT | EPOLLWRNORM; 1882 return mask; 1883 } 1884 1885 static int 1886 write_pool(struct entropy_store *r, const char __user *buffer, size_t count) 1887 { 1888 size_t bytes; 1889 __u32 t, buf[16]; 1890 const char __user *p = buffer; 1891 1892 while (count > 0) { 1893 int b, i = 0; 1894 1895 bytes = min(count, sizeof(buf)); 1896 if (copy_from_user(&buf, p, bytes)) 1897 return -EFAULT; 1898 1899 for (b = bytes ; b > 0 ; b -= sizeof(__u32), i++) { 1900 if (!arch_get_random_int(&t)) 1901 break; 1902 buf[i] ^= t; 1903 } 1904 1905 count -= bytes; 1906 p += bytes; 1907 1908 mix_pool_bytes(r, buf, bytes); 1909 cond_resched(); 1910 } 1911 1912 return 0; 1913 } 1914 1915 static ssize_t random_write(struct file *file, const char __user *buffer, 1916 size_t count, loff_t *ppos) 1917 { 1918 size_t ret; 1919 1920 ret = write_pool(&input_pool, buffer, count); 1921 if (ret) 1922 return ret; 1923 1924 return (ssize_t)count; 1925 } 1926 1927 static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg) 1928 { 1929 int size, ent_count; 1930 int __user *p = (int __user *)arg; 1931 int retval; 1932 1933 switch (cmd) { 1934 case RNDGETENTCNT: 1935 /* inherently racy, no point locking */ 1936 ent_count = ENTROPY_BITS(&input_pool); 1937 if (put_user(ent_count, p)) 1938 return -EFAULT; 1939 return 0; 1940 case RNDADDTOENTCNT: 1941 if (!capable(CAP_SYS_ADMIN)) 1942 return -EPERM; 1943 if (get_user(ent_count, p)) 1944 return -EFAULT; 1945 return credit_entropy_bits_safe(&input_pool, ent_count); 1946 case RNDADDENTROPY: 1947 if (!capable(CAP_SYS_ADMIN)) 1948 return -EPERM; 1949 if (get_user(ent_count, p++)) 1950 return -EFAULT; 1951 if (ent_count < 0) 1952 return -EINVAL; 1953 if (get_user(size, p++)) 1954 return -EFAULT; 1955 retval = write_pool(&input_pool, (const char __user *)p, 1956 size); 1957 if (retval < 0) 1958 return retval; 1959 return credit_entropy_bits_safe(&input_pool, ent_count); 1960 case RNDZAPENTCNT: 1961 case RNDCLEARPOOL: 1962 /* 1963 * Clear the entropy pool counters. We no longer clear 1964 * the entropy pool, as that's silly. 1965 */ 1966 if (!capable(CAP_SYS_ADMIN)) 1967 return -EPERM; 1968 input_pool.entropy_count = 0; 1969 return 0; 1970 case RNDRESEEDCRNG: 1971 if (!capable(CAP_SYS_ADMIN)) 1972 return -EPERM; 1973 if (crng_init < 2) 1974 return -ENODATA; 1975 crng_reseed(&primary_crng, NULL); 1976 crng_global_init_time = jiffies - 1; 1977 return 0; 1978 default: 1979 return -EINVAL; 1980 } 1981 } 1982 1983 static int random_fasync(int fd, struct file *filp, int on) 1984 { 1985 return fasync_helper(fd, filp, on, &fasync); 1986 } 1987 1988 const struct file_operations random_fops = { 1989 .read = random_read, 1990 .write = random_write, 1991 .poll = random_poll, 1992 .unlocked_ioctl = random_ioctl, 1993 .compat_ioctl = compat_ptr_ioctl, 1994 .fasync = random_fasync, 1995 .llseek = noop_llseek, 1996 }; 1997 1998 const struct file_operations urandom_fops = { 1999 .read = urandom_read, 2000 .write = random_write, 2001 .unlocked_ioctl = random_ioctl, 2002 .compat_ioctl = compat_ptr_ioctl, 2003 .fasync = random_fasync, 2004 .llseek = noop_llseek, 2005 }; 2006 2007 SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count, 2008 unsigned int, flags) 2009 { 2010 int ret; 2011 2012 if (flags & ~(GRND_NONBLOCK|GRND_RANDOM|GRND_INSECURE)) 2013 return -EINVAL; 2014 2015 /* 2016 * Requesting insecure and blocking randomness at the same time makes 2017 * no sense. 2018 */ 2019 if ((flags & (GRND_INSECURE|GRND_RANDOM)) == (GRND_INSECURE|GRND_RANDOM)) 2020 return -EINVAL; 2021 2022 if (count > INT_MAX) 2023 count = INT_MAX; 2024 2025 if (!(flags & GRND_INSECURE) && !crng_ready()) { 2026 if (flags & GRND_NONBLOCK) 2027 return -EAGAIN; 2028 ret = wait_for_random_bytes(); 2029 if (unlikely(ret)) 2030 return ret; 2031 } 2032 return urandom_read_nowarn(NULL, buf, count, NULL); 2033 } 2034 2035 /******************************************************************** 2036 * 2037 * Sysctl interface 2038 * 2039 ********************************************************************/ 2040 2041 #ifdef CONFIG_SYSCTL 2042 2043 #include <linux/sysctl.h> 2044 2045 static int min_write_thresh; 2046 static int max_write_thresh = INPUT_POOL_WORDS * 32; 2047 static int random_min_urandom_seed = 60; 2048 static char sysctl_bootid[16]; 2049 2050 /* 2051 * This function is used to return both the bootid UUID, and random 2052 * UUID. The difference is in whether table->data is NULL; if it is, 2053 * then a new UUID is generated and returned to the user. 2054 * 2055 * If the user accesses this via the proc interface, the UUID will be 2056 * returned as an ASCII string in the standard UUID format; if via the 2057 * sysctl system call, as 16 bytes of binary data. 2058 */ 2059 static int proc_do_uuid(struct ctl_table *table, int write, 2060 void __user *buffer, size_t *lenp, loff_t *ppos) 2061 { 2062 struct ctl_table fake_table; 2063 unsigned char buf[64], tmp_uuid[16], *uuid; 2064 2065 uuid = table->data; 2066 if (!uuid) { 2067 uuid = tmp_uuid; 2068 generate_random_uuid(uuid); 2069 } else { 2070 static DEFINE_SPINLOCK(bootid_spinlock); 2071 2072 spin_lock(&bootid_spinlock); 2073 if (!uuid[8]) 2074 generate_random_uuid(uuid); 2075 spin_unlock(&bootid_spinlock); 2076 } 2077 2078 sprintf(buf, "%pU", uuid); 2079 2080 fake_table.data = buf; 2081 fake_table.maxlen = sizeof(buf); 2082 2083 return proc_dostring(&fake_table, write, buffer, lenp, ppos); 2084 } 2085 2086 /* 2087 * Return entropy available scaled to integral bits 2088 */ 2089 static int proc_do_entropy(struct ctl_table *table, int write, 2090 void __user *buffer, size_t *lenp, loff_t *ppos) 2091 { 2092 struct ctl_table fake_table; 2093 int entropy_count; 2094 2095 entropy_count = *(int *)table->data >> ENTROPY_SHIFT; 2096 2097 fake_table.data = &entropy_count; 2098 fake_table.maxlen = sizeof(entropy_count); 2099 2100 return proc_dointvec(&fake_table, write, buffer, lenp, ppos); 2101 } 2102 2103 static int sysctl_poolsize = INPUT_POOL_WORDS * 32; 2104 extern struct ctl_table random_table[]; 2105 struct ctl_table random_table[] = { 2106 { 2107 .procname = "poolsize", 2108 .data = &sysctl_poolsize, 2109 .maxlen = sizeof(int), 2110 .mode = 0444, 2111 .proc_handler = proc_dointvec, 2112 }, 2113 { 2114 .procname = "entropy_avail", 2115 .maxlen = sizeof(int), 2116 .mode = 0444, 2117 .proc_handler = proc_do_entropy, 2118 .data = &input_pool.entropy_count, 2119 }, 2120 { 2121 .procname = "write_wakeup_threshold", 2122 .data = &random_write_wakeup_bits, 2123 .maxlen = sizeof(int), 2124 .mode = 0644, 2125 .proc_handler = proc_dointvec_minmax, 2126 .extra1 = &min_write_thresh, 2127 .extra2 = &max_write_thresh, 2128 }, 2129 { 2130 .procname = "urandom_min_reseed_secs", 2131 .data = &random_min_urandom_seed, 2132 .maxlen = sizeof(int), 2133 .mode = 0644, 2134 .proc_handler = proc_dointvec, 2135 }, 2136 { 2137 .procname = "boot_id", 2138 .data = &sysctl_bootid, 2139 .maxlen = 16, 2140 .mode = 0444, 2141 .proc_handler = proc_do_uuid, 2142 }, 2143 { 2144 .procname = "uuid", 2145 .maxlen = 16, 2146 .mode = 0444, 2147 .proc_handler = proc_do_uuid, 2148 }, 2149 #ifdef ADD_INTERRUPT_BENCH 2150 { 2151 .procname = "add_interrupt_avg_cycles", 2152 .data = &avg_cycles, 2153 .maxlen = sizeof(avg_cycles), 2154 .mode = 0444, 2155 .proc_handler = proc_doulongvec_minmax, 2156 }, 2157 { 2158 .procname = "add_interrupt_avg_deviation", 2159 .data = &avg_deviation, 2160 .maxlen = sizeof(avg_deviation), 2161 .mode = 0444, 2162 .proc_handler = proc_doulongvec_minmax, 2163 }, 2164 #endif 2165 { } 2166 }; 2167 #endif /* CONFIG_SYSCTL */ 2168 2169 struct batched_entropy { 2170 union { 2171 u64 entropy_u64[CHACHA_BLOCK_SIZE / sizeof(u64)]; 2172 u32 entropy_u32[CHACHA_BLOCK_SIZE / sizeof(u32)]; 2173 }; 2174 unsigned int position; 2175 spinlock_t batch_lock; 2176 }; 2177 2178 /* 2179 * Get a random word for internal kernel use only. The quality of the random 2180 * number is good as /dev/urandom, but there is no backtrack protection, with 2181 * the goal of being quite fast and not depleting entropy. In order to ensure 2182 * that the randomness provided by this function is okay, the function 2183 * wait_for_random_bytes() should be called and return 0 at least once at any 2184 * point prior. 2185 */ 2186 static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = { 2187 .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u64.lock), 2188 }; 2189 2190 u64 get_random_u64(void) 2191 { 2192 u64 ret; 2193 unsigned long flags; 2194 struct batched_entropy *batch; 2195 static void *previous; 2196 2197 warn_unseeded_randomness(&previous); 2198 2199 batch = raw_cpu_ptr(&batched_entropy_u64); 2200 spin_lock_irqsave(&batch->batch_lock, flags); 2201 if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) { 2202 extract_crng((u8 *)batch->entropy_u64); 2203 batch->position = 0; 2204 } 2205 ret = batch->entropy_u64[batch->position++]; 2206 spin_unlock_irqrestore(&batch->batch_lock, flags); 2207 return ret; 2208 } 2209 EXPORT_SYMBOL(get_random_u64); 2210 2211 static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = { 2212 .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u32.lock), 2213 }; 2214 u32 get_random_u32(void) 2215 { 2216 u32 ret; 2217 unsigned long flags; 2218 struct batched_entropy *batch; 2219 static void *previous; 2220 2221 warn_unseeded_randomness(&previous); 2222 2223 batch = raw_cpu_ptr(&batched_entropy_u32); 2224 spin_lock_irqsave(&batch->batch_lock, flags); 2225 if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) { 2226 extract_crng((u8 *)batch->entropy_u32); 2227 batch->position = 0; 2228 } 2229 ret = batch->entropy_u32[batch->position++]; 2230 spin_unlock_irqrestore(&batch->batch_lock, flags); 2231 return ret; 2232 } 2233 EXPORT_SYMBOL(get_random_u32); 2234 2235 /* It's important to invalidate all potential batched entropy that might 2236 * be stored before the crng is initialized, which we can do lazily by 2237 * simply resetting the counter to zero so that it's re-extracted on the 2238 * next usage. */ 2239 static void invalidate_batched_entropy(void) 2240 { 2241 int cpu; 2242 unsigned long flags; 2243 2244 for_each_possible_cpu (cpu) { 2245 struct batched_entropy *batched_entropy; 2246 2247 batched_entropy = per_cpu_ptr(&batched_entropy_u32, cpu); 2248 spin_lock_irqsave(&batched_entropy->batch_lock, flags); 2249 batched_entropy->position = 0; 2250 spin_unlock(&batched_entropy->batch_lock); 2251 2252 batched_entropy = per_cpu_ptr(&batched_entropy_u64, cpu); 2253 spin_lock(&batched_entropy->batch_lock); 2254 batched_entropy->position = 0; 2255 spin_unlock_irqrestore(&batched_entropy->batch_lock, flags); 2256 } 2257 } 2258 2259 /** 2260 * randomize_page - Generate a random, page aligned address 2261 * @start: The smallest acceptable address the caller will take. 2262 * @range: The size of the area, starting at @start, within which the 2263 * random address must fall. 2264 * 2265 * If @start + @range would overflow, @range is capped. 2266 * 2267 * NOTE: Historical use of randomize_range, which this replaces, presumed that 2268 * @start was already page aligned. We now align it regardless. 2269 * 2270 * Return: A page aligned address within [start, start + range). On error, 2271 * @start is returned. 2272 */ 2273 unsigned long 2274 randomize_page(unsigned long start, unsigned long range) 2275 { 2276 if (!PAGE_ALIGNED(start)) { 2277 range -= PAGE_ALIGN(start) - start; 2278 start = PAGE_ALIGN(start); 2279 } 2280 2281 if (start > ULONG_MAX - range) 2282 range = ULONG_MAX - start; 2283 2284 range >>= PAGE_SHIFT; 2285 2286 if (range == 0) 2287 return start; 2288 2289 return start + (get_random_long() % range << PAGE_SHIFT); 2290 } 2291 2292 /* Interface for in-kernel drivers of true hardware RNGs. 2293 * Those devices may produce endless random bits and will be throttled 2294 * when our pool is full. 2295 */ 2296 void add_hwgenerator_randomness(const char *buffer, size_t count, 2297 size_t entropy) 2298 { 2299 struct entropy_store *poolp = &input_pool; 2300 2301 if (unlikely(crng_init == 0)) { 2302 crng_fast_load(buffer, count); 2303 return; 2304 } 2305 2306 /* Suspend writing if we're above the trickle threshold. 2307 * We'll be woken up again once below random_write_wakeup_thresh, 2308 * or when the calling thread is about to terminate. 2309 */ 2310 wait_event_interruptible(random_write_wait, kthread_should_stop() || 2311 ENTROPY_BITS(&input_pool) <= random_write_wakeup_bits); 2312 mix_pool_bytes(poolp, buffer, count); 2313 credit_entropy_bits(poolp, entropy); 2314 } 2315 EXPORT_SYMBOL_GPL(add_hwgenerator_randomness); 2316 2317 /* Handle random seed passed by bootloader. 2318 * If the seed is trustworthy, it would be regarded as hardware RNGs. Otherwise 2319 * it would be regarded as device data. 2320 * The decision is controlled by CONFIG_RANDOM_TRUST_BOOTLOADER. 2321 */ 2322 void add_bootloader_randomness(const void *buf, unsigned int size) 2323 { 2324 if (IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER)) 2325 add_hwgenerator_randomness(buf, size, size * 8); 2326 else 2327 add_device_randomness(buf, size); 2328 } 2329 EXPORT_SYMBOL_GPL(add_bootloader_randomness); 2330