1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241#include <linux/utsname.h>
242#include <linux/module.h>
243#include <linux/kernel.h>
244#include <linux/major.h>
245#include <linux/string.h>
246#include <linux/fcntl.h>
247#include <linux/slab.h>
248#include <linux/random.h>
249#include <linux/poll.h>
250#include <linux/init.h>
251#include <linux/fs.h>
252#include <linux/genhd.h>
253#include <linux/interrupt.h>
254#include <linux/mm.h>
255#include <linux/nodemask.h>
256#include <linux/spinlock.h>
257#include <linux/kthread.h>
258#include <linux/percpu.h>
259#include <linux/cryptohash.h>
260#include <linux/fips.h>
261#include <linux/ptrace.h>
262#include <linux/workqueue.h>
263#include <linux/irq.h>
264#include <linux/ratelimit.h>
265#include <linux/syscalls.h>
266#include <linux/completion.h>
267#include <linux/uuid.h>
268#include <crypto/chacha20.h>
269
270#include <asm/processor.h>
271#include <linux/uaccess.h>
272#include <asm/irq.h>
273#include <asm/irq_regs.h>
274#include <asm/io.h>
275
276#define CREATE_TRACE_POINTS
277#include <trace/events/random.h>
278
279
280
281
282
283
284#define INPUT_POOL_SHIFT 12
285#define INPUT_POOL_WORDS (1 << (INPUT_POOL_SHIFT-5))
286#define OUTPUT_POOL_SHIFT 10
287#define OUTPUT_POOL_WORDS (1 << (OUTPUT_POOL_SHIFT-5))
288#define SEC_XFER_SIZE 512
289#define EXTRACT_SIZE 10
290
291
292#define LONGS(x) (((x) + sizeof(unsigned long) - 1)/sizeof(unsigned long))
293
294
295
296
297
298
299
300
301#define ENTROPY_SHIFT 3
302#define ENTROPY_BITS(r) ((r)->entropy_count >> ENTROPY_SHIFT)
303
304
305
306
307
308static int random_read_wakeup_bits = 64;
309
310
311
312
313
314
315static int random_write_wakeup_bits = 28 * OUTPUT_POOL_WORDS;
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362static struct poolinfo {
363 int poolbitshift, poolwords, poolbytes, poolbits, poolfracbits;
364#define S(x) ilog2(x)+5, (x), (x)*4, (x)*32, (x) << (ENTROPY_SHIFT+5)
365 int tap1, tap2, tap3, tap4, tap5;
366} poolinfo_table[] = {
367
368
369 { S(128), 104, 76, 51, 25, 1 },
370
371
372 { S(32), 26, 19, 14, 7, 1 },
373#if 0
374
375 { S(2048), 1638, 1231, 819, 411, 1 },
376
377
378 { S(1024), 817, 615, 412, 204, 1 },
379
380
381 { S(1024), 819, 616, 410, 207, 2 },
382
383
384 { S(512), 411, 308, 208, 104, 1 },
385
386
387 { S(512), 409, 307, 206, 102, 2 },
388
389 { S(512), 409, 309, 205, 103, 2 },
390
391
392 { S(256), 205, 155, 101, 52, 1 },
393
394
395 { S(128), 103, 78, 51, 27, 2 },
396
397
398 { S(64), 52, 39, 26, 14, 1 },
399#endif
400};
401
402
403
404
405static DECLARE_WAIT_QUEUE_HEAD(random_read_wait);
406static DECLARE_WAIT_QUEUE_HEAD(random_write_wait);
407static struct fasync_struct *fasync;
408
409static DEFINE_SPINLOCK(random_ready_list_lock);
410static LIST_HEAD(random_ready_list);
411
412struct crng_state {
413 __u32 state[16];
414 unsigned long init_time;
415 spinlock_t lock;
416};
417
418struct crng_state primary_crng = {
419 .lock = __SPIN_LOCK_UNLOCKED(primary_crng.lock),
420};
421
422
423
424
425
426
427
428
429
430static int crng_init = 0;
431#define crng_ready() (likely(crng_init > 1))
432static int crng_init_cnt = 0;
433static unsigned long crng_global_init_time = 0;
434#define CRNG_INIT_CNT_THRESH (2*CHACHA20_KEY_SIZE)
435static void _extract_crng(struct crng_state *crng,
436 __u32 out[CHACHA20_BLOCK_WORDS]);
437static void _crng_backtrack_protect(struct crng_state *crng,
438 __u32 tmp[CHACHA20_BLOCK_WORDS], int used);
439static void process_random_ready_list(void);
440static void _get_random_bytes(void *buf, int nbytes);
441
442static struct ratelimit_state unseeded_warning =
443 RATELIMIT_STATE_INIT("warn_unseeded_randomness", HZ, 3);
444static struct ratelimit_state urandom_warning =
445 RATELIMIT_STATE_INIT("warn_urandom_randomness", HZ, 3);
446
447static int ratelimit_disable __read_mostly;
448
449module_param_named(ratelimit_disable, ratelimit_disable, int, 0644);
450MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression");
451
452
453
454
455
456
457
458
459struct entropy_store;
460struct entropy_store {
461
462 const struct poolinfo *poolinfo;
463 __u32 *pool;
464 const char *name;
465 struct entropy_store *pull;
466 struct work_struct push_work;
467
468
469 unsigned long last_pulled;
470 spinlock_t lock;
471 unsigned short add_ptr;
472 unsigned short input_rotate;
473 int entropy_count;
474 int entropy_total;
475 unsigned int initialized:1;
476 unsigned int last_data_init:1;
477 __u8 last_data[EXTRACT_SIZE];
478};
479
480static ssize_t extract_entropy(struct entropy_store *r, void *buf,
481 size_t nbytes, int min, int rsvd);
482static ssize_t _extract_entropy(struct entropy_store *r, void *buf,
483 size_t nbytes, int fips);
484
485static void crng_reseed(struct crng_state *crng, struct entropy_store *r);
486static void push_to_pool(struct work_struct *work);
487static __u32 input_pool_data[INPUT_POOL_WORDS] __latent_entropy;
488static __u32 blocking_pool_data[OUTPUT_POOL_WORDS] __latent_entropy;
489
490static struct entropy_store input_pool = {
491 .poolinfo = &poolinfo_table[0],
492 .name = "input",
493 .lock = __SPIN_LOCK_UNLOCKED(input_pool.lock),
494 .pool = input_pool_data
495};
496
497static struct entropy_store blocking_pool = {
498 .poolinfo = &poolinfo_table[1],
499 .name = "blocking",
500 .pull = &input_pool,
501 .lock = __SPIN_LOCK_UNLOCKED(blocking_pool.lock),
502 .pool = blocking_pool_data,
503 .push_work = __WORK_INITIALIZER(blocking_pool.push_work,
504 push_to_pool),
505};
506
507static __u32 const twist_table[8] = {
508 0x00000000, 0x3b6e20c8, 0x76dc4190, 0x4db26158,
509 0xedb88320, 0xd6d6a3e8, 0x9b64c2b0, 0xa00ae278 };
510
511
512
513
514
515
516
517
518
519
520
521static void _mix_pool_bytes(struct entropy_store *r, const void *in,
522 int nbytes)
523{
524 unsigned long i, tap1, tap2, tap3, tap4, tap5;
525 int input_rotate;
526 int wordmask = r->poolinfo->poolwords - 1;
527 const char *bytes = in;
528 __u32 w;
529
530 tap1 = r->poolinfo->tap1;
531 tap2 = r->poolinfo->tap2;
532 tap3 = r->poolinfo->tap3;
533 tap4 = r->poolinfo->tap4;
534 tap5 = r->poolinfo->tap5;
535
536 input_rotate = r->input_rotate;
537 i = r->add_ptr;
538
539
540 while (nbytes--) {
541 w = rol32(*bytes++, input_rotate);
542 i = (i - 1) & wordmask;
543
544
545 w ^= r->pool[i];
546 w ^= r->pool[(i + tap1) & wordmask];
547 w ^= r->pool[(i + tap2) & wordmask];
548 w ^= r->pool[(i + tap3) & wordmask];
549 w ^= r->pool[(i + tap4) & wordmask];
550 w ^= r->pool[(i + tap5) & wordmask];
551
552
553 r->pool[i] = (w >> 3) ^ twist_table[w & 7];
554
555
556
557
558
559
560
561 input_rotate = (input_rotate + (i ? 7 : 14)) & 31;
562 }
563
564 r->input_rotate = input_rotate;
565 r->add_ptr = i;
566}
567
568static void __mix_pool_bytes(struct entropy_store *r, const void *in,
569 int nbytes)
570{
571 trace_mix_pool_bytes_nolock(r->name, nbytes, _RET_IP_);
572 _mix_pool_bytes(r, in, nbytes);
573}
574
575static void mix_pool_bytes(struct entropy_store *r, const void *in,
576 int nbytes)
577{
578 unsigned long flags;
579
580 trace_mix_pool_bytes(r->name, nbytes, _RET_IP_);
581 spin_lock_irqsave(&r->lock, flags);
582 _mix_pool_bytes(r, in, nbytes);
583 spin_unlock_irqrestore(&r->lock, flags);
584}
585
586struct fast_pool {
587 __u32 pool[4];
588 unsigned long last;
589 unsigned short reg_idx;
590 unsigned char count;
591};
592
593
594
595
596
597
598static void fast_mix(struct fast_pool *f)
599{
600 __u32 a = f->pool[0], b = f->pool[1];
601 __u32 c = f->pool[2], d = f->pool[3];
602
603 a += b; c += d;
604 b = rol32(b, 6); d = rol32(d, 27);
605 d ^= a; b ^= c;
606
607 a += b; c += d;
608 b = rol32(b, 16); d = rol32(d, 14);
609 d ^= a; b ^= c;
610
611 a += b; c += d;
612 b = rol32(b, 6); d = rol32(d, 27);
613 d ^= a; b ^= c;
614
615 a += b; c += d;
616 b = rol32(b, 16); d = rol32(d, 14);
617 d ^= a; b ^= c;
618
619 f->pool[0] = a; f->pool[1] = b;
620 f->pool[2] = c; f->pool[3] = d;
621 f->count++;
622}
623
624static void process_random_ready_list(void)
625{
626 unsigned long flags;
627 struct random_ready_callback *rdy, *tmp;
628
629 spin_lock_irqsave(&random_ready_list_lock, flags);
630 list_for_each_entry_safe(rdy, tmp, &random_ready_list, list) {
631 struct module *owner = rdy->owner;
632
633 list_del_init(&rdy->list);
634 rdy->func(rdy);
635 module_put(owner);
636 }
637 spin_unlock_irqrestore(&random_ready_list_lock, flags);
638}
639
640
641
642
643
644
645static void credit_entropy_bits(struct entropy_store *r, int nbits)
646{
647 int entropy_count, orig;
648 const int pool_size = r->poolinfo->poolfracbits;
649 int nfrac = nbits << ENTROPY_SHIFT;
650
651 if (!nbits)
652 return;
653
654retry:
655 entropy_count = orig = READ_ONCE(r->entropy_count);
656 if (nfrac < 0) {
657
658 entropy_count += nfrac;
659 } else {
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681 int pnfrac = nfrac;
682 const int s = r->poolinfo->poolbitshift + ENTROPY_SHIFT + 2;
683
684
685 do {
686 unsigned int anfrac = min(pnfrac, pool_size/2);
687 unsigned int add =
688 ((pool_size - entropy_count)*anfrac*3) >> s;
689
690 entropy_count += add;
691 pnfrac -= anfrac;
692 } while (unlikely(entropy_count < pool_size-2 && pnfrac));
693 }
694
695 if (unlikely(entropy_count < 0)) {
696 pr_warn("random: negative entropy/overflow: pool %s count %d\n",
697 r->name, entropy_count);
698 WARN_ON(1);
699 entropy_count = 0;
700 } else if (entropy_count > pool_size)
701 entropy_count = pool_size;
702 if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig)
703 goto retry;
704
705 r->entropy_total += nbits;
706 if (!r->initialized && r->entropy_total > 128) {
707 r->initialized = 1;
708 r->entropy_total = 0;
709 }
710
711 trace_credit_entropy_bits(r->name, nbits,
712 entropy_count >> ENTROPY_SHIFT,
713 r->entropy_total, _RET_IP_);
714
715 if (r == &input_pool) {
716 int entropy_bits = entropy_count >> ENTROPY_SHIFT;
717
718 if (crng_init < 2 && entropy_bits >= 128) {
719 crng_reseed(&primary_crng, r);
720 entropy_bits = r->entropy_count >> ENTROPY_SHIFT;
721 }
722
723
724 if (entropy_bits >= random_read_wakeup_bits &&
725 wq_has_sleeper(&random_read_wait)) {
726 wake_up_interruptible(&random_read_wait);
727 kill_fasync(&fasync, SIGIO, POLL_IN);
728 }
729
730
731
732 if (entropy_bits > random_write_wakeup_bits &&
733 r->initialized &&
734 r->entropy_total >= 2*random_read_wakeup_bits) {
735 struct entropy_store *other = &blocking_pool;
736
737 if (other->entropy_count <=
738 3 * other->poolinfo->poolfracbits / 4) {
739 schedule_work(&other->push_work);
740 r->entropy_total = 0;
741 }
742 }
743 }
744}
745
746static int credit_entropy_bits_safe(struct entropy_store *r, int nbits)
747{
748 const int nbits_max = r->poolinfo->poolwords * 32;
749
750 if (nbits < 0)
751 return -EINVAL;
752
753
754 nbits = min(nbits, nbits_max);
755
756 credit_entropy_bits(r, nbits);
757 return 0;
758}
759
760
761
762
763
764
765
766#define CRNG_RESEED_INTERVAL (300*HZ)
767
768static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);
769
770#ifdef CONFIG_NUMA
771
772
773
774
775
776
777static struct crng_state **crng_node_pool __read_mostly;
778#endif
779
780static void invalidate_batched_entropy(void);
781
782static void crng_initialize(struct crng_state *crng)
783{
784 int i;
785 unsigned long rv;
786
787 memcpy(&crng->state[0], "expand 32-byte k", 16);
788 if (crng == &primary_crng)
789 _extract_entropy(&input_pool, &crng->state[4],
790 sizeof(__u32) * 12, 0);
791 else
792 _get_random_bytes(&crng->state[4], sizeof(__u32) * 12);
793 for (i = 4; i < 16; i++) {
794 if (!arch_get_random_seed_long(&rv) &&
795 !arch_get_random_long(&rv))
796 rv = random_get_entropy();
797 crng->state[i] ^= rv;
798 }
799 crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1;
800}
801
802#ifdef CONFIG_NUMA
803static void do_numa_crng_init(struct work_struct *work)
804{
805 int i;
806 struct crng_state *crng;
807 struct crng_state **pool;
808
809 pool = kcalloc(nr_node_ids, sizeof(*pool), GFP_KERNEL|__GFP_NOFAIL);
810 for_each_online_node(i) {
811 crng = kmalloc_node(sizeof(struct crng_state),
812 GFP_KERNEL | __GFP_NOFAIL, i);
813 spin_lock_init(&crng->lock);
814 crng_initialize(crng);
815 pool[i] = crng;
816 }
817 mb();
818 if (cmpxchg(&crng_node_pool, NULL, pool)) {
819 for_each_node(i)
820 kfree(pool[i]);
821 kfree(pool);
822 }
823}
824
825static DECLARE_WORK(numa_crng_init_work, do_numa_crng_init);
826
827static void numa_crng_init(void)
828{
829 schedule_work(&numa_crng_init_work);
830}
831#else
832static void numa_crng_init(void) {}
833#endif
834
835
836
837
838
839static int crng_fast_load(const char *cp, size_t len)
840{
841 unsigned long flags;
842 char *p;
843
844 if (!spin_trylock_irqsave(&primary_crng.lock, flags))
845 return 0;
846 if (crng_init != 0) {
847 spin_unlock_irqrestore(&primary_crng.lock, flags);
848 return 0;
849 }
850 p = (unsigned char *) &primary_crng.state[4];
851 while (len > 0 && crng_init_cnt < CRNG_INIT_CNT_THRESH) {
852 p[crng_init_cnt % CHACHA20_KEY_SIZE] ^= *cp;
853 cp++; crng_init_cnt++; len--;
854 }
855 spin_unlock_irqrestore(&primary_crng.lock, flags);
856 if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) {
857 invalidate_batched_entropy();
858 crng_init = 1;
859 wake_up_interruptible(&crng_init_wait);
860 pr_notice("random: fast init done\n");
861 }
862 return 1;
863}
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879static int crng_slow_load(const char *cp, size_t len)
880{
881 unsigned long flags;
882 static unsigned char lfsr = 1;
883 unsigned char tmp;
884 unsigned i, max = CHACHA20_KEY_SIZE;
885 const char * src_buf = cp;
886 char * dest_buf = (char *) &primary_crng.state[4];
887
888 if (!spin_trylock_irqsave(&primary_crng.lock, flags))
889 return 0;
890 if (crng_init != 0) {
891 spin_unlock_irqrestore(&primary_crng.lock, flags);
892 return 0;
893 }
894 if (len > max)
895 max = len;
896
897 for (i = 0; i < max ; i++) {
898 tmp = lfsr;
899 lfsr >>= 1;
900 if (tmp & 1)
901 lfsr ^= 0xE1;
902 tmp = dest_buf[i % CHACHA20_KEY_SIZE];
903 dest_buf[i % CHACHA20_KEY_SIZE] ^= src_buf[i % len] ^ lfsr;
904 lfsr += (tmp << 3) | (tmp >> 5);
905 }
906 spin_unlock_irqrestore(&primary_crng.lock, flags);
907 return 1;
908}
909
910static void crng_reseed(struct crng_state *crng, struct entropy_store *r)
911{
912 unsigned long flags;
913 int i, num;
914 union {
915 __u32 block[CHACHA20_BLOCK_WORDS];
916 __u32 key[8];
917 } buf;
918
919 if (r) {
920 num = extract_entropy(r, &buf, 32, 16, 0);
921 if (num == 0)
922 return;
923 } else {
924 _extract_crng(&primary_crng, buf.block);
925 _crng_backtrack_protect(&primary_crng, buf.block,
926 CHACHA20_KEY_SIZE);
927 }
928 spin_lock_irqsave(&crng->lock, flags);
929 for (i = 0; i < 8; i++) {
930 unsigned long rv;
931 if (!arch_get_random_seed_long(&rv) &&
932 !arch_get_random_long(&rv))
933 rv = random_get_entropy();
934 crng->state[i+4] ^= buf.key[i] ^ rv;
935 }
936 memzero_explicit(&buf, sizeof(buf));
937 crng->init_time = jiffies;
938 spin_unlock_irqrestore(&crng->lock, flags);
939 if (crng == &primary_crng && crng_init < 2) {
940 invalidate_batched_entropy();
941 numa_crng_init();
942 crng_init = 2;
943 process_random_ready_list();
944 wake_up_interruptible(&crng_init_wait);
945 pr_notice("random: crng init done\n");
946 if (unseeded_warning.missed) {
947 pr_notice("random: %d get_random_xx warning(s) missed "
948 "due to ratelimiting\n",
949 unseeded_warning.missed);
950 unseeded_warning.missed = 0;
951 }
952 if (urandom_warning.missed) {
953 pr_notice("random: %d urandom warning(s) missed "
954 "due to ratelimiting\n",
955 urandom_warning.missed);
956 urandom_warning.missed = 0;
957 }
958 }
959}
960
961static void _extract_crng(struct crng_state *crng,
962 __u32 out[CHACHA20_BLOCK_WORDS])
963{
964 unsigned long v, flags;
965
966 if (crng_ready() &&
967 (time_after(crng_global_init_time, crng->init_time) ||
968 time_after(jiffies, crng->init_time + CRNG_RESEED_INTERVAL)))
969 crng_reseed(crng, crng == &primary_crng ? &input_pool : NULL);
970 spin_lock_irqsave(&crng->lock, flags);
971 if (arch_get_random_long(&v))
972 crng->state[14] ^= v;
973 chacha20_block(&crng->state[0], out);
974 if (crng->state[12] == 0)
975 crng->state[13]++;
976 spin_unlock_irqrestore(&crng->lock, flags);
977}
978
979static void extract_crng(__u32 out[CHACHA20_BLOCK_WORDS])
980{
981 struct crng_state *crng = NULL;
982
983#ifdef CONFIG_NUMA
984 if (crng_node_pool)
985 crng = crng_node_pool[numa_node_id()];
986 if (crng == NULL)
987#endif
988 crng = &primary_crng;
989 _extract_crng(crng, out);
990}
991
992
993
994
995
996static void _crng_backtrack_protect(struct crng_state *crng,
997 __u32 tmp[CHACHA20_BLOCK_WORDS], int used)
998{
999 unsigned long flags;
1000 __u32 *s, *d;
1001 int i;
1002
1003 used = round_up(used, sizeof(__u32));
1004 if (used + CHACHA20_KEY_SIZE > CHACHA20_BLOCK_SIZE) {
1005 extract_crng(tmp);
1006 used = 0;
1007 }
1008 spin_lock_irqsave(&crng->lock, flags);
1009 s = &tmp[used / sizeof(__u32)];
1010 d = &crng->state[4];
1011 for (i=0; i < 8; i++)
1012 *d++ ^= *s++;
1013 spin_unlock_irqrestore(&crng->lock, flags);
1014}
1015
1016static void crng_backtrack_protect(__u32 tmp[CHACHA20_BLOCK_WORDS], int used)
1017{
1018 struct crng_state *crng = NULL;
1019
1020#ifdef CONFIG_NUMA
1021 if (crng_node_pool)
1022 crng = crng_node_pool[numa_node_id()];
1023 if (crng == NULL)
1024#endif
1025 crng = &primary_crng;
1026 _crng_backtrack_protect(crng, tmp, used);
1027}
1028
1029static ssize_t extract_crng_user(void __user *buf, size_t nbytes)
1030{
1031 ssize_t ret = 0, i = CHACHA20_BLOCK_SIZE;
1032 __u32 tmp[CHACHA20_BLOCK_WORDS];
1033 int large_request = (nbytes > 256);
1034
1035 while (nbytes) {
1036 if (large_request && need_resched()) {
1037 if (signal_pending(current)) {
1038 if (ret == 0)
1039 ret = -ERESTARTSYS;
1040 break;
1041 }
1042 schedule();
1043 }
1044
1045 extract_crng(tmp);
1046 i = min_t(int, nbytes, CHACHA20_BLOCK_SIZE);
1047 if (copy_to_user(buf, tmp, i)) {
1048 ret = -EFAULT;
1049 break;
1050 }
1051
1052 nbytes -= i;
1053 buf += i;
1054 ret += i;
1055 }
1056 crng_backtrack_protect(tmp, i);
1057
1058
1059 memzero_explicit(tmp, sizeof(tmp));
1060
1061 return ret;
1062}
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072struct timer_rand_state {
1073 cycles_t last_time;
1074 long last_delta, last_delta2;
1075};
1076
1077#define INIT_TIMER_RAND_STATE { INITIAL_JIFFIES, };
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087void add_device_randomness(const void *buf, unsigned int size)
1088{
1089 unsigned long time = random_get_entropy() ^ jiffies;
1090 unsigned long flags;
1091
1092 if (!crng_ready() && size)
1093 crng_slow_load(buf, size);
1094
1095 trace_add_device_randomness(size, _RET_IP_);
1096 spin_lock_irqsave(&input_pool.lock, flags);
1097 _mix_pool_bytes(&input_pool, buf, size);
1098 _mix_pool_bytes(&input_pool, &time, sizeof(time));
1099 spin_unlock_irqrestore(&input_pool.lock, flags);
1100}
1101EXPORT_SYMBOL(add_device_randomness);
1102
1103static struct timer_rand_state input_timer_state = INIT_TIMER_RAND_STATE;
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115static void add_timer_randomness(struct timer_rand_state *state, unsigned num)
1116{
1117 struct entropy_store *r;
1118 struct {
1119 long jiffies;
1120 unsigned cycles;
1121 unsigned num;
1122 } sample;
1123 long delta, delta2, delta3;
1124
1125 preempt_disable();
1126
1127 sample.jiffies = jiffies;
1128 sample.cycles = random_get_entropy();
1129 sample.num = num;
1130 r = &input_pool;
1131 mix_pool_bytes(r, &sample, sizeof(sample));
1132
1133
1134
1135
1136
1137
1138 delta = sample.jiffies - state->last_time;
1139 state->last_time = sample.jiffies;
1140
1141 delta2 = delta - state->last_delta;
1142 state->last_delta = delta;
1143
1144 delta3 = delta2 - state->last_delta2;
1145 state->last_delta2 = delta2;
1146
1147 if (delta < 0)
1148 delta = -delta;
1149 if (delta2 < 0)
1150 delta2 = -delta2;
1151 if (delta3 < 0)
1152 delta3 = -delta3;
1153 if (delta > delta2)
1154 delta = delta2;
1155 if (delta > delta3)
1156 delta = delta3;
1157
1158
1159
1160
1161
1162
1163 credit_entropy_bits(r, min_t(int, fls(delta>>1), 11));
1164
1165 preempt_enable();
1166}
1167
1168void add_input_randomness(unsigned int type, unsigned int code,
1169 unsigned int value)
1170{
1171 static unsigned char last_value;
1172
1173
1174 if (value == last_value)
1175 return;
1176
1177 last_value = value;
1178 add_timer_randomness(&input_timer_state,
1179 (type << 4) ^ code ^ (code >> 4) ^ value);
1180 trace_add_input_randomness(ENTROPY_BITS(&input_pool));
1181}
1182EXPORT_SYMBOL_GPL(add_input_randomness);
1183
1184static DEFINE_PER_CPU(struct fast_pool, irq_randomness);
1185
1186#ifdef ADD_INTERRUPT_BENCH
1187static unsigned long avg_cycles, avg_deviation;
1188
1189#define AVG_SHIFT 8
1190#define FIXED_1_2 (1 << (AVG_SHIFT-1))
1191
1192static void add_interrupt_bench(cycles_t start)
1193{
1194 long delta = random_get_entropy() - start;
1195
1196
1197 delta = delta - ((avg_cycles + FIXED_1_2) >> AVG_SHIFT);
1198 avg_cycles += delta;
1199
1200 delta = abs(delta) - ((avg_deviation + FIXED_1_2) >> AVG_SHIFT);
1201 avg_deviation += delta;
1202}
1203#else
1204#define add_interrupt_bench(x)
1205#endif
1206
1207static __u32 get_reg(struct fast_pool *f, struct pt_regs *regs)
1208{
1209 __u32 *ptr = (__u32 *) regs;
1210 unsigned int idx;
1211
1212 if (regs == NULL)
1213 return 0;
1214 idx = READ_ONCE(f->reg_idx);
1215 if (idx >= sizeof(struct pt_regs) / sizeof(__u32))
1216 idx = 0;
1217 ptr += idx++;
1218 WRITE_ONCE(f->reg_idx, idx);
1219 return *ptr;
1220}
1221
1222void add_interrupt_randomness(int irq, int irq_flags)
1223{
1224 struct entropy_store *r;
1225 struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness);
1226 struct pt_regs *regs = get_irq_regs();
1227 unsigned long now = jiffies;
1228 cycles_t cycles = random_get_entropy();
1229 __u32 c_high, j_high;
1230 __u64 ip;
1231 unsigned long seed;
1232 int credit = 0;
1233
1234 if (cycles == 0)
1235 cycles = get_reg(fast_pool, regs);
1236 c_high = (sizeof(cycles) > 4) ? cycles >> 32 : 0;
1237 j_high = (sizeof(now) > 4) ? now >> 32 : 0;
1238 fast_pool->pool[0] ^= cycles ^ j_high ^ irq;
1239 fast_pool->pool[1] ^= now ^ c_high;
1240 ip = regs ? instruction_pointer(regs) : _RET_IP_;
1241 fast_pool->pool[2] ^= ip;
1242 fast_pool->pool[3] ^= (sizeof(ip) > 4) ? ip >> 32 :
1243 get_reg(fast_pool, regs);
1244
1245 fast_mix(fast_pool);
1246 add_interrupt_bench(cycles);
1247
1248 if (unlikely(crng_init == 0)) {
1249 if ((fast_pool->count >= 64) &&
1250 crng_fast_load((char *) fast_pool->pool,
1251 sizeof(fast_pool->pool))) {
1252 fast_pool->count = 0;
1253 fast_pool->last = now;
1254 }
1255 return;
1256 }
1257
1258 if ((fast_pool->count < 64) &&
1259 !time_after(now, fast_pool->last + HZ))
1260 return;
1261
1262 r = &input_pool;
1263 if (!spin_trylock(&r->lock))
1264 return;
1265
1266 fast_pool->last = now;
1267 __mix_pool_bytes(r, &fast_pool->pool, sizeof(fast_pool->pool));
1268
1269
1270
1271
1272
1273
1274
1275 if (arch_get_random_seed_long(&seed)) {
1276 __mix_pool_bytes(r, &seed, sizeof(seed));
1277 credit = 1;
1278 }
1279 spin_unlock(&r->lock);
1280
1281 fast_pool->count = 0;
1282
1283
1284 credit_entropy_bits(r, credit + 1);
1285}
1286EXPORT_SYMBOL_GPL(add_interrupt_randomness);
1287
1288#ifdef CONFIG_BLOCK
1289void add_disk_randomness(struct gendisk *disk)
1290{
1291 if (!disk || !disk->random)
1292 return;
1293
1294 add_timer_randomness(disk->random, 0x100 + disk_devt(disk));
1295 trace_add_disk_randomness(disk_devt(disk), ENTROPY_BITS(&input_pool));
1296}
1297EXPORT_SYMBOL_GPL(add_disk_randomness);
1298#endif
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311static void _xfer_secondary_pool(struct entropy_store *r, size_t nbytes);
1312static void xfer_secondary_pool(struct entropy_store *r, size_t nbytes)
1313{
1314 if (!r->pull ||
1315 r->entropy_count >= (nbytes << (ENTROPY_SHIFT + 3)) ||
1316 r->entropy_count > r->poolinfo->poolfracbits)
1317 return;
1318
1319 _xfer_secondary_pool(r, nbytes);
1320}
1321
1322static void _xfer_secondary_pool(struct entropy_store *r, size_t nbytes)
1323{
1324 __u32 tmp[OUTPUT_POOL_WORDS];
1325
1326 int bytes = nbytes;
1327
1328
1329 bytes = max_t(int, bytes, random_read_wakeup_bits / 8);
1330
1331 bytes = min_t(int, bytes, sizeof(tmp));
1332
1333 trace_xfer_secondary_pool(r->name, bytes * 8, nbytes * 8,
1334 ENTROPY_BITS(r), ENTROPY_BITS(r->pull));
1335 bytes = extract_entropy(r->pull, tmp, bytes,
1336 random_read_wakeup_bits / 8, 0);
1337 mix_pool_bytes(r, tmp, bytes);
1338 credit_entropy_bits(r, bytes*8);
1339}
1340
1341
1342
1343
1344
1345
1346
1347static void push_to_pool(struct work_struct *work)
1348{
1349 struct entropy_store *r = container_of(work, struct entropy_store,
1350 push_work);
1351 BUG_ON(!r);
1352 _xfer_secondary_pool(r, random_read_wakeup_bits/8);
1353 trace_push_to_pool(r->name, r->entropy_count >> ENTROPY_SHIFT,
1354 r->pull->entropy_count >> ENTROPY_SHIFT);
1355}
1356
1357
1358
1359
1360
1361static size_t account(struct entropy_store *r, size_t nbytes, int min,
1362 int reserved)
1363{
1364 int entropy_count, orig, have_bytes;
1365 size_t ibytes, nfrac;
1366
1367 BUG_ON(r->entropy_count > r->poolinfo->poolfracbits);
1368
1369
1370retry:
1371 entropy_count = orig = READ_ONCE(r->entropy_count);
1372 ibytes = nbytes;
1373
1374 have_bytes = entropy_count >> (ENTROPY_SHIFT + 3);
1375
1376 if ((have_bytes -= reserved) < 0)
1377 have_bytes = 0;
1378 ibytes = min_t(size_t, ibytes, have_bytes);
1379 if (ibytes < min)
1380 ibytes = 0;
1381
1382 if (unlikely(entropy_count < 0)) {
1383 pr_warn("random: negative entropy count: pool %s count %d\n",
1384 r->name, entropy_count);
1385 WARN_ON(1);
1386 entropy_count = 0;
1387 }
1388 nfrac = ibytes << (ENTROPY_SHIFT + 3);
1389 if ((size_t) entropy_count > nfrac)
1390 entropy_count -= nfrac;
1391 else
1392 entropy_count = 0;
1393
1394 if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig)
1395 goto retry;
1396
1397 trace_debit_entropy(r->name, 8 * ibytes);
1398 if (ibytes &&
1399 (r->entropy_count >> ENTROPY_SHIFT) < random_write_wakeup_bits) {
1400 wake_up_interruptible(&random_write_wait);
1401 kill_fasync(&fasync, SIGIO, POLL_OUT);
1402 }
1403
1404 return ibytes;
1405}
1406
1407
1408
1409
1410
1411
1412
1413static void extract_buf(struct entropy_store *r, __u8 *out)
1414{
1415 int i;
1416 union {
1417 __u32 w[5];
1418 unsigned long l[LONGS(20)];
1419 } hash;
1420 __u32 workspace[SHA_WORKSPACE_WORDS];
1421 unsigned long flags;
1422
1423
1424
1425
1426
1427 sha_init(hash.w);
1428 for (i = 0; i < LONGS(20); i++) {
1429 unsigned long v;
1430 if (!arch_get_random_long(&v))
1431 break;
1432 hash.l[i] = v;
1433 }
1434
1435
1436 spin_lock_irqsave(&r->lock, flags);
1437 for (i = 0; i < r->poolinfo->poolwords; i += 16)
1438 sha_transform(hash.w, (__u8 *)(r->pool + i), workspace);
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449 __mix_pool_bytes(r, hash.w, sizeof(hash.w));
1450 spin_unlock_irqrestore(&r->lock, flags);
1451
1452 memzero_explicit(workspace, sizeof(workspace));
1453
1454
1455
1456
1457
1458
1459 hash.w[0] ^= hash.w[3];
1460 hash.w[1] ^= hash.w[4];
1461 hash.w[2] ^= rol32(hash.w[2], 16);
1462
1463 memcpy(out, &hash, EXTRACT_SIZE);
1464 memzero_explicit(&hash, sizeof(hash));
1465}
1466
1467static ssize_t _extract_entropy(struct entropy_store *r, void *buf,
1468 size_t nbytes, int fips)
1469{
1470 ssize_t ret = 0, i;
1471 __u8 tmp[EXTRACT_SIZE];
1472 unsigned long flags;
1473
1474 while (nbytes) {
1475 extract_buf(r, tmp);
1476
1477 if (fips) {
1478 spin_lock_irqsave(&r->lock, flags);
1479 if (!memcmp(tmp, r->last_data, EXTRACT_SIZE))
1480 panic("Hardware RNG duplicated output!\n");
1481 memcpy(r->last_data, tmp, EXTRACT_SIZE);
1482 spin_unlock_irqrestore(&r->lock, flags);
1483 }
1484 i = min_t(int, nbytes, EXTRACT_SIZE);
1485 memcpy(buf, tmp, i);
1486 nbytes -= i;
1487 buf += i;
1488 ret += i;
1489 }
1490
1491
1492 memzero_explicit(tmp, sizeof(tmp));
1493
1494 return ret;
1495}
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506static ssize_t extract_entropy(struct entropy_store *r, void *buf,
1507 size_t nbytes, int min, int reserved)
1508{
1509 __u8 tmp[EXTRACT_SIZE];
1510 unsigned long flags;
1511
1512
1513 if (fips_enabled) {
1514 spin_lock_irqsave(&r->lock, flags);
1515 if (!r->last_data_init) {
1516 r->last_data_init = 1;
1517 spin_unlock_irqrestore(&r->lock, flags);
1518 trace_extract_entropy(r->name, EXTRACT_SIZE,
1519 ENTROPY_BITS(r), _RET_IP_);
1520 xfer_secondary_pool(r, EXTRACT_SIZE);
1521 extract_buf(r, tmp);
1522 spin_lock_irqsave(&r->lock, flags);
1523 memcpy(r->last_data, tmp, EXTRACT_SIZE);
1524 }
1525 spin_unlock_irqrestore(&r->lock, flags);
1526 }
1527
1528 trace_extract_entropy(r->name, nbytes, ENTROPY_BITS(r), _RET_IP_);
1529 xfer_secondary_pool(r, nbytes);
1530 nbytes = account(r, nbytes, min, reserved);
1531
1532 return _extract_entropy(r, buf, nbytes, fips_enabled);
1533}
1534
1535
1536
1537
1538
1539static ssize_t extract_entropy_user(struct entropy_store *r, void __user *buf,
1540 size_t nbytes)
1541{
1542 ssize_t ret = 0, i;
1543 __u8 tmp[EXTRACT_SIZE];
1544 int large_request = (nbytes > 256);
1545
1546 trace_extract_entropy_user(r->name, nbytes, ENTROPY_BITS(r), _RET_IP_);
1547 xfer_secondary_pool(r, nbytes);
1548 nbytes = account(r, nbytes, 0, 0);
1549
1550 while (nbytes) {
1551 if (large_request && need_resched()) {
1552 if (signal_pending(current)) {
1553 if (ret == 0)
1554 ret = -ERESTARTSYS;
1555 break;
1556 }
1557 schedule();
1558 }
1559
1560 extract_buf(r, tmp);
1561 i = min_t(int, nbytes, EXTRACT_SIZE);
1562 if (copy_to_user(buf, tmp, i)) {
1563 ret = -EFAULT;
1564 break;
1565 }
1566
1567 nbytes -= i;
1568 buf += i;
1569 ret += i;
1570 }
1571
1572
1573 memzero_explicit(tmp, sizeof(tmp));
1574
1575 return ret;
1576}
1577
1578#define warn_unseeded_randomness(previous) \
1579 _warn_unseeded_randomness(__func__, (void *) _RET_IP_, (previous))
1580
1581static void _warn_unseeded_randomness(const char *func_name, void *caller,
1582 void **previous)
1583{
1584#ifdef CONFIG_WARN_ALL_UNSEEDED_RANDOM
1585 const bool print_once = false;
1586#else
1587 static bool print_once __read_mostly;
1588#endif
1589
1590 if (print_once ||
1591 crng_ready() ||
1592 (previous && (caller == READ_ONCE(*previous))))
1593 return;
1594 WRITE_ONCE(*previous, caller);
1595#ifndef CONFIG_WARN_ALL_UNSEEDED_RANDOM
1596 print_once = true;
1597#endif
1598 if (__ratelimit(&unseeded_warning))
1599 pr_notice("random: %s called from %pS with crng_init=%d\n",
1600 func_name, caller, crng_init);
1601}
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613static void _get_random_bytes(void *buf, int nbytes)
1614{
1615 __u32 tmp[CHACHA20_BLOCK_WORDS];
1616
1617 trace_get_random_bytes(nbytes, _RET_IP_);
1618
1619 while (nbytes >= CHACHA20_BLOCK_SIZE) {
1620 extract_crng(buf);
1621 buf += CHACHA20_BLOCK_SIZE;
1622 nbytes -= CHACHA20_BLOCK_SIZE;
1623 }
1624
1625 if (nbytes > 0) {
1626 extract_crng(tmp);
1627 memcpy(buf, tmp, nbytes);
1628 crng_backtrack_protect(tmp, nbytes);
1629 } else
1630 crng_backtrack_protect(tmp, CHACHA20_BLOCK_SIZE);
1631 memzero_explicit(tmp, sizeof(tmp));
1632}
1633
1634void get_random_bytes(void *buf, int nbytes)
1635{
1636 static void *previous;
1637
1638 warn_unseeded_randomness(&previous);
1639 _get_random_bytes(buf, nbytes);
1640}
1641EXPORT_SYMBOL(get_random_bytes);
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653int wait_for_random_bytes(void)
1654{
1655 if (likely(crng_ready()))
1656 return 0;
1657 return wait_event_interruptible(crng_init_wait, crng_ready());
1658}
1659EXPORT_SYMBOL(wait_for_random_bytes);
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669int add_random_ready_callback(struct random_ready_callback *rdy)
1670{
1671 struct module *owner;
1672 unsigned long flags;
1673 int err = -EALREADY;
1674
1675 if (crng_ready())
1676 return err;
1677
1678 owner = rdy->owner;
1679 if (!try_module_get(owner))
1680 return -ENOENT;
1681
1682 spin_lock_irqsave(&random_ready_list_lock, flags);
1683 if (crng_ready())
1684 goto out;
1685
1686 owner = NULL;
1687
1688 list_add(&rdy->list, &random_ready_list);
1689 err = 0;
1690
1691out:
1692 spin_unlock_irqrestore(&random_ready_list_lock, flags);
1693
1694 module_put(owner);
1695
1696 return err;
1697}
1698EXPORT_SYMBOL(add_random_ready_callback);
1699
1700
1701
1702
1703void del_random_ready_callback(struct random_ready_callback *rdy)
1704{
1705 unsigned long flags;
1706 struct module *owner = NULL;
1707
1708 spin_lock_irqsave(&random_ready_list_lock, flags);
1709 if (!list_empty(&rdy->list)) {
1710 list_del_init(&rdy->list);
1711 owner = rdy->owner;
1712 }
1713 spin_unlock_irqrestore(&random_ready_list_lock, flags);
1714
1715 module_put(owner);
1716}
1717EXPORT_SYMBOL(del_random_ready_callback);
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729void get_random_bytes_arch(void *buf, int nbytes)
1730{
1731 char *p = buf;
1732
1733 trace_get_random_bytes_arch(nbytes, _RET_IP_);
1734 while (nbytes) {
1735 unsigned long v;
1736 int chunk = min(nbytes, (int)sizeof(unsigned long));
1737
1738 if (!arch_get_random_long(&v))
1739 break;
1740
1741 memcpy(p, &v, chunk);
1742 p += chunk;
1743 nbytes -= chunk;
1744 }
1745
1746 if (nbytes)
1747 get_random_bytes(p, nbytes);
1748}
1749EXPORT_SYMBOL(get_random_bytes_arch);
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761static void init_std_data(struct entropy_store *r)
1762{
1763 int i;
1764 ktime_t now = ktime_get_real();
1765 unsigned long rv;
1766
1767 r->last_pulled = jiffies;
1768 mix_pool_bytes(r, &now, sizeof(now));
1769 for (i = r->poolinfo->poolbytes; i > 0; i -= sizeof(rv)) {
1770 if (!arch_get_random_seed_long(&rv) &&
1771 !arch_get_random_long(&rv))
1772 rv = random_get_entropy();
1773 mix_pool_bytes(r, &rv, sizeof(rv));
1774 }
1775 mix_pool_bytes(r, utsname(), sizeof(*(utsname())));
1776}
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788static int rand_initialize(void)
1789{
1790 init_std_data(&input_pool);
1791 init_std_data(&blocking_pool);
1792 crng_initialize(&primary_crng);
1793 crng_global_init_time = jiffies;
1794 if (ratelimit_disable) {
1795 urandom_warning.interval = 0;
1796 unseeded_warning.interval = 0;
1797 }
1798 return 0;
1799}
1800early_initcall(rand_initialize);
1801
1802#ifdef CONFIG_BLOCK
1803void rand_initialize_disk(struct gendisk *disk)
1804{
1805 struct timer_rand_state *state;
1806
1807
1808
1809
1810
1811 state = kzalloc(sizeof(struct timer_rand_state), GFP_KERNEL);
1812 if (state) {
1813 state->last_time = INITIAL_JIFFIES;
1814 disk->random = state;
1815 }
1816}
1817#endif
1818
1819static ssize_t
1820_random_read(int nonblock, char __user *buf, size_t nbytes)
1821{
1822 ssize_t n;
1823
1824 if (nbytes == 0)
1825 return 0;
1826
1827 nbytes = min_t(size_t, nbytes, SEC_XFER_SIZE);
1828 while (1) {
1829 n = extract_entropy_user(&blocking_pool, buf, nbytes);
1830 if (n < 0)
1831 return n;
1832 trace_random_read(n*8, (nbytes-n)*8,
1833 ENTROPY_BITS(&blocking_pool),
1834 ENTROPY_BITS(&input_pool));
1835 if (n > 0)
1836 return n;
1837
1838
1839 if (nonblock)
1840 return -EAGAIN;
1841
1842 wait_event_interruptible(random_read_wait,
1843 ENTROPY_BITS(&input_pool) >=
1844 random_read_wakeup_bits);
1845 if (signal_pending(current))
1846 return -ERESTARTSYS;
1847 }
1848}
1849
1850static ssize_t
1851random_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos)
1852{
1853 return _random_read(file->f_flags & O_NONBLOCK, buf, nbytes);
1854}
1855
1856static ssize_t
1857urandom_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos)
1858{
1859 unsigned long flags;
1860 static int maxwarn = 10;
1861 int ret;
1862
1863 if (!crng_ready() && maxwarn > 0) {
1864 maxwarn--;
1865 if (__ratelimit(&urandom_warning))
1866 printk(KERN_NOTICE "random: %s: uninitialized "
1867 "urandom read (%zd bytes read)\n",
1868 current->comm, nbytes);
1869 spin_lock_irqsave(&primary_crng.lock, flags);
1870 crng_init_cnt = 0;
1871 spin_unlock_irqrestore(&primary_crng.lock, flags);
1872 }
1873 nbytes = min_t(size_t, nbytes, INT_MAX >> (ENTROPY_SHIFT + 3));
1874 ret = extract_crng_user(buf, nbytes);
1875 trace_urandom_read(8 * nbytes, 0, ENTROPY_BITS(&input_pool));
1876 return ret;
1877}
1878
1879static __poll_t
1880random_poll(struct file *file, poll_table * wait)
1881{
1882 __poll_t mask;
1883
1884 poll_wait(file, &random_read_wait, wait);
1885 poll_wait(file, &random_write_wait, wait);
1886 mask = 0;
1887 if (ENTROPY_BITS(&input_pool) >= random_read_wakeup_bits)
1888 mask |= EPOLLIN | EPOLLRDNORM;
1889 if (ENTROPY_BITS(&input_pool) < random_write_wakeup_bits)
1890 mask |= EPOLLOUT | EPOLLWRNORM;
1891 return mask;
1892}
1893
1894static int
1895write_pool(struct entropy_store *r, const char __user *buffer, size_t count)
1896{
1897 size_t bytes;
1898 __u32 buf[16];
1899 const char __user *p = buffer;
1900
1901 while (count > 0) {
1902 bytes = min(count, sizeof(buf));
1903 if (copy_from_user(&buf, p, bytes))
1904 return -EFAULT;
1905
1906 count -= bytes;
1907 p += bytes;
1908
1909 mix_pool_bytes(r, buf, bytes);
1910 cond_resched();
1911 }
1912
1913 return 0;
1914}
1915
1916static ssize_t random_write(struct file *file, const char __user *buffer,
1917 size_t count, loff_t *ppos)
1918{
1919 size_t ret;
1920
1921 ret = write_pool(&input_pool, buffer, count);
1922 if (ret)
1923 return ret;
1924
1925 return (ssize_t)count;
1926}
1927
1928static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
1929{
1930 int size, ent_count;
1931 int __user *p = (int __user *)arg;
1932 int retval;
1933
1934 switch (cmd) {
1935 case RNDGETENTCNT:
1936
1937 ent_count = ENTROPY_BITS(&input_pool);
1938 if (put_user(ent_count, p))
1939 return -EFAULT;
1940 return 0;
1941 case RNDADDTOENTCNT:
1942 if (!capable(CAP_SYS_ADMIN))
1943 return -EPERM;
1944 if (get_user(ent_count, p))
1945 return -EFAULT;
1946 return credit_entropy_bits_safe(&input_pool, ent_count);
1947 case RNDADDENTROPY:
1948 if (!capable(CAP_SYS_ADMIN))
1949 return -EPERM;
1950 if (get_user(ent_count, p++))
1951 return -EFAULT;
1952 if (ent_count < 0)
1953 return -EINVAL;
1954 if (get_user(size, p++))
1955 return -EFAULT;
1956 retval = write_pool(&input_pool, (const char __user *)p,
1957 size);
1958 if (retval < 0)
1959 return retval;
1960 return credit_entropy_bits_safe(&input_pool, ent_count);
1961 case RNDZAPENTCNT:
1962 case RNDCLEARPOOL:
1963
1964
1965
1966
1967 if (!capable(CAP_SYS_ADMIN))
1968 return -EPERM;
1969 input_pool.entropy_count = 0;
1970 blocking_pool.entropy_count = 0;
1971 return 0;
1972 case RNDRESEEDCRNG:
1973 if (!capable(CAP_SYS_ADMIN))
1974 return -EPERM;
1975 if (crng_init < 2)
1976 return -ENODATA;
1977 crng_reseed(&primary_crng, NULL);
1978 crng_global_init_time = jiffies - 1;
1979 return 0;
1980 default:
1981 return -EINVAL;
1982 }
1983}
1984
1985static int random_fasync(int fd, struct file *filp, int on)
1986{
1987 return fasync_helper(fd, filp, on, &fasync);
1988}
1989
1990const struct file_operations random_fops = {
1991 .read = random_read,
1992 .write = random_write,
1993 .poll = random_poll,
1994 .unlocked_ioctl = random_ioctl,
1995 .fasync = random_fasync,
1996 .llseek = noop_llseek,
1997};
1998
1999const struct file_operations urandom_fops = {
2000 .read = urandom_read,
2001 .write = random_write,
2002 .unlocked_ioctl = random_ioctl,
2003 .fasync = random_fasync,
2004 .llseek = noop_llseek,
2005};
2006
2007SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count,
2008 unsigned int, flags)
2009{
2010 int ret;
2011
2012 if (flags & ~(GRND_NONBLOCK|GRND_RANDOM))
2013 return -EINVAL;
2014
2015 if (count > INT_MAX)
2016 count = INT_MAX;
2017
2018 if (flags & GRND_RANDOM)
2019 return _random_read(flags & GRND_NONBLOCK, buf, count);
2020
2021 if (!crng_ready()) {
2022 if (flags & GRND_NONBLOCK)
2023 return -EAGAIN;
2024 ret = wait_for_random_bytes();
2025 if (unlikely(ret))
2026 return ret;
2027 }
2028 return urandom_read(NULL, buf, count, NULL);
2029}
2030
2031
2032
2033
2034
2035
2036
2037#ifdef CONFIG_SYSCTL
2038
2039#include <linux/sysctl.h>
2040
2041static int min_read_thresh = 8, min_write_thresh;
2042static int max_read_thresh = OUTPUT_POOL_WORDS * 32;
2043static int max_write_thresh = INPUT_POOL_WORDS * 32;
2044static int random_min_urandom_seed = 60;
2045static char sysctl_bootid[16];
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056static int proc_do_uuid(struct ctl_table *table, int write,
2057 void __user *buffer, size_t *lenp, loff_t *ppos)
2058{
2059 struct ctl_table fake_table;
2060 unsigned char buf[64], tmp_uuid[16], *uuid;
2061
2062 uuid = table->data;
2063 if (!uuid) {
2064 uuid = tmp_uuid;
2065 generate_random_uuid(uuid);
2066 } else {
2067 static DEFINE_SPINLOCK(bootid_spinlock);
2068
2069 spin_lock(&bootid_spinlock);
2070 if (!uuid[8])
2071 generate_random_uuid(uuid);
2072 spin_unlock(&bootid_spinlock);
2073 }
2074
2075 sprintf(buf, "%pU", uuid);
2076
2077 fake_table.data = buf;
2078 fake_table.maxlen = sizeof(buf);
2079
2080 return proc_dostring(&fake_table, write, buffer, lenp, ppos);
2081}
2082
2083
2084
2085
2086static int proc_do_entropy(struct ctl_table *table, int write,
2087 void __user *buffer, size_t *lenp, loff_t *ppos)
2088{
2089 struct ctl_table fake_table;
2090 int entropy_count;
2091
2092 entropy_count = *(int *)table->data >> ENTROPY_SHIFT;
2093
2094 fake_table.data = &entropy_count;
2095 fake_table.maxlen = sizeof(entropy_count);
2096
2097 return proc_dointvec(&fake_table, write, buffer, lenp, ppos);
2098}
2099
2100static int sysctl_poolsize = INPUT_POOL_WORDS * 32;
2101extern struct ctl_table random_table[];
2102struct ctl_table random_table[] = {
2103 {
2104 .procname = "poolsize",
2105 .data = &sysctl_poolsize,
2106 .maxlen = sizeof(int),
2107 .mode = 0444,
2108 .proc_handler = proc_dointvec,
2109 },
2110 {
2111 .procname = "entropy_avail",
2112 .maxlen = sizeof(int),
2113 .mode = 0444,
2114 .proc_handler = proc_do_entropy,
2115 .data = &input_pool.entropy_count,
2116 },
2117 {
2118 .procname = "read_wakeup_threshold",
2119 .data = &random_read_wakeup_bits,
2120 .maxlen = sizeof(int),
2121 .mode = 0644,
2122 .proc_handler = proc_dointvec_minmax,
2123 .extra1 = &min_read_thresh,
2124 .extra2 = &max_read_thresh,
2125 },
2126 {
2127 .procname = "write_wakeup_threshold",
2128 .data = &random_write_wakeup_bits,
2129 .maxlen = sizeof(int),
2130 .mode = 0644,
2131 .proc_handler = proc_dointvec_minmax,
2132 .extra1 = &min_write_thresh,
2133 .extra2 = &max_write_thresh,
2134 },
2135 {
2136 .procname = "urandom_min_reseed_secs",
2137 .data = &random_min_urandom_seed,
2138 .maxlen = sizeof(int),
2139 .mode = 0644,
2140 .proc_handler = proc_dointvec,
2141 },
2142 {
2143 .procname = "boot_id",
2144 .data = &sysctl_bootid,
2145 .maxlen = 16,
2146 .mode = 0444,
2147 .proc_handler = proc_do_uuid,
2148 },
2149 {
2150 .procname = "uuid",
2151 .maxlen = 16,
2152 .mode = 0444,
2153 .proc_handler = proc_do_uuid,
2154 },
2155#ifdef ADD_INTERRUPT_BENCH
2156 {
2157 .procname = "add_interrupt_avg_cycles",
2158 .data = &avg_cycles,
2159 .maxlen = sizeof(avg_cycles),
2160 .mode = 0444,
2161 .proc_handler = proc_doulongvec_minmax,
2162 },
2163 {
2164 .procname = "add_interrupt_avg_deviation",
2165 .data = &avg_deviation,
2166 .maxlen = sizeof(avg_deviation),
2167 .mode = 0444,
2168 .proc_handler = proc_doulongvec_minmax,
2169 },
2170#endif
2171 { }
2172};
2173#endif
2174
2175struct batched_entropy {
2176 union {
2177 u64 entropy_u64[CHACHA20_BLOCK_SIZE / sizeof(u64)];
2178 u32 entropy_u32[CHACHA20_BLOCK_SIZE / sizeof(u32)];
2179 };
2180 unsigned int position;
2181};
2182static rwlock_t batched_entropy_reset_lock = __RW_LOCK_UNLOCKED(batched_entropy_reset_lock);
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64);
2193u64 get_random_u64(void)
2194{
2195 u64 ret;
2196 bool use_lock;
2197 unsigned long flags = 0;
2198 struct batched_entropy *batch;
2199 static void *previous;
2200
2201#if BITS_PER_LONG == 64
2202 if (arch_get_random_long((unsigned long *)&ret))
2203 return ret;
2204#else
2205 if (arch_get_random_long((unsigned long *)&ret) &&
2206 arch_get_random_long((unsigned long *)&ret + 1))
2207 return ret;
2208#endif
2209
2210 warn_unseeded_randomness(&previous);
2211
2212 use_lock = READ_ONCE(crng_init) < 2;
2213 batch = &get_cpu_var(batched_entropy_u64);
2214 if (use_lock)
2215 read_lock_irqsave(&batched_entropy_reset_lock, flags);
2216 if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) {
2217 extract_crng((__u32 *)batch->entropy_u64);
2218 batch->position = 0;
2219 }
2220 ret = batch->entropy_u64[batch->position++];
2221 if (use_lock)
2222 read_unlock_irqrestore(&batched_entropy_reset_lock, flags);
2223 put_cpu_var(batched_entropy_u64);
2224 return ret;
2225}
2226EXPORT_SYMBOL(get_random_u64);
2227
2228static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32);
2229u32 get_random_u32(void)
2230{
2231 u32 ret;
2232 bool use_lock;
2233 unsigned long flags = 0;
2234 struct batched_entropy *batch;
2235 static void *previous;
2236
2237 if (arch_get_random_int(&ret))
2238 return ret;
2239
2240 warn_unseeded_randomness(&previous);
2241
2242 use_lock = READ_ONCE(crng_init) < 2;
2243 batch = &get_cpu_var(batched_entropy_u32);
2244 if (use_lock)
2245 read_lock_irqsave(&batched_entropy_reset_lock, flags);
2246 if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) {
2247 extract_crng(batch->entropy_u32);
2248 batch->position = 0;
2249 }
2250 ret = batch->entropy_u32[batch->position++];
2251 if (use_lock)
2252 read_unlock_irqrestore(&batched_entropy_reset_lock, flags);
2253 put_cpu_var(batched_entropy_u32);
2254 return ret;
2255}
2256EXPORT_SYMBOL(get_random_u32);
2257
2258
2259
2260
2261
2262static void invalidate_batched_entropy(void)
2263{
2264 int cpu;
2265 unsigned long flags;
2266
2267 write_lock_irqsave(&batched_entropy_reset_lock, flags);
2268 for_each_possible_cpu (cpu) {
2269 per_cpu_ptr(&batched_entropy_u32, cpu)->position = 0;
2270 per_cpu_ptr(&batched_entropy_u64, cpu)->position = 0;
2271 }
2272 write_unlock_irqrestore(&batched_entropy_reset_lock, flags);
2273}
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289unsigned long
2290randomize_page(unsigned long start, unsigned long range)
2291{
2292 if (!PAGE_ALIGNED(start)) {
2293 range -= PAGE_ALIGN(start) - start;
2294 start = PAGE_ALIGN(start);
2295 }
2296
2297 if (start > ULONG_MAX - range)
2298 range = ULONG_MAX - start;
2299
2300 range >>= PAGE_SHIFT;
2301
2302 if (range == 0)
2303 return start;
2304
2305 return start + (get_random_long() % range << PAGE_SHIFT);
2306}
2307
2308
2309
2310
2311
2312void add_hwgenerator_randomness(const char *buffer, size_t count,
2313 size_t entropy)
2314{
2315 struct entropy_store *poolp = &input_pool;
2316
2317 if (unlikely(crng_init == 0)) {
2318 crng_fast_load(buffer, count);
2319 return;
2320 }
2321
2322
2323
2324
2325
2326 wait_event_interruptible(random_write_wait, kthread_should_stop() ||
2327 ENTROPY_BITS(&input_pool) <= random_write_wakeup_bits);
2328 mix_pool_bytes(poolp, buffer, count);
2329 credit_entropy_bits(poolp, entropy);
2330}
2331EXPORT_SYMBOL_GPL(add_hwgenerator_randomness);
2332