1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#define BOOT_CTYPE_H
21
22
23
24
25
26
27
28
29#define _LINUX_EXPORT_H
30#define EXPORT_SYMBOL(sym)
31
32#include "misc.h"
33#include "error.h"
34#include "../string.h"
35
36#include <generated/compile.h>
37#include <linux/module.h>
38#include <linux/uts.h>
39#include <linux/utsname.h>
40#include <linux/ctype.h>
41#include <linux/efi.h>
42#include <generated/utsrelease.h>
43#include <asm/efi.h>
44
45
46#define STATIC
47#include <linux/decompress/mm.h>
48
49#ifdef CONFIG_X86_5LEVEL
50unsigned int pgtable_l5_enabled __ro_after_init;
51unsigned int pgdir_shift __ro_after_init = 39;
52unsigned int ptrs_per_p4d __ro_after_init = 1;
53#endif
54
55extern unsigned long get_cmd_line_ptr(void);
56
57
58pteval_t __default_kernel_pte_mask __read_mostly = ~0;
59
60
61static const char build_str[] = UTS_RELEASE " (" LINUX_COMPILE_BY "@"
62 LINUX_COMPILE_HOST ") (" LINUX_COMPILER ") " UTS_VERSION;
63
64static unsigned long rotate_xor(unsigned long hash, const void *area,
65 size_t size)
66{
67 size_t i;
68 unsigned long *ptr = (unsigned long *)area;
69
70 for (i = 0; i < size / sizeof(hash); i++) {
71
72 hash = (hash << ((sizeof(hash) * 8) - 7)) | (hash >> 7);
73 hash ^= ptr[i];
74 }
75
76 return hash;
77}
78
79
80static unsigned long get_boot_seed(void)
81{
82 unsigned long hash = 0;
83
84 hash = rotate_xor(hash, build_str, sizeof(build_str));
85 hash = rotate_xor(hash, boot_params, sizeof(*boot_params));
86
87 return hash;
88}
89
90#define KASLR_COMPRESSED_BOOT
91#include "../../lib/kaslr.c"
92
93struct mem_vector {
94 unsigned long long start;
95 unsigned long long size;
96};
97
98
99#define MAX_MEMMAP_REGIONS 4
100
101static bool memmap_too_large;
102
103
104
105unsigned long long mem_limit = ULLONG_MAX;
106
107
108enum mem_avoid_index {
109 MEM_AVOID_ZO_RANGE = 0,
110 MEM_AVOID_INITRD,
111 MEM_AVOID_CMDLINE,
112 MEM_AVOID_BOOTPARAMS,
113 MEM_AVOID_MEMMAP_BEGIN,
114 MEM_AVOID_MEMMAP_END = MEM_AVOID_MEMMAP_BEGIN + MAX_MEMMAP_REGIONS - 1,
115 MEM_AVOID_MAX,
116};
117
118static struct mem_vector mem_avoid[MEM_AVOID_MAX];
119
120static bool mem_overlaps(struct mem_vector *one, struct mem_vector *two)
121{
122
123 if (one->start + one->size <= two->start)
124 return false;
125
126 if (one->start >= two->start + two->size)
127 return false;
128 return true;
129}
130
131char *skip_spaces(const char *str)
132{
133 while (isspace(*str))
134 ++str;
135 return (char *)str;
136}
137#include "../../../../lib/ctype.c"
138#include "../../../../lib/cmdline.c"
139
140static int
141parse_memmap(char *p, unsigned long long *start, unsigned long long *size)
142{
143 char *oldp;
144
145 if (!p)
146 return -EINVAL;
147
148
149 if (!strncmp(p, "exactmap", 8))
150 return -EINVAL;
151
152 oldp = p;
153 *size = memparse(p, &p);
154 if (p == oldp)
155 return -EINVAL;
156
157 switch (*p) {
158 case '#':
159 case '$':
160 case '!':
161 *start = memparse(p + 1, &p);
162 return 0;
163 case '@':
164
165 *size = 0;
166
167 default:
168
169
170
171
172
173 *start = 0;
174 return 0;
175 }
176
177 return -EINVAL;
178}
179
180static void mem_avoid_memmap(char *str)
181{
182 static int i;
183
184 if (i >= MAX_MEMMAP_REGIONS)
185 return;
186
187 while (str && (i < MAX_MEMMAP_REGIONS)) {
188 int rc;
189 unsigned long long start, size;
190 char *k = strchr(str, ',');
191
192 if (k)
193 *k++ = 0;
194
195 rc = parse_memmap(str, &start, &size);
196 if (rc < 0)
197 break;
198 str = k;
199
200 if (start == 0) {
201
202 if (size > 0)
203 mem_limit = size;
204
205 continue;
206 }
207
208 mem_avoid[MEM_AVOID_MEMMAP_BEGIN + i].start = start;
209 mem_avoid[MEM_AVOID_MEMMAP_BEGIN + i].size = size;
210 i++;
211 }
212
213
214 if ((i >= MAX_MEMMAP_REGIONS) && str)
215 memmap_too_large = true;
216}
217
218static int handle_mem_memmap(void)
219{
220 char *args = (char *)get_cmd_line_ptr();
221 size_t len = strlen((char *)args);
222 char *tmp_cmdline;
223 char *param, *val;
224 u64 mem_size;
225
226 if (!strstr(args, "memmap=") && !strstr(args, "mem="))
227 return 0;
228
229 tmp_cmdline = malloc(len + 1);
230 if (!tmp_cmdline)
231 error("Failed to allocate space for tmp_cmdline");
232
233 memcpy(tmp_cmdline, args, len);
234 tmp_cmdline[len] = 0;
235 args = tmp_cmdline;
236
237
238 args = skip_spaces(args);
239
240 while (*args) {
241 args = next_arg(args, ¶m, &val);
242
243 if (!val && strcmp(param, "--") == 0) {
244 warn("Only '--' specified in cmdline");
245 free(tmp_cmdline);
246 return -1;
247 }
248
249 if (!strcmp(param, "memmap")) {
250 mem_avoid_memmap(val);
251 } else if (!strcmp(param, "mem")) {
252 char *p = val;
253
254 if (!strcmp(p, "nopentium"))
255 continue;
256 mem_size = memparse(p, &p);
257 if (mem_size == 0) {
258 free(tmp_cmdline);
259 return -EINVAL;
260 }
261 mem_limit = mem_size;
262 }
263 }
264
265 free(tmp_cmdline);
266 return 0;
267}
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343static void mem_avoid_init(unsigned long input, unsigned long input_size,
344 unsigned long output)
345{
346 unsigned long init_size = boot_params->hdr.init_size;
347 u64 initrd_start, initrd_size;
348 u64 cmd_line, cmd_line_size;
349 char *ptr;
350
351
352
353
354
355 mem_avoid[MEM_AVOID_ZO_RANGE].start = input;
356 mem_avoid[MEM_AVOID_ZO_RANGE].size = (output + init_size) - input;
357 add_identity_map(mem_avoid[MEM_AVOID_ZO_RANGE].start,
358 mem_avoid[MEM_AVOID_ZO_RANGE].size);
359
360
361 initrd_start = (u64)boot_params->ext_ramdisk_image << 32;
362 initrd_start |= boot_params->hdr.ramdisk_image;
363 initrd_size = (u64)boot_params->ext_ramdisk_size << 32;
364 initrd_size |= boot_params->hdr.ramdisk_size;
365 mem_avoid[MEM_AVOID_INITRD].start = initrd_start;
366 mem_avoid[MEM_AVOID_INITRD].size = initrd_size;
367
368
369
370 cmd_line = (u64)boot_params->ext_cmd_line_ptr << 32;
371 cmd_line |= boot_params->hdr.cmd_line_ptr;
372
373 ptr = (char *)(unsigned long)cmd_line;
374 for (cmd_line_size = 0; ptr[cmd_line_size++];)
375 ;
376 mem_avoid[MEM_AVOID_CMDLINE].start = cmd_line;
377 mem_avoid[MEM_AVOID_CMDLINE].size = cmd_line_size;
378 add_identity_map(mem_avoid[MEM_AVOID_CMDLINE].start,
379 mem_avoid[MEM_AVOID_CMDLINE].size);
380
381
382 mem_avoid[MEM_AVOID_BOOTPARAMS].start = (unsigned long)boot_params;
383 mem_avoid[MEM_AVOID_BOOTPARAMS].size = sizeof(*boot_params);
384 add_identity_map(mem_avoid[MEM_AVOID_BOOTPARAMS].start,
385 mem_avoid[MEM_AVOID_BOOTPARAMS].size);
386
387
388
389
390 handle_mem_memmap();
391
392#ifdef CONFIG_X86_VERBOSE_BOOTUP
393
394 add_identity_map(0, PMD_SIZE);
395#endif
396}
397
398
399
400
401
402static bool mem_avoid_overlap(struct mem_vector *img,
403 struct mem_vector *overlap)
404{
405 int i;
406 struct setup_data *ptr;
407 unsigned long earliest = img->start + img->size;
408 bool is_overlapping = false;
409
410 for (i = 0; i < MEM_AVOID_MAX; i++) {
411 if (mem_overlaps(img, &mem_avoid[i]) &&
412 mem_avoid[i].start < earliest) {
413 *overlap = mem_avoid[i];
414 earliest = overlap->start;
415 is_overlapping = true;
416 }
417 }
418
419
420 ptr = (struct setup_data *)(unsigned long)boot_params->hdr.setup_data;
421 while (ptr) {
422 struct mem_vector avoid;
423
424 avoid.start = (unsigned long)ptr;
425 avoid.size = sizeof(*ptr) + ptr->len;
426
427 if (mem_overlaps(img, &avoid) && (avoid.start < earliest)) {
428 *overlap = avoid;
429 earliest = overlap->start;
430 is_overlapping = true;
431 }
432
433 ptr = (struct setup_data *)(unsigned long)ptr->next;
434 }
435
436 return is_overlapping;
437}
438
439struct slot_area {
440 unsigned long addr;
441 int num;
442};
443
444#define MAX_SLOT_AREA 100
445
446static struct slot_area slot_areas[MAX_SLOT_AREA];
447
448static unsigned long slot_max;
449
450static unsigned long slot_area_index;
451
452static void store_slot_info(struct mem_vector *region, unsigned long image_size)
453{
454 struct slot_area slot_area;
455
456 if (slot_area_index == MAX_SLOT_AREA)
457 return;
458
459 slot_area.addr = region->start;
460 slot_area.num = (region->size - image_size) /
461 CONFIG_PHYSICAL_ALIGN + 1;
462
463 if (slot_area.num > 0) {
464 slot_areas[slot_area_index++] = slot_area;
465 slot_max += slot_area.num;
466 }
467}
468
469static unsigned long slots_fetch_random(void)
470{
471 unsigned long slot;
472 int i;
473
474
475 if (slot_max == 0)
476 return 0;
477
478 slot = kaslr_get_random_long("Physical") % slot_max;
479
480 for (i = 0; i < slot_area_index; i++) {
481 if (slot >= slot_areas[i].num) {
482 slot -= slot_areas[i].num;
483 continue;
484 }
485 return slot_areas[i].addr + slot * CONFIG_PHYSICAL_ALIGN;
486 }
487
488 if (i == slot_area_index)
489 debug_putstr("slots_fetch_random() failed!?\n");
490 return 0;
491}
492
493static void process_mem_region(struct mem_vector *entry,
494 unsigned long minimum,
495 unsigned long image_size)
496{
497 struct mem_vector region, overlap;
498 struct slot_area slot_area;
499 unsigned long start_orig, end;
500 struct mem_vector cur_entry;
501
502
503 if (IS_ENABLED(CONFIG_X86_32) && entry->start >= KERNEL_IMAGE_SIZE)
504 return;
505
506
507 if (entry->start + entry->size < minimum)
508 return;
509
510
511 end = min(entry->size + entry->start, mem_limit);
512 if (entry->start >= end)
513 return;
514 cur_entry.start = entry->start;
515 cur_entry.size = end - entry->start;
516
517 region.start = cur_entry.start;
518 region.size = cur_entry.size;
519
520
521 while (slot_area_index < MAX_SLOT_AREA) {
522 start_orig = region.start;
523
524
525 if (region.start < minimum)
526 region.start = minimum;
527
528
529 region.start = ALIGN(region.start, CONFIG_PHYSICAL_ALIGN);
530
531
532 if (region.start > cur_entry.start + cur_entry.size)
533 return;
534
535
536 region.size -= region.start - start_orig;
537
538
539 if (IS_ENABLED(CONFIG_X86_32) &&
540 region.start + region.size > KERNEL_IMAGE_SIZE)
541 region.size = KERNEL_IMAGE_SIZE - region.start;
542
543
544 if (region.size < image_size)
545 return;
546
547
548 if (!mem_avoid_overlap(®ion, &overlap)) {
549 store_slot_info(®ion, image_size);
550 return;
551 }
552
553
554 if (overlap.start > region.start + image_size) {
555 struct mem_vector beginning;
556
557 beginning.start = region.start;
558 beginning.size = overlap.start - region.start;
559 store_slot_info(&beginning, image_size);
560 }
561
562
563 if (overlap.start + overlap.size >= region.start + region.size)
564 return;
565
566
567 region.size -= overlap.start - region.start + overlap.size;
568 region.start = overlap.start + overlap.size;
569 }
570}
571
572#ifdef CONFIG_EFI
573
574
575
576
577static bool
578process_efi_entries(unsigned long minimum, unsigned long image_size)
579{
580 struct efi_info *e = &boot_params->efi_info;
581 bool efi_mirror_found = false;
582 struct mem_vector region;
583 efi_memory_desc_t *md;
584 unsigned long pmap;
585 char *signature;
586 u32 nr_desc;
587 int i;
588
589 signature = (char *)&e->efi_loader_signature;
590 if (strncmp(signature, EFI32_LOADER_SIGNATURE, 4) &&
591 strncmp(signature, EFI64_LOADER_SIGNATURE, 4))
592 return false;
593
594#ifdef CONFIG_X86_32
595
596 if (e->efi_memmap_hi) {
597 warn("EFI memmap is above 4GB, can't be handled now on x86_32. EFI should be disabled.\n");
598 return false;
599 }
600 pmap = e->efi_memmap;
601#else
602 pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
603#endif
604
605 nr_desc = e->efi_memmap_size / e->efi_memdesc_size;
606 for (i = 0; i < nr_desc; i++) {
607 md = efi_early_memdesc_ptr(pmap, e->efi_memdesc_size, i);
608 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) {
609 efi_mirror_found = true;
610 break;
611 }
612 }
613
614 for (i = 0; i < nr_desc; i++) {
615 md = efi_early_memdesc_ptr(pmap, e->efi_memdesc_size, i);
616
617
618
619
620
621
622
623
624
625
626
627
628 if (md->type != EFI_CONVENTIONAL_MEMORY)
629 continue;
630
631 if (efi_mirror_found &&
632 !(md->attribute & EFI_MEMORY_MORE_RELIABLE))
633 continue;
634
635 region.start = md->phys_addr;
636 region.size = md->num_pages << EFI_PAGE_SHIFT;
637 process_mem_region(®ion, minimum, image_size);
638 if (slot_area_index == MAX_SLOT_AREA) {
639 debug_putstr("Aborted EFI scan (slot_areas full)!\n");
640 break;
641 }
642 }
643 return true;
644}
645#else
646static inline bool
647process_efi_entries(unsigned long minimum, unsigned long image_size)
648{
649 return false;
650}
651#endif
652
653static void process_e820_entries(unsigned long minimum,
654 unsigned long image_size)
655{
656 int i;
657 struct mem_vector region;
658 struct boot_e820_entry *entry;
659
660
661 for (i = 0; i < boot_params->e820_entries; i++) {
662 entry = &boot_params->e820_table[i];
663
664 if (entry->type != E820_TYPE_RAM)
665 continue;
666 region.start = entry->addr;
667 region.size = entry->size;
668 process_mem_region(®ion, minimum, image_size);
669 if (slot_area_index == MAX_SLOT_AREA) {
670 debug_putstr("Aborted e820 scan (slot_areas full)!\n");
671 break;
672 }
673 }
674}
675
676static unsigned long find_random_phys_addr(unsigned long minimum,
677 unsigned long image_size)
678{
679
680 if (memmap_too_large) {
681 debug_putstr("Aborted memory entries scan (more than 4 memmap= args)!\n");
682 return 0;
683 }
684
685
686 minimum = ALIGN(minimum, CONFIG_PHYSICAL_ALIGN);
687
688 if (process_efi_entries(minimum, image_size))
689 return slots_fetch_random();
690
691 process_e820_entries(minimum, image_size);
692 return slots_fetch_random();
693}
694
695static unsigned long find_random_virt_addr(unsigned long minimum,
696 unsigned long image_size)
697{
698 unsigned long slots, random_addr;
699
700
701 minimum = ALIGN(minimum, CONFIG_PHYSICAL_ALIGN);
702
703 image_size = ALIGN(image_size, CONFIG_PHYSICAL_ALIGN);
704
705
706
707
708
709
710 slots = (KERNEL_IMAGE_SIZE - minimum - image_size) /
711 CONFIG_PHYSICAL_ALIGN + 1;
712
713 random_addr = kaslr_get_random_long("Virtual") % slots;
714
715 return random_addr * CONFIG_PHYSICAL_ALIGN + minimum;
716}
717
718
719
720
721
722void choose_random_location(unsigned long input,
723 unsigned long input_size,
724 unsigned long *output,
725 unsigned long output_size,
726 unsigned long *virt_addr)
727{
728 unsigned long random_addr, min_addr;
729
730 if (cmdline_find_option_bool("nokaslr")) {
731 warn("KASLR disabled: 'nokaslr' on cmdline.");
732 return;
733 }
734
735#ifdef CONFIG_X86_5LEVEL
736 if (__read_cr4() & X86_CR4_LA57) {
737 pgtable_l5_enabled = 1;
738 pgdir_shift = 48;
739 ptrs_per_p4d = 512;
740 }
741#endif
742
743 boot_params->hdr.loadflags |= KASLR_FLAG;
744
745
746 initialize_identity_maps();
747
748
749 mem_avoid_init(input, input_size, *output);
750
751
752
753
754
755
756 min_addr = min(*output, 512UL << 20);
757
758
759 random_addr = find_random_phys_addr(min_addr, output_size);
760 if (!random_addr) {
761 warn("Physical KASLR disabled: no suitable memory region!");
762 } else {
763
764 if (*output != random_addr) {
765 add_identity_map(random_addr, output_size);
766 *output = random_addr;
767 }
768
769
770
771
772
773
774
775
776 finalize_identity_maps();
777 }
778
779
780
781 if (IS_ENABLED(CONFIG_X86_64))
782 random_addr = find_random_virt_addr(LOAD_PHYSICAL_ADDR, output_size);
783 *virt_addr = random_addr;
784}
785