1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
33
34#include <linux/kernel.h>
35#include <linux/init.h>
36#include <linux/efi.h>
37#include <linux/efi-bgrt.h>
38#include <linux/export.h>
39#include <linux/memblock.h>
40#include <linux/slab.h>
41#include <linux/spinlock.h>
42#include <linux/uaccess.h>
43#include <linux/time.h>
44#include <linux/io.h>
45#include <linux/reboot.h>
46#include <linux/bcd.h>
47
48#include <asm/setup.h>
49#include <asm/efi.h>
50#include <asm/e820/api.h>
51#include <asm/time.h>
52#include <asm/set_memory.h>
53#include <asm/tlbflush.h>
54#include <asm/x86_init.h>
55#include <asm/uv/uv.h>
56
57static struct efi efi_phys __initdata;
58static efi_system_table_t efi_systab __initdata;
59
60static efi_config_table_type_t arch_tables[] __initdata = {
61#ifdef CONFIG_X86_UV
62 {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab},
63#endif
64 {NULL_GUID, NULL, NULL},
65};
66
67u64 efi_setup;
68
69static int add_efi_memmap __initdata;
70static int __init setup_add_efi_memmap(char *arg)
71{
72 add_efi_memmap = 1;
73 return 0;
74}
75early_param("add_efi_memmap", setup_add_efi_memmap);
76
77static efi_status_t __init phys_efi_set_virtual_address_map(
78 unsigned long memory_map_size,
79 unsigned long descriptor_size,
80 u32 descriptor_version,
81 efi_memory_desc_t *virtual_map)
82{
83 efi_status_t status;
84 unsigned long flags;
85 pgd_t *save_pgd;
86
87 save_pgd = efi_call_phys_prolog();
88 if (!save_pgd)
89 return EFI_ABORTED;
90
91
92 local_irq_save(flags);
93 status = efi_call_phys(efi_phys.set_virtual_address_map,
94 memory_map_size, descriptor_size,
95 descriptor_version, virtual_map);
96 local_irq_restore(flags);
97
98 efi_call_phys_epilog(save_pgd);
99
100 return status;
101}
102
103void __init efi_find_mirror(void)
104{
105 efi_memory_desc_t *md;
106 u64 mirror_size = 0, total_size = 0;
107
108 for_each_efi_memory_desc(md) {
109 unsigned long long start = md->phys_addr;
110 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
111
112 total_size += size;
113 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) {
114 memblock_mark_mirror(start, size);
115 mirror_size += size;
116 }
117 }
118 if (mirror_size)
119 pr_info("Memory: %lldM/%lldM mirrored memory\n",
120 mirror_size>>20, total_size>>20);
121}
122
123
124
125
126
127
128
129static void __init do_add_efi_memmap(void)
130{
131 efi_memory_desc_t *md;
132
133 for_each_efi_memory_desc(md) {
134 unsigned long long start = md->phys_addr;
135 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
136 int e820_type;
137
138 switch (md->type) {
139 case EFI_LOADER_CODE:
140 case EFI_LOADER_DATA:
141 case EFI_BOOT_SERVICES_CODE:
142 case EFI_BOOT_SERVICES_DATA:
143 case EFI_CONVENTIONAL_MEMORY:
144 if (md->attribute & EFI_MEMORY_WB)
145 e820_type = E820_TYPE_RAM;
146 else
147 e820_type = E820_TYPE_RESERVED;
148 break;
149 case EFI_ACPI_RECLAIM_MEMORY:
150 e820_type = E820_TYPE_ACPI;
151 break;
152 case EFI_ACPI_MEMORY_NVS:
153 e820_type = E820_TYPE_NVS;
154 break;
155 case EFI_UNUSABLE_MEMORY:
156 e820_type = E820_TYPE_UNUSABLE;
157 break;
158 case EFI_PERSISTENT_MEMORY:
159 e820_type = E820_TYPE_PMEM;
160 break;
161 default:
162
163
164
165
166
167 e820_type = E820_TYPE_RESERVED;
168 break;
169 }
170 e820__range_add(start, size, e820_type);
171 }
172 e820__update_table(e820_table);
173}
174
175int __init efi_memblock_x86_reserve_range(void)
176{
177 struct efi_info *e = &boot_params.efi_info;
178 struct efi_memory_map_data data;
179 phys_addr_t pmap;
180 int rv;
181
182 if (efi_enabled(EFI_PARAVIRT))
183 return 0;
184
185#ifdef CONFIG_X86_32
186
187 if (e->efi_memmap_hi) {
188 pr_err("Memory map is above 4GB, disabling EFI.\n");
189 return -EINVAL;
190 }
191 pmap = e->efi_memmap;
192#else
193 pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
194#endif
195 data.phys_map = pmap;
196 data.size = e->efi_memmap_size;
197 data.desc_size = e->efi_memdesc_size;
198 data.desc_version = e->efi_memdesc_version;
199
200 rv = efi_memmap_init_early(&data);
201 if (rv)
202 return rv;
203
204 if (add_efi_memmap)
205 do_add_efi_memmap();
206
207 WARN(efi.memmap.desc_version != 1,
208 "Unexpected EFI_MEMORY_DESCRIPTOR version %ld",
209 efi.memmap.desc_version);
210
211 memblock_reserve(pmap, efi.memmap.nr_map * efi.memmap.desc_size);
212
213 return 0;
214}
215
216#define OVERFLOW_ADDR_SHIFT (64 - EFI_PAGE_SHIFT)
217#define OVERFLOW_ADDR_MASK (U64_MAX << OVERFLOW_ADDR_SHIFT)
218#define U64_HIGH_BIT (~(U64_MAX >> 1))
219
220static bool __init efi_memmap_entry_valid(const efi_memory_desc_t *md, int i)
221{
222 u64 end = (md->num_pages << EFI_PAGE_SHIFT) + md->phys_addr - 1;
223 u64 end_hi = 0;
224 char buf[64];
225
226 if (md->num_pages == 0) {
227 end = 0;
228 } else if (md->num_pages > EFI_PAGES_MAX ||
229 EFI_PAGES_MAX - md->num_pages <
230 (md->phys_addr >> EFI_PAGE_SHIFT)) {
231 end_hi = (md->num_pages & OVERFLOW_ADDR_MASK)
232 >> OVERFLOW_ADDR_SHIFT;
233
234 if ((md->phys_addr & U64_HIGH_BIT) && !(end & U64_HIGH_BIT))
235 end_hi += 1;
236 } else {
237 return true;
238 }
239
240 pr_warn_once(FW_BUG "Invalid EFI memory map entries:\n");
241
242 if (end_hi) {
243 pr_warn("mem%02u: %s range=[0x%016llx-0x%llx%016llx] (invalid)\n",
244 i, efi_md_typeattr_format(buf, sizeof(buf), md),
245 md->phys_addr, end_hi, end);
246 } else {
247 pr_warn("mem%02u: %s range=[0x%016llx-0x%016llx] (invalid)\n",
248 i, efi_md_typeattr_format(buf, sizeof(buf), md),
249 md->phys_addr, end);
250 }
251 return false;
252}
253
254static void __init efi_clean_memmap(void)
255{
256 efi_memory_desc_t *out = efi.memmap.map;
257 const efi_memory_desc_t *in = out;
258 const efi_memory_desc_t *end = efi.memmap.map_end;
259 int i, n_removal;
260
261 for (i = n_removal = 0; in < end; i++) {
262 if (efi_memmap_entry_valid(in, i)) {
263 if (out != in)
264 memcpy(out, in, efi.memmap.desc_size);
265 out = (void *)out + efi.memmap.desc_size;
266 } else {
267 n_removal++;
268 }
269 in = (void *)in + efi.memmap.desc_size;
270 }
271
272 if (n_removal > 0) {
273 u64 size = efi.memmap.nr_map - n_removal;
274
275 pr_warn("Removing %d invalid memory map entries.\n", n_removal);
276 efi_memmap_install(efi.memmap.phys_map, size);
277 }
278}
279
280void __init efi_print_memmap(void)
281{
282 efi_memory_desc_t *md;
283 int i = 0;
284
285 for_each_efi_memory_desc(md) {
286 char buf[64];
287
288 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx] (%lluMB)\n",
289 i++, efi_md_typeattr_format(buf, sizeof(buf), md),
290 md->phys_addr,
291 md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1,
292 (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
293 }
294}
295
296static int __init efi_systab_init(void *phys)
297{
298 if (efi_enabled(EFI_64BIT)) {
299 efi_system_table_64_t *systab64;
300 struct efi_setup_data *data = NULL;
301 u64 tmp = 0;
302
303 if (efi_setup) {
304 data = early_memremap(efi_setup, sizeof(*data));
305 if (!data)
306 return -ENOMEM;
307 }
308 systab64 = early_memremap((unsigned long)phys,
309 sizeof(*systab64));
310 if (systab64 == NULL) {
311 pr_err("Couldn't map the system table!\n");
312 if (data)
313 early_memunmap(data, sizeof(*data));
314 return -ENOMEM;
315 }
316
317 efi_systab.hdr = systab64->hdr;
318 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
319 systab64->fw_vendor;
320 tmp |= data ? data->fw_vendor : systab64->fw_vendor;
321 efi_systab.fw_revision = systab64->fw_revision;
322 efi_systab.con_in_handle = systab64->con_in_handle;
323 tmp |= systab64->con_in_handle;
324 efi_systab.con_in = systab64->con_in;
325 tmp |= systab64->con_in;
326 efi_systab.con_out_handle = systab64->con_out_handle;
327 tmp |= systab64->con_out_handle;
328 efi_systab.con_out = systab64->con_out;
329 tmp |= systab64->con_out;
330 efi_systab.stderr_handle = systab64->stderr_handle;
331 tmp |= systab64->stderr_handle;
332 efi_systab.stderr = systab64->stderr;
333 tmp |= systab64->stderr;
334 efi_systab.runtime = data ?
335 (void *)(unsigned long)data->runtime :
336 (void *)(unsigned long)systab64->runtime;
337 tmp |= data ? data->runtime : systab64->runtime;
338 efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
339 tmp |= systab64->boottime;
340 efi_systab.nr_tables = systab64->nr_tables;
341 efi_systab.tables = data ? (unsigned long)data->tables :
342 systab64->tables;
343 tmp |= data ? data->tables : systab64->tables;
344
345 early_memunmap(systab64, sizeof(*systab64));
346 if (data)
347 early_memunmap(data, sizeof(*data));
348#ifdef CONFIG_X86_32
349 if (tmp >> 32) {
350 pr_err("EFI data located above 4GB, disabling EFI.\n");
351 return -EINVAL;
352 }
353#endif
354 } else {
355 efi_system_table_32_t *systab32;
356
357 systab32 = early_memremap((unsigned long)phys,
358 sizeof(*systab32));
359 if (systab32 == NULL) {
360 pr_err("Couldn't map the system table!\n");
361 return -ENOMEM;
362 }
363
364 efi_systab.hdr = systab32->hdr;
365 efi_systab.fw_vendor = systab32->fw_vendor;
366 efi_systab.fw_revision = systab32->fw_revision;
367 efi_systab.con_in_handle = systab32->con_in_handle;
368 efi_systab.con_in = systab32->con_in;
369 efi_systab.con_out_handle = systab32->con_out_handle;
370 efi_systab.con_out = systab32->con_out;
371 efi_systab.stderr_handle = systab32->stderr_handle;
372 efi_systab.stderr = systab32->stderr;
373 efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
374 efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
375 efi_systab.nr_tables = systab32->nr_tables;
376 efi_systab.tables = systab32->tables;
377
378 early_memunmap(systab32, sizeof(*systab32));
379 }
380
381 efi.systab = &efi_systab;
382
383
384
385
386 if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
387 pr_err("System table signature incorrect!\n");
388 return -EINVAL;
389 }
390 if ((efi.systab->hdr.revision >> 16) == 0)
391 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
392 efi.systab->hdr.revision >> 16,
393 efi.systab->hdr.revision & 0xffff);
394
395 return 0;
396}
397
398static int __init efi_runtime_init32(void)
399{
400 efi_runtime_services_32_t *runtime;
401
402 runtime = early_memremap((unsigned long)efi.systab->runtime,
403 sizeof(efi_runtime_services_32_t));
404 if (!runtime) {
405 pr_err("Could not map the runtime service table!\n");
406 return -ENOMEM;
407 }
408
409
410
411
412
413
414 efi_phys.set_virtual_address_map =
415 (efi_set_virtual_address_map_t *)
416 (unsigned long)runtime->set_virtual_address_map;
417 early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
418
419 return 0;
420}
421
422static int __init efi_runtime_init64(void)
423{
424 efi_runtime_services_64_t *runtime;
425
426 runtime = early_memremap((unsigned long)efi.systab->runtime,
427 sizeof(efi_runtime_services_64_t));
428 if (!runtime) {
429 pr_err("Could not map the runtime service table!\n");
430 return -ENOMEM;
431 }
432
433
434
435
436
437
438 efi_phys.set_virtual_address_map =
439 (efi_set_virtual_address_map_t *)
440 (unsigned long)runtime->set_virtual_address_map;
441 early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
442
443 return 0;
444}
445
446static int __init efi_runtime_init(void)
447{
448 int rv;
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464 if (!efi_enabled(EFI_PARAVIRT)) {
465 if (efi_enabled(EFI_64BIT))
466 rv = efi_runtime_init64();
467 else
468 rv = efi_runtime_init32();
469
470 if (rv)
471 return rv;
472 }
473
474 set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
475
476 return 0;
477}
478
479void __init efi_init(void)
480{
481 efi_char16_t *c16;
482 char vendor[100] = "unknown";
483 int i = 0;
484 void *tmp;
485
486#ifdef CONFIG_X86_32
487 if (boot_params.efi_info.efi_systab_hi ||
488 boot_params.efi_info.efi_memmap_hi) {
489 pr_info("Table located above 4GB, disabling EFI.\n");
490 return;
491 }
492 efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
493#else
494 efi_phys.systab = (efi_system_table_t *)
495 (boot_params.efi_info.efi_systab |
496 ((__u64)boot_params.efi_info.efi_systab_hi<<32));
497#endif
498
499 if (efi_systab_init(efi_phys.systab))
500 return;
501
502 efi.config_table = (unsigned long)efi.systab->tables;
503 efi.fw_vendor = (unsigned long)efi.systab->fw_vendor;
504 efi.runtime = (unsigned long)efi.systab->runtime;
505
506
507
508
509 c16 = tmp = early_memremap(efi.systab->fw_vendor, 2);
510 if (c16) {
511 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i)
512 vendor[i] = *c16++;
513 vendor[i] = '\0';
514 } else
515 pr_err("Could not map the firmware vendor!\n");
516 early_memunmap(tmp, 2);
517
518 pr_info("EFI v%u.%.02u by %s\n",
519 efi.systab->hdr.revision >> 16,
520 efi.systab->hdr.revision & 0xffff, vendor);
521
522 if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
523 return;
524
525 if (efi_config_init(arch_tables))
526 return;
527
528
529
530
531
532
533 if (!efi_runtime_supported())
534 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
535 else {
536 if (efi_runtime_disabled() || efi_runtime_init()) {
537 efi_memmap_unmap();
538 return;
539 }
540 }
541
542 efi_clean_memmap();
543
544 if (efi_enabled(EFI_DBG))
545 efi_print_memmap();
546}
547
548void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
549{
550 u64 addr, npages;
551
552 addr = md->virt_addr;
553 npages = md->num_pages;
554
555 memrange_efi_to_native(&addr, &npages);
556
557 if (executable)
558 set_memory_x(addr, npages);
559 else
560 set_memory_nx(addr, npages);
561}
562
563void __init runtime_code_page_mkexec(void)
564{
565 efi_memory_desc_t *md;
566
567
568 for_each_efi_memory_desc(md) {
569 if (md->type != EFI_RUNTIME_SERVICES_CODE)
570 continue;
571
572 efi_set_executable(md, true);
573 }
574}
575
576void __init efi_memory_uc(u64 addr, unsigned long size)
577{
578 unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
579 u64 npages;
580
581 npages = round_up(size, page_shift) / page_shift;
582 memrange_efi_to_native(&addr, &npages);
583 set_memory_uc(addr, npages);
584}
585
586void __init old_map_region(efi_memory_desc_t *md)
587{
588 u64 start_pfn, end_pfn, end;
589 unsigned long size;
590 void *va;
591
592 start_pfn = PFN_DOWN(md->phys_addr);
593 size = md->num_pages << PAGE_SHIFT;
594 end = md->phys_addr + size;
595 end_pfn = PFN_UP(end);
596
597 if (pfn_range_is_mapped(start_pfn, end_pfn)) {
598 va = __va(md->phys_addr);
599
600 if (!(md->attribute & EFI_MEMORY_WB))
601 efi_memory_uc((u64)(unsigned long)va, size);
602 } else
603 va = efi_ioremap(md->phys_addr, size,
604 md->type, md->attribute);
605
606 md->virt_addr = (u64) (unsigned long) va;
607 if (!va)
608 pr_err("ioremap of 0x%llX failed!\n",
609 (unsigned long long)md->phys_addr);
610}
611
612
613static void __init efi_merge_regions(void)
614{
615 efi_memory_desc_t *md, *prev_md = NULL;
616
617 for_each_efi_memory_desc(md) {
618 u64 prev_size;
619
620 if (!prev_md) {
621 prev_md = md;
622 continue;
623 }
624
625 if (prev_md->type != md->type ||
626 prev_md->attribute != md->attribute) {
627 prev_md = md;
628 continue;
629 }
630
631 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
632
633 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
634 prev_md->num_pages += md->num_pages;
635 md->type = EFI_RESERVED_TYPE;
636 md->attribute = 0;
637 continue;
638 }
639 prev_md = md;
640 }
641}
642
643static void __init get_systab_virt_addr(efi_memory_desc_t *md)
644{
645 unsigned long size;
646 u64 end, systab;
647
648 size = md->num_pages << EFI_PAGE_SHIFT;
649 end = md->phys_addr + size;
650 systab = (u64)(unsigned long)efi_phys.systab;
651 if (md->phys_addr <= systab && systab < end) {
652 systab += md->virt_addr - md->phys_addr;
653 efi.systab = (efi_system_table_t *)(unsigned long)systab;
654 }
655}
656
657static void *realloc_pages(void *old_memmap, int old_shift)
658{
659 void *ret;
660
661 ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
662 if (!ret)
663 goto out;
664
665
666
667
668 if (!old_memmap)
669 return ret;
670
671 memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
672
673out:
674 free_pages((unsigned long)old_memmap, old_shift);
675 return ret;
676}
677
678
679
680
681
682
683
684static inline void *efi_map_next_entry_reverse(void *entry)
685{
686
687 if (!entry)
688 return efi.memmap.map_end - efi.memmap.desc_size;
689
690 entry -= efi.memmap.desc_size;
691 if (entry < efi.memmap.map)
692 return NULL;
693
694 return entry;
695}
696
697
698
699
700
701
702
703
704
705
706
707
708static void *efi_map_next_entry(void *entry)
709{
710 if (!efi_enabled(EFI_OLD_MEMMAP) && efi_enabled(EFI_64BIT)) {
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728 return efi_map_next_entry_reverse(entry);
729 }
730
731
732 if (!entry)
733 return efi.memmap.map;
734
735 entry += efi.memmap.desc_size;
736 if (entry >= efi.memmap.map_end)
737 return NULL;
738
739 return entry;
740}
741
742static bool should_map_region(efi_memory_desc_t *md)
743{
744
745
746
747 if (md->attribute & EFI_MEMORY_RUNTIME)
748 return true;
749
750
751
752
753
754
755 if (IS_ENABLED(CONFIG_X86_32))
756 return false;
757
758
759
760
761
762 if (IS_ENABLED(CONFIG_EFI_MIXED) && !efi_is_native()) {
763 if (md->type == EFI_CONVENTIONAL_MEMORY ||
764 md->type == EFI_LOADER_DATA ||
765 md->type == EFI_LOADER_CODE)
766 return true;
767 }
768
769
770
771
772
773
774
775 if (md->type == EFI_BOOT_SERVICES_CODE ||
776 md->type == EFI_BOOT_SERVICES_DATA)
777 return true;
778
779 return false;
780}
781
782
783
784
785
786static void * __init efi_map_regions(int *count, int *pg_shift)
787{
788 void *p, *new_memmap = NULL;
789 unsigned long left = 0;
790 unsigned long desc_size;
791 efi_memory_desc_t *md;
792
793 desc_size = efi.memmap.desc_size;
794
795 p = NULL;
796 while ((p = efi_map_next_entry(p))) {
797 md = p;
798
799 if (!should_map_region(md))
800 continue;
801
802 efi_map_region(md);
803 get_systab_virt_addr(md);
804
805 if (left < desc_size) {
806 new_memmap = realloc_pages(new_memmap, *pg_shift);
807 if (!new_memmap)
808 return NULL;
809
810 left += PAGE_SIZE << *pg_shift;
811 (*pg_shift)++;
812 }
813
814 memcpy(new_memmap + (*count * desc_size), md, desc_size);
815
816 left -= desc_size;
817 (*count)++;
818 }
819
820 return new_memmap;
821}
822
823static void __init kexec_enter_virtual_mode(void)
824{
825#ifdef CONFIG_KEXEC_CORE
826 efi_memory_desc_t *md;
827 unsigned int num_pages;
828
829 efi.systab = NULL;
830
831
832
833
834
835
836
837 if (!efi_is_native() || efi_enabled(EFI_OLD_MEMMAP)) {
838 efi_memmap_unmap();
839 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
840 return;
841 }
842
843 if (efi_alloc_page_tables()) {
844 pr_err("Failed to allocate EFI page tables\n");
845 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
846 return;
847 }
848
849
850
851
852
853 for_each_efi_memory_desc(md) {
854 efi_map_region_fixed(md);
855 get_systab_virt_addr(md);
856 }
857
858
859
860
861
862 efi_memmap_unmap();
863
864 if (efi_memmap_init_late(efi.memmap.phys_map,
865 efi.memmap.desc_size * efi.memmap.nr_map)) {
866 pr_err("Failed to remap late EFI memory map\n");
867 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
868 return;
869 }
870
871 BUG_ON(!efi.systab);
872
873 num_pages = ALIGN(efi.memmap.nr_map * efi.memmap.desc_size, PAGE_SIZE);
874 num_pages >>= PAGE_SHIFT;
875
876 if (efi_setup_page_tables(efi.memmap.phys_map, num_pages)) {
877 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
878 return;
879 }
880
881 efi_sync_low_kernel_mappings();
882
883
884
885
886
887
888
889 efi.runtime_version = efi_systab.hdr.revision;
890
891 efi_native_runtime_setup();
892
893 efi.set_virtual_address_map = NULL;
894
895 if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
896 runtime_code_page_mkexec();
897
898
899 efi_delete_dummy_variable();
900#endif
901}
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925static void __init __efi_enter_virtual_mode(void)
926{
927 int count = 0, pg_shift = 0;
928 void *new_memmap = NULL;
929 efi_status_t status;
930 unsigned long pa;
931
932 efi.systab = NULL;
933
934 if (efi_alloc_page_tables()) {
935 pr_err("Failed to allocate EFI page tables\n");
936 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
937 return;
938 }
939
940 efi_merge_regions();
941 new_memmap = efi_map_regions(&count, &pg_shift);
942 if (!new_memmap) {
943 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
944 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
945 return;
946 }
947
948 pa = __pa(new_memmap);
949
950
951
952
953
954
955 efi_memmap_unmap();
956
957 if (efi_memmap_init_late(pa, efi.memmap.desc_size * count)) {
958 pr_err("Failed to remap late EFI memory map\n");
959 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
960 return;
961 }
962
963 if (efi_enabled(EFI_DBG)) {
964 pr_info("EFI runtime memory map:\n");
965 efi_print_memmap();
966 }
967
968 BUG_ON(!efi.systab);
969
970 if (efi_setup_page_tables(pa, 1 << pg_shift)) {
971 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
972 return;
973 }
974
975 efi_sync_low_kernel_mappings();
976
977 if (efi_is_native()) {
978 status = phys_efi_set_virtual_address_map(
979 efi.memmap.desc_size * count,
980 efi.memmap.desc_size,
981 efi.memmap.desc_version,
982 (efi_memory_desc_t *)pa);
983 } else {
984 status = efi_thunk_set_virtual_address_map(
985 efi_phys.set_virtual_address_map,
986 efi.memmap.desc_size * count,
987 efi.memmap.desc_size,
988 efi.memmap.desc_version,
989 (efi_memory_desc_t *)pa);
990 }
991
992 if (status != EFI_SUCCESS) {
993 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
994 status);
995 panic("EFI call to SetVirtualAddressMap() failed!");
996 }
997
998 efi_free_boot_services();
999
1000
1001
1002
1003
1004
1005
1006 efi.runtime_version = efi_systab.hdr.revision;
1007
1008 if (efi_is_native())
1009 efi_native_runtime_setup();
1010 else
1011 efi_thunk_runtime_setup();
1012
1013 efi.set_virtual_address_map = NULL;
1014
1015
1016
1017
1018
1019
1020 efi_runtime_update_mappings();
1021
1022
1023 efi_delete_dummy_variable();
1024}
1025
1026void __init efi_enter_virtual_mode(void)
1027{
1028 if (efi_enabled(EFI_PARAVIRT))
1029 return;
1030
1031 if (efi_setup)
1032 kexec_enter_virtual_mode();
1033 else
1034 __efi_enter_virtual_mode();
1035
1036 efi_dump_pagetable();
1037}
1038
1039static int __init arch_parse_efi_cmdline(char *str)
1040{
1041 if (!str) {
1042 pr_warn("need at least one option\n");
1043 return -EINVAL;
1044 }
1045
1046 if (parse_option_str(str, "old_map"))
1047 set_bit(EFI_OLD_MEMMAP, &efi.flags);
1048
1049 return 0;
1050}
1051early_param("efi", arch_parse_efi_cmdline);
1052