1
2
3
4
5
6
7
8
9
10#include <linux/memblock.h>
11#include <linux/init.h>
12#include <linux/io.h>
13#include <linux/ioport.h>
14#include <linux/slab.h>
15#include <linux/vmalloc.h>
16#include <linux/mmiotrace.h>
17#include <linux/mem_encrypt.h>
18#include <linux/efi.h>
19
20#include <asm/set_memory.h>
21#include <asm/e820/api.h>
22#include <asm/fixmap.h>
23#include <asm/pgtable.h>
24#include <asm/tlbflush.h>
25#include <asm/pgalloc.h>
26#include <asm/pat.h>
27#include <asm/setup.h>
28
29#include "physaddr.h"
30
31
32
33
34struct ioremap_desc {
35 unsigned int flags;
36};
37
38
39
40
41
42int ioremap_change_attr(unsigned long vaddr, unsigned long size,
43 enum page_cache_mode pcm)
44{
45 unsigned long nrpages = size >> PAGE_SHIFT;
46 int err;
47
48 switch (pcm) {
49 case _PAGE_CACHE_MODE_UC:
50 default:
51 err = _set_memory_uc(vaddr, nrpages);
52 break;
53 case _PAGE_CACHE_MODE_WC:
54 err = _set_memory_wc(vaddr, nrpages);
55 break;
56 case _PAGE_CACHE_MODE_WT:
57 err = _set_memory_wt(vaddr, nrpages);
58 break;
59 case _PAGE_CACHE_MODE_WB:
60 err = _set_memory_wb(vaddr, nrpages);
61 break;
62 }
63
64 return err;
65}
66
67
68static unsigned int __ioremap_check_ram(struct resource *res)
69{
70 unsigned long start_pfn, stop_pfn;
71 unsigned long i;
72
73 if ((res->flags & IORESOURCE_SYSTEM_RAM) != IORESOURCE_SYSTEM_RAM)
74 return 0;
75
76 start_pfn = (res->start + PAGE_SIZE - 1) >> PAGE_SHIFT;
77 stop_pfn = (res->end + 1) >> PAGE_SHIFT;
78 if (stop_pfn > start_pfn) {
79 for (i = 0; i < (stop_pfn - start_pfn); ++i)
80 if (pfn_valid(start_pfn + i) &&
81 !PageReserved(pfn_to_page(start_pfn + i)))
82 return IORES_MAP_SYSTEM_RAM;
83 }
84
85 return 0;
86}
87
88
89
90
91
92static unsigned int __ioremap_check_encrypted(struct resource *res)
93{
94 if (!sev_active())
95 return 0;
96
97 switch (res->desc) {
98 case IORES_DESC_NONE:
99 case IORES_DESC_RESERVED:
100 break;
101 default:
102 return IORES_MAP_ENCRYPTED;
103 }
104
105 return 0;
106}
107
108static int __ioremap_collect_map_flags(struct resource *res, void *arg)
109{
110 struct ioremap_desc *desc = arg;
111
112 if (!(desc->flags & IORES_MAP_SYSTEM_RAM))
113 desc->flags |= __ioremap_check_ram(res);
114
115 if (!(desc->flags & IORES_MAP_ENCRYPTED))
116 desc->flags |= __ioremap_check_encrypted(res);
117
118 return ((desc->flags & (IORES_MAP_SYSTEM_RAM | IORES_MAP_ENCRYPTED)) ==
119 (IORES_MAP_SYSTEM_RAM | IORES_MAP_ENCRYPTED));
120}
121
122
123
124
125
126
127static void __ioremap_check_mem(resource_size_t addr, unsigned long size,
128 struct ioremap_desc *desc)
129{
130 u64 start, end;
131
132 start = (u64)addr;
133 end = start + size - 1;
134 memset(desc, 0, sizeof(struct ioremap_desc));
135
136 walk_mem_res(start, end, desc, __ioremap_collect_map_flags);
137}
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153static void __iomem *
154__ioremap_caller(resource_size_t phys_addr, unsigned long size,
155 enum page_cache_mode pcm, void *caller, bool encrypted)
156{
157 unsigned long offset, vaddr;
158 resource_size_t last_addr;
159 const resource_size_t unaligned_phys_addr = phys_addr;
160 const unsigned long unaligned_size = size;
161 struct ioremap_desc io_desc;
162 struct vm_struct *area;
163 enum page_cache_mode new_pcm;
164 pgprot_t prot;
165 int retval;
166 void __iomem *ret_addr;
167
168
169 last_addr = phys_addr + size - 1;
170 if (!size || last_addr < phys_addr)
171 return NULL;
172
173 if (!phys_addr_valid(phys_addr)) {
174 printk(KERN_WARNING "ioremap: invalid physical address %llx\n",
175 (unsigned long long)phys_addr);
176 WARN_ON_ONCE(1);
177 return NULL;
178 }
179
180 __ioremap_check_mem(phys_addr, size, &io_desc);
181
182
183
184
185 if (io_desc.flags & IORES_MAP_SYSTEM_RAM) {
186 WARN_ONCE(1, "ioremap on RAM at %pa - %pa\n",
187 &phys_addr, &last_addr);
188 return NULL;
189 }
190
191
192
193
194 offset = phys_addr & ~PAGE_MASK;
195 phys_addr &= PHYSICAL_PAGE_MASK;
196 size = PAGE_ALIGN(last_addr+1) - phys_addr;
197
198 retval = reserve_memtype(phys_addr, (u64)phys_addr + size,
199 pcm, &new_pcm);
200 if (retval) {
201 printk(KERN_ERR "ioremap reserve_memtype failed %d\n", retval);
202 return NULL;
203 }
204
205 if (pcm != new_pcm) {
206 if (!is_new_memtype_allowed(phys_addr, size, pcm, new_pcm)) {
207 printk(KERN_ERR
208 "ioremap error for 0x%llx-0x%llx, requested 0x%x, got 0x%x\n",
209 (unsigned long long)phys_addr,
210 (unsigned long long)(phys_addr + size),
211 pcm, new_pcm);
212 goto err_free_memtype;
213 }
214 pcm = new_pcm;
215 }
216
217
218
219
220
221
222 prot = PAGE_KERNEL_IO;
223 if ((io_desc.flags & IORES_MAP_ENCRYPTED) || encrypted)
224 prot = pgprot_encrypted(prot);
225
226 switch (pcm) {
227 case _PAGE_CACHE_MODE_UC:
228 default:
229 prot = __pgprot(pgprot_val(prot) |
230 cachemode2protval(_PAGE_CACHE_MODE_UC));
231 break;
232 case _PAGE_CACHE_MODE_UC_MINUS:
233 prot = __pgprot(pgprot_val(prot) |
234 cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS));
235 break;
236 case _PAGE_CACHE_MODE_WC:
237 prot = __pgprot(pgprot_val(prot) |
238 cachemode2protval(_PAGE_CACHE_MODE_WC));
239 break;
240 case _PAGE_CACHE_MODE_WT:
241 prot = __pgprot(pgprot_val(prot) |
242 cachemode2protval(_PAGE_CACHE_MODE_WT));
243 break;
244 case _PAGE_CACHE_MODE_WB:
245 break;
246 }
247
248
249
250
251 area = get_vm_area_caller(size, VM_IOREMAP, caller);
252 if (!area)
253 goto err_free_memtype;
254 area->phys_addr = phys_addr;
255 vaddr = (unsigned long) area->addr;
256
257 if (kernel_map_sync_memtype(phys_addr, size, pcm))
258 goto err_free_area;
259
260 if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot))
261 goto err_free_area;
262
263 ret_addr = (void __iomem *) (vaddr + offset);
264 mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr);
265
266
267
268
269
270 if (iomem_map_sanity_check(unaligned_phys_addr, unaligned_size))
271 pr_warn("caller %pS mapping multiple BARs\n", caller);
272
273 return ret_addr;
274err_free_area:
275 free_vm_area(area);
276err_free_memtype:
277 free_memtype(phys_addr, phys_addr + size);
278 return NULL;
279}
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size)
303{
304
305
306
307
308
309
310
311
312 enum page_cache_mode pcm = _PAGE_CACHE_MODE_UC_MINUS;
313
314 return __ioremap_caller(phys_addr, size, pcm,
315 __builtin_return_address(0), false);
316}
317EXPORT_SYMBOL(ioremap_nocache);
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343void __iomem *ioremap_uc(resource_size_t phys_addr, unsigned long size)
344{
345 enum page_cache_mode pcm = _PAGE_CACHE_MODE_UC;
346
347 return __ioremap_caller(phys_addr, size, pcm,
348 __builtin_return_address(0), false);
349}
350EXPORT_SYMBOL_GPL(ioremap_uc);
351
352
353
354
355
356
357
358
359
360
361
362void __iomem *ioremap_wc(resource_size_t phys_addr, unsigned long size)
363{
364 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WC,
365 __builtin_return_address(0), false);
366}
367EXPORT_SYMBOL(ioremap_wc);
368
369
370
371
372
373
374
375
376
377
378
379void __iomem *ioremap_wt(resource_size_t phys_addr, unsigned long size)
380{
381 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WT,
382 __builtin_return_address(0), false);
383}
384EXPORT_SYMBOL(ioremap_wt);
385
386void __iomem *ioremap_encrypted(resource_size_t phys_addr, unsigned long size)
387{
388 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WB,
389 __builtin_return_address(0), true);
390}
391EXPORT_SYMBOL(ioremap_encrypted);
392
393void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
394{
395 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WB,
396 __builtin_return_address(0), false);
397}
398EXPORT_SYMBOL(ioremap_cache);
399
400void __iomem *ioremap_prot(resource_size_t phys_addr, unsigned long size,
401 unsigned long prot_val)
402{
403 return __ioremap_caller(phys_addr, size,
404 pgprot2cachemode(__pgprot(prot_val)),
405 __builtin_return_address(0), false);
406}
407EXPORT_SYMBOL(ioremap_prot);
408
409
410
411
412
413
414
415void iounmap(volatile void __iomem *addr)
416{
417 struct vm_struct *p, *o;
418
419 if ((void __force *)addr <= high_memory)
420 return;
421
422
423
424
425
426
427
428
429 if ((void __force *)addr >= phys_to_virt(ISA_START_ADDRESS) &&
430 (void __force *)addr < phys_to_virt(ISA_END_ADDRESS)) {
431 WARN(1, "iounmap() called for ISA range not obtained using ioremap()\n");
432 return;
433 }
434
435 mmiotrace_iounmap(addr);
436
437 addr = (volatile void __iomem *)
438 (PAGE_MASK & (unsigned long __force)addr);
439
440
441
442
443
444
445 p = find_vm_area((void __force *)addr);
446
447 if (!p) {
448 printk(KERN_ERR "iounmap: bad address %p\n", addr);
449 dump_stack();
450 return;
451 }
452
453 free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p));
454
455
456 o = remove_vm_area((void __force *)addr);
457 BUG_ON(p != o || o == NULL);
458 kfree(p);
459}
460EXPORT_SYMBOL(iounmap);
461
462int __init arch_ioremap_p4d_supported(void)
463{
464 return 0;
465}
466
467int __init arch_ioremap_pud_supported(void)
468{
469#ifdef CONFIG_X86_64
470 return boot_cpu_has(X86_FEATURE_GBPAGES);
471#else
472 return 0;
473#endif
474}
475
476int __init arch_ioremap_pmd_supported(void)
477{
478 return boot_cpu_has(X86_FEATURE_PSE);
479}
480
481
482
483
484
485void *xlate_dev_mem_ptr(phys_addr_t phys)
486{
487 unsigned long start = phys & PAGE_MASK;
488 unsigned long offset = phys & ~PAGE_MASK;
489 void *vaddr;
490
491
492 vaddr = memremap(start, PAGE_SIZE, MEMREMAP_WB);
493
494
495 if (vaddr)
496 vaddr += offset;
497
498 return vaddr;
499}
500
501void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr)
502{
503 memunmap((void *)((unsigned long)addr & PAGE_MASK));
504}
505
506
507
508
509
510
511
512
513
514
515
516
517static bool memremap_should_map_decrypted(resource_size_t phys_addr,
518 unsigned long size)
519{
520 int is_pmem;
521
522
523
524
525
526 is_pmem = region_intersects(phys_addr, size, IORESOURCE_MEM,
527 IORES_DESC_PERSISTENT_MEMORY);
528 if (is_pmem != REGION_DISJOINT)
529 return true;
530
531
532
533
534
535 if (efi_enabled(EFI_BOOT)) {
536 switch (efi_mem_type(phys_addr)) {
537 case EFI_RESERVED_TYPE:
538 if (efi_mem_attributes(phys_addr) & EFI_MEMORY_NV)
539 return true;
540 break;
541 default:
542 break;
543 }
544 }
545
546
547 switch (e820__get_entry_type(phys_addr, phys_addr + size - 1)) {
548 case E820_TYPE_RESERVED:
549 case E820_TYPE_ACPI:
550 case E820_TYPE_NVS:
551 case E820_TYPE_UNUSABLE:
552
553 if (sev_active())
554 break;
555
556
557 case E820_TYPE_PRAM:
558 return true;
559 default:
560 break;
561 }
562
563 return false;
564}
565
566
567
568
569
570static bool memremap_is_efi_data(resource_size_t phys_addr,
571 unsigned long size)
572{
573 u64 paddr;
574
575
576 if (!efi_enabled(EFI_BOOT))
577 return false;
578
579 paddr = boot_params.efi_info.efi_memmap_hi;
580 paddr <<= 32;
581 paddr |= boot_params.efi_info.efi_memmap;
582 if (phys_addr == paddr)
583 return true;
584
585 paddr = boot_params.efi_info.efi_systab_hi;
586 paddr <<= 32;
587 paddr |= boot_params.efi_info.efi_systab;
588 if (phys_addr == paddr)
589 return true;
590
591 if (efi_is_table_address(phys_addr))
592 return true;
593
594 switch (efi_mem_type(phys_addr)) {
595 case EFI_BOOT_SERVICES_DATA:
596 case EFI_RUNTIME_SERVICES_DATA:
597 return true;
598 default:
599 break;
600 }
601
602 return false;
603}
604
605
606
607
608
609static bool memremap_is_setup_data(resource_size_t phys_addr,
610 unsigned long size)
611{
612 struct setup_data *data;
613 u64 paddr, paddr_next;
614
615 paddr = boot_params.hdr.setup_data;
616 while (paddr) {
617 unsigned int len;
618
619 if (phys_addr == paddr)
620 return true;
621
622 data = memremap(paddr, sizeof(*data),
623 MEMREMAP_WB | MEMREMAP_DEC);
624
625 paddr_next = data->next;
626 len = data->len;
627
628 memunmap(data);
629
630 if ((phys_addr > paddr) && (phys_addr < (paddr + len)))
631 return true;
632
633 paddr = paddr_next;
634 }
635
636 return false;
637}
638
639
640
641
642
643static bool __init early_memremap_is_setup_data(resource_size_t phys_addr,
644 unsigned long size)
645{
646 struct setup_data *data;
647 u64 paddr, paddr_next;
648
649 paddr = boot_params.hdr.setup_data;
650 while (paddr) {
651 unsigned int len;
652
653 if (phys_addr == paddr)
654 return true;
655
656 data = early_memremap_decrypted(paddr, sizeof(*data));
657
658 paddr_next = data->next;
659 len = data->len;
660
661 early_memunmap(data, sizeof(*data));
662
663 if ((phys_addr > paddr) && (phys_addr < (paddr + len)))
664 return true;
665
666 paddr = paddr_next;
667 }
668
669 return false;
670}
671
672
673
674
675
676
677bool arch_memremap_can_ram_remap(resource_size_t phys_addr, unsigned long size,
678 unsigned long flags)
679{
680 if (!mem_encrypt_active())
681 return true;
682
683 if (flags & MEMREMAP_ENC)
684 return true;
685
686 if (flags & MEMREMAP_DEC)
687 return false;
688
689 if (sme_active()) {
690 if (memremap_is_setup_data(phys_addr, size) ||
691 memremap_is_efi_data(phys_addr, size))
692 return false;
693 }
694
695 return !memremap_should_map_decrypted(phys_addr, size);
696}
697
698
699
700
701
702
703
704pgprot_t __init early_memremap_pgprot_adjust(resource_size_t phys_addr,
705 unsigned long size,
706 pgprot_t prot)
707{
708 bool encrypted_prot;
709
710 if (!mem_encrypt_active())
711 return prot;
712
713 encrypted_prot = true;
714
715 if (sme_active()) {
716 if (early_memremap_is_setup_data(phys_addr, size) ||
717 memremap_is_efi_data(phys_addr, size))
718 encrypted_prot = false;
719 }
720
721 if (encrypted_prot && memremap_should_map_decrypted(phys_addr, size))
722 encrypted_prot = false;
723
724 return encrypted_prot ? pgprot_encrypted(prot)
725 : pgprot_decrypted(prot);
726}
727
728bool phys_mem_access_encrypted(unsigned long phys_addr, unsigned long size)
729{
730 return arch_memremap_can_ram_remap(phys_addr, size, 0);
731}
732
733#ifdef CONFIG_AMD_MEM_ENCRYPT
734
735void __init *early_memremap_encrypted(resource_size_t phys_addr,
736 unsigned long size)
737{
738 return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_ENC);
739}
740
741
742
743
744
745void __init *early_memremap_encrypted_wp(resource_size_t phys_addr,
746 unsigned long size)
747{
748
749 if (__pte2cachemode_tbl[_PAGE_CACHE_MODE_WP] != _PAGE_CACHE_MODE_WP)
750 return NULL;
751
752 return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_ENC_WP);
753}
754
755
756void __init *early_memremap_decrypted(resource_size_t phys_addr,
757 unsigned long size)
758{
759 return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_NOENC);
760}
761
762
763
764
765
766void __init *early_memremap_decrypted_wp(resource_size_t phys_addr,
767 unsigned long size)
768{
769
770 if (__pte2cachemode_tbl[_PAGE_CACHE_MODE_WP] != _PAGE_CACHE_MODE_WP)
771 return NULL;
772
773 return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_NOENC_WP);
774}
775#endif
776
777static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __page_aligned_bss;
778
779static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
780{
781
782 pgd_t *base = __va(read_cr3_pa());
783 pgd_t *pgd = &base[pgd_index(addr)];
784 p4d_t *p4d = p4d_offset(pgd, addr);
785 pud_t *pud = pud_offset(p4d, addr);
786 pmd_t *pmd = pmd_offset(pud, addr);
787
788 return pmd;
789}
790
791static inline pte_t * __init early_ioremap_pte(unsigned long addr)
792{
793 return &bm_pte[pte_index(addr)];
794}
795
796bool __init is_early_ioremap_ptep(pte_t *ptep)
797{
798 return ptep >= &bm_pte[0] && ptep < &bm_pte[PAGE_SIZE/sizeof(pte_t)];
799}
800
801void __init early_ioremap_init(void)
802{
803 pmd_t *pmd;
804
805#ifdef CONFIG_X86_64
806 BUILD_BUG_ON((fix_to_virt(0) + PAGE_SIZE) & ((1 << PMD_SHIFT) - 1));
807#else
808 WARN_ON((fix_to_virt(0) + PAGE_SIZE) & ((1 << PMD_SHIFT) - 1));
809#endif
810
811 early_ioremap_setup();
812
813 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
814 memset(bm_pte, 0, sizeof(bm_pte));
815 pmd_populate_kernel(&init_mm, pmd, bm_pte);
816
817
818
819
820
821#define __FIXADDR_TOP (-PAGE_SIZE)
822 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
823 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
824#undef __FIXADDR_TOP
825 if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
826 WARN_ON(1);
827 printk(KERN_WARNING "pmd %p != %p\n",
828 pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
829 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
830 fix_to_virt(FIX_BTMAP_BEGIN));
831 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END): %08lx\n",
832 fix_to_virt(FIX_BTMAP_END));
833
834 printk(KERN_WARNING "FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
835 printk(KERN_WARNING "FIX_BTMAP_BEGIN: %d\n",
836 FIX_BTMAP_BEGIN);
837 }
838}
839
840void __init __early_set_fixmap(enum fixed_addresses idx,
841 phys_addr_t phys, pgprot_t flags)
842{
843 unsigned long addr = __fix_to_virt(idx);
844 pte_t *pte;
845
846 if (idx >= __end_of_fixed_addresses) {
847 BUG();
848 return;
849 }
850 pte = early_ioremap_pte(addr);
851
852
853 pgprot_val(flags) &= __supported_pte_mask;
854
855 if (pgprot_val(flags))
856 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
857 else
858 pte_clear(&init_mm, addr, pte);
859 __flush_tlb_one_kernel(addr);
860}
861