1
2
3
4
5
6
7
8
9#include <linux/bootmem.h>
10#include <linux/init.h>
11#include <linux/io.h>
12#include <linux/module.h>
13#include <linux/slab.h>
14#include <linux/vmalloc.h>
15#include <linux/mmiotrace.h>
16
17#include <asm/cacheflush.h>
18#include <asm/e820.h>
19#include <asm/fixmap.h>
20#include <asm/pgtable.h>
21#include <asm/tlbflush.h>
22#include <asm/pgalloc.h>
23#include <asm/pat.h>
24
25#include "physaddr.h"
26
27int page_is_ram(unsigned long pagenr)
28{
29 resource_size_t addr, end;
30 int i;
31
32
33
34
35
36
37 if (pagenr == 0)
38 return 0;
39
40
41
42
43
44 if (pagenr >= (BIOS_BEGIN >> PAGE_SHIFT) &&
45 pagenr < (BIOS_END >> PAGE_SHIFT))
46 return 0;
47
48 for (i = 0; i < e820.nr_map; i++) {
49
50
51
52 if (e820.map[i].type != E820_RAM)
53 continue;
54 addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT;
55 end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT;
56
57
58 if ((pagenr >= addr) && (pagenr < end))
59 return 1;
60 }
61 return 0;
62}
63
64
65
66
67
68int ioremap_change_attr(unsigned long vaddr, unsigned long size,
69 unsigned long prot_val)
70{
71 unsigned long nrpages = size >> PAGE_SHIFT;
72 int err;
73
74 switch (prot_val) {
75 case _PAGE_CACHE_UC:
76 default:
77 err = _set_memory_uc(vaddr, nrpages);
78 break;
79 case _PAGE_CACHE_WC:
80 err = _set_memory_wc(vaddr, nrpages);
81 break;
82 case _PAGE_CACHE_WB:
83 err = _set_memory_wb(vaddr, nrpages);
84 break;
85 }
86
87 return err;
88}
89
90
91
92
93
94
95
96
97
98
99static void __iomem *__ioremap_caller(resource_size_t phys_addr,
100 unsigned long size, unsigned long prot_val, void *caller)
101{
102 unsigned long pfn, offset, vaddr;
103 resource_size_t last_addr;
104 const resource_size_t unaligned_phys_addr = phys_addr;
105 const unsigned long unaligned_size = size;
106 struct vm_struct *area;
107 unsigned long new_prot_val;
108 pgprot_t prot;
109 int retval;
110 void __iomem *ret_addr;
111
112
113 last_addr = phys_addr + size - 1;
114 if (!size || last_addr < phys_addr)
115 return NULL;
116
117 if (!phys_addr_valid(phys_addr)) {
118 printk(KERN_WARNING "ioremap: invalid physical address %llx\n",
119 (unsigned long long)phys_addr);
120 WARN_ON_ONCE(1);
121 return NULL;
122 }
123
124
125
126
127 if (is_ISA_range(phys_addr, last_addr))
128 return (__force void __iomem *)phys_to_virt(phys_addr);
129
130
131
132
133
134 WARN_ONCE(iomem_map_sanity_check(phys_addr, size),
135 KERN_INFO "Info: mapping multiple BARs. Your kernel is fine.");
136
137
138
139
140 for (pfn = phys_addr >> PAGE_SHIFT;
141 (pfn << PAGE_SHIFT) < (last_addr & PAGE_MASK);
142 pfn++) {
143
144 int is_ram = page_is_ram(pfn);
145
146 if (is_ram && pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn)))
147 return NULL;
148 WARN_ON_ONCE(is_ram);
149 }
150
151
152
153
154 offset = phys_addr & ~PAGE_MASK;
155 phys_addr &= PAGE_MASK;
156 size = PAGE_ALIGN(last_addr+1) - phys_addr;
157
158 retval = reserve_memtype(phys_addr, (u64)phys_addr + size,
159 prot_val, &new_prot_val);
160 if (retval) {
161 printk(KERN_ERR "ioremap reserve_memtype failed %d\n", retval);
162 return NULL;
163 }
164
165 if (prot_val != new_prot_val) {
166 if (!is_new_memtype_allowed(phys_addr, size,
167 prot_val, new_prot_val)) {
168 printk(KERN_ERR
169 "ioremap error for 0x%llx-0x%llx, requested 0x%lx, got 0x%lx\n",
170 (unsigned long long)phys_addr,
171 (unsigned long long)(phys_addr + size),
172 prot_val, new_prot_val);
173 goto err_free_memtype;
174 }
175 prot_val = new_prot_val;
176 }
177
178 switch (prot_val) {
179 case _PAGE_CACHE_UC:
180 default:
181 prot = PAGE_KERNEL_IO_NOCACHE;
182 break;
183 case _PAGE_CACHE_UC_MINUS:
184 prot = PAGE_KERNEL_IO_UC_MINUS;
185 break;
186 case _PAGE_CACHE_WC:
187 prot = PAGE_KERNEL_IO_WC;
188 break;
189 case _PAGE_CACHE_WB:
190 prot = PAGE_KERNEL_IO;
191 break;
192 }
193
194
195
196
197 area = get_vm_area_caller(size, VM_IOREMAP, caller);
198 if (!area)
199 goto err_free_memtype;
200 area->phys_addr = phys_addr;
201 vaddr = (unsigned long) area->addr;
202
203 if (kernel_map_sync_memtype(phys_addr, size, prot_val))
204 goto err_free_area;
205
206 if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot))
207 goto err_free_area;
208
209 ret_addr = (void __iomem *) (vaddr + offset);
210 mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr);
211
212 return ret_addr;
213err_free_area:
214 free_vm_area(area);
215err_free_memtype:
216 free_memtype(phys_addr, phys_addr + size);
217 return NULL;
218}
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size)
242{
243
244
245
246
247
248
249
250 unsigned long val = _PAGE_CACHE_UC_MINUS;
251
252 return __ioremap_caller(phys_addr, size, val,
253 __builtin_return_address(0));
254}
255EXPORT_SYMBOL(ioremap_nocache);
256
257
258
259
260
261
262
263
264
265
266
267void __iomem *ioremap_wc(resource_size_t phys_addr, unsigned long size)
268{
269 if (pat_enabled)
270 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_WC,
271 __builtin_return_address(0));
272 else
273 return ioremap_nocache(phys_addr, size);
274}
275EXPORT_SYMBOL(ioremap_wc);
276
277void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
278{
279 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_WB,
280 __builtin_return_address(0));
281}
282EXPORT_SYMBOL(ioremap_cache);
283
284static void __iomem *ioremap_default(resource_size_t phys_addr,
285 unsigned long size)
286{
287 unsigned long flags;
288 void __iomem *ret;
289 int err;
290
291
292
293
294
295
296 err = reserve_memtype(phys_addr, phys_addr + size,
297 _PAGE_CACHE_WB, &flags);
298 if (err < 0)
299 return NULL;
300
301 ret = __ioremap_caller(phys_addr, size, flags,
302 __builtin_return_address(0));
303
304 free_memtype(phys_addr, phys_addr + size);
305 return ret;
306}
307
308void __iomem *ioremap_prot(resource_size_t phys_addr, unsigned long size,
309 unsigned long prot_val)
310{
311 return __ioremap_caller(phys_addr, size, (prot_val & _PAGE_CACHE_MASK),
312 __builtin_return_address(0));
313}
314EXPORT_SYMBOL(ioremap_prot);
315
316
317
318
319
320
321
322void iounmap(volatile void __iomem *addr)
323{
324 struct vm_struct *p, *o;
325
326 if ((void __force *)addr <= high_memory)
327 return;
328
329
330
331
332
333
334 if ((void __force *)addr >= phys_to_virt(ISA_START_ADDRESS) &&
335 (void __force *)addr < phys_to_virt(ISA_END_ADDRESS))
336 return;
337
338 addr = (volatile void __iomem *)
339 (PAGE_MASK & (unsigned long __force)addr);
340
341 mmiotrace_iounmap(addr);
342
343
344
345
346
347
348 read_lock(&vmlist_lock);
349 for (p = vmlist; p; p = p->next) {
350 if (p->addr == (void __force *)addr)
351 break;
352 }
353 read_unlock(&vmlist_lock);
354
355 if (!p) {
356 printk(KERN_ERR "iounmap: bad address %p\n", addr);
357 dump_stack();
358 return;
359 }
360
361 free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p));
362
363
364 o = remove_vm_area((void __force *)addr);
365 BUG_ON(p != o || o == NULL);
366 kfree(p);
367}
368EXPORT_SYMBOL(iounmap);
369
370
371
372
373
374void *xlate_dev_mem_ptr(unsigned long phys)
375{
376 void *addr;
377 unsigned long start = phys & PAGE_MASK;
378
379
380 if (page_is_ram(start >> PAGE_SHIFT))
381 return __va(phys);
382
383 addr = (void __force *)ioremap_default(start, PAGE_SIZE);
384 if (addr)
385 addr = (void *)((unsigned long)addr | (phys & ~PAGE_MASK));
386
387 return addr;
388}
389
390void unxlate_dev_mem_ptr(unsigned long phys, void *addr)
391{
392 if (page_is_ram(phys >> PAGE_SHIFT))
393 return;
394
395 iounmap((void __iomem *)((unsigned long)addr & PAGE_MASK));
396 return;
397}
398
399static int __initdata early_ioremap_debug;
400
401static int __init early_ioremap_debug_setup(char *str)
402{
403 early_ioremap_debug = 1;
404
405 return 0;
406}
407early_param("early_ioremap_debug", early_ioremap_debug_setup);
408
409static __initdata int after_paging_init;
410static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __page_aligned_bss;
411
412static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
413{
414
415 pgd_t *base = __va(read_cr3());
416 pgd_t *pgd = &base[pgd_index(addr)];
417 pud_t *pud = pud_offset(pgd, addr);
418 pmd_t *pmd = pmd_offset(pud, addr);
419
420 return pmd;
421}
422
423static inline pte_t * __init early_ioremap_pte(unsigned long addr)
424{
425 return &bm_pte[pte_index(addr)];
426}
427
428static unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata;
429
430void __init early_ioremap_init(void)
431{
432 pmd_t *pmd;
433 int i;
434
435 if (early_ioremap_debug)
436 printk(KERN_INFO "early_ioremap_init()\n");
437
438 for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
439 slot_virt[i] = __fix_to_virt(FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*i);
440
441 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
442 memset(bm_pte, 0, sizeof(bm_pte));
443 pmd_populate_kernel(&init_mm, pmd, bm_pte);
444
445
446
447
448
449 if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
450 WARN_ON(1);
451 printk(KERN_WARNING "pmd %p != %p\n",
452 pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
453 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
454 fix_to_virt(FIX_BTMAP_BEGIN));
455 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END): %08lx\n",
456 fix_to_virt(FIX_BTMAP_END));
457
458 printk(KERN_WARNING "FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
459 printk(KERN_WARNING "FIX_BTMAP_BEGIN: %d\n",
460 FIX_BTMAP_BEGIN);
461 }
462}
463
464void __init early_ioremap_reset(void)
465{
466 after_paging_init = 1;
467}
468
469static void __init __early_set_fixmap(enum fixed_addresses idx,
470 phys_addr_t phys, pgprot_t flags)
471{
472 unsigned long addr = __fix_to_virt(idx);
473 pte_t *pte;
474
475 if (idx >= __end_of_fixed_addresses) {
476 BUG();
477 return;
478 }
479 pte = early_ioremap_pte(addr);
480
481 if (pgprot_val(flags))
482 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
483 else
484 pte_clear(&init_mm, addr, pte);
485 __flush_tlb_one(addr);
486}
487
488static inline void __init early_set_fixmap(enum fixed_addresses idx,
489 phys_addr_t phys, pgprot_t prot)
490{
491 if (after_paging_init)
492 __set_fixmap(idx, phys, prot);
493 else
494 __early_set_fixmap(idx, phys, prot);
495}
496
497static inline void __init early_clear_fixmap(enum fixed_addresses idx)
498{
499 if (after_paging_init)
500 clear_fixmap(idx);
501 else
502 __early_set_fixmap(idx, 0, __pgprot(0));
503}
504
505static void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata;
506static unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata;
507
508static int __init check_early_ioremap_leak(void)
509{
510 int count = 0;
511 int i;
512
513 for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
514 if (prev_map[i])
515 count++;
516
517 if (!count)
518 return 0;
519 WARN(1, KERN_WARNING
520 "Debug warning: early ioremap leak of %d areas detected.\n",
521 count);
522 printk(KERN_WARNING
523 "please boot with early_ioremap_debug and report the dmesg.\n");
524
525 return 1;
526}
527late_initcall(check_early_ioremap_leak);
528
529static void __init __iomem *
530__early_ioremap(resource_size_t phys_addr, unsigned long size, pgprot_t prot)
531{
532 unsigned long offset;
533 resource_size_t last_addr;
534 unsigned int nrpages;
535 enum fixed_addresses idx0, idx;
536 int i, slot;
537
538 WARN_ON(system_state != SYSTEM_BOOTING);
539
540 slot = -1;
541 for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
542 if (!prev_map[i]) {
543 slot = i;
544 break;
545 }
546 }
547
548 if (slot < 0) {
549 printk(KERN_INFO "early_iomap(%08llx, %08lx) not found slot\n",
550 (u64)phys_addr, size);
551 WARN_ON(1);
552 return NULL;
553 }
554
555 if (early_ioremap_debug) {
556 printk(KERN_INFO "early_ioremap(%08llx, %08lx) [%d] => ",
557 (u64)phys_addr, size, slot);
558 dump_stack();
559 }
560
561
562 last_addr = phys_addr + size - 1;
563 if (!size || last_addr < phys_addr) {
564 WARN_ON(1);
565 return NULL;
566 }
567
568 prev_size[slot] = size;
569
570
571
572 offset = phys_addr & ~PAGE_MASK;
573 phys_addr &= PAGE_MASK;
574 size = PAGE_ALIGN(last_addr + 1) - phys_addr;
575
576
577
578
579 nrpages = size >> PAGE_SHIFT;
580 if (nrpages > NR_FIX_BTMAPS) {
581 WARN_ON(1);
582 return NULL;
583 }
584
585
586
587
588 idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
589 idx = idx0;
590 while (nrpages > 0) {
591 early_set_fixmap(idx, phys_addr, prot);
592 phys_addr += PAGE_SIZE;
593 --idx;
594 --nrpages;
595 }
596 if (early_ioremap_debug)
597 printk(KERN_CONT "%08lx + %08lx\n", offset, slot_virt[slot]);
598
599 prev_map[slot] = (void __iomem *)(offset + slot_virt[slot]);
600 return prev_map[slot];
601}
602
603
604void __init __iomem *
605early_ioremap(resource_size_t phys_addr, unsigned long size)
606{
607 return __early_ioremap(phys_addr, size, PAGE_KERNEL_IO);
608}
609
610
611void __init __iomem *
612early_memremap(resource_size_t phys_addr, unsigned long size)
613{
614 return __early_ioremap(phys_addr, size, PAGE_KERNEL);
615}
616
617void __init early_iounmap(void __iomem *addr, unsigned long size)
618{
619 unsigned long virt_addr;
620 unsigned long offset;
621 unsigned int nrpages;
622 enum fixed_addresses idx;
623 int i, slot;
624
625 slot = -1;
626 for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
627 if (prev_map[i] == addr) {
628 slot = i;
629 break;
630 }
631 }
632
633 if (slot < 0) {
634 printk(KERN_INFO "early_iounmap(%p, %08lx) not found slot\n",
635 addr, size);
636 WARN_ON(1);
637 return;
638 }
639
640 if (prev_size[slot] != size) {
641 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d] size not consistent %08lx\n",
642 addr, size, slot, prev_size[slot]);
643 WARN_ON(1);
644 return;
645 }
646
647 if (early_ioremap_debug) {
648 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
649 size, slot);
650 dump_stack();
651 }
652
653 virt_addr = (unsigned long)addr;
654 if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
655 WARN_ON(1);
656 return;
657 }
658 offset = virt_addr & ~PAGE_MASK;
659 nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT;
660
661 idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
662 while (nrpages > 0) {
663 early_clear_fixmap(idx);
664 --idx;
665 --nrpages;
666 }
667 prev_map[slot] = NULL;
668}
669