1
2
3
4
5
6
7
8
9#include <linux/bootmem.h>
10#include <linux/init.h>
11#include <linux/io.h>
12#include <linux/module.h>
13#include <linux/slab.h>
14#include <linux/vmalloc.h>
15#include <linux/mmiotrace.h>
16
17#include <asm/cacheflush.h>
18#include <asm/e820.h>
19#include <asm/fixmap.h>
20#include <asm/pgtable.h>
21#include <asm/tlbflush.h>
22#include <asm/pgalloc.h>
23#include <asm/pat.h>
24
25#include "physaddr.h"
26
27
28
29
30
31int ioremap_change_attr(unsigned long vaddr, unsigned long size,
32 unsigned long prot_val)
33{
34 unsigned long nrpages = size >> PAGE_SHIFT;
35 int err;
36
37 switch (prot_val) {
38 case _PAGE_CACHE_UC:
39 default:
40 err = _set_memory_uc(vaddr, nrpages);
41 break;
42 case _PAGE_CACHE_WC:
43 err = _set_memory_wc(vaddr, nrpages);
44 break;
45 case _PAGE_CACHE_WB:
46 err = _set_memory_wb(vaddr, nrpages);
47 break;
48 }
49
50 return err;
51}
52
53
54
55
56
57
58
59
60
61
62static void __iomem *__ioremap_caller(resource_size_t phys_addr,
63 unsigned long size, unsigned long prot_val, void *caller)
64{
65 unsigned long offset, vaddr;
66 resource_size_t pfn, last_pfn, last_addr;
67 const resource_size_t unaligned_phys_addr = phys_addr;
68 const unsigned long unaligned_size = size;
69 struct vm_struct *area;
70 unsigned long new_prot_val;
71 pgprot_t prot;
72 int retval;
73 void __iomem *ret_addr;
74
75
76 last_addr = phys_addr + size - 1;
77 if (!size || last_addr < phys_addr)
78 return NULL;
79
80 if (!phys_addr_valid(phys_addr)) {
81 printk(KERN_WARNING "ioremap: invalid physical address %llx\n",
82 (unsigned long long)phys_addr);
83 WARN_ON_ONCE(1);
84 return NULL;
85 }
86
87
88
89
90 if (is_ISA_range(phys_addr, last_addr))
91 return (__force void __iomem *)phys_to_virt(phys_addr);
92
93
94
95
96 last_pfn = last_addr >> PAGE_SHIFT;
97 for (pfn = phys_addr >> PAGE_SHIFT; pfn <= last_pfn; pfn++) {
98 int is_ram = page_is_ram(pfn);
99
100 if (is_ram && pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn)))
101 return NULL;
102 WARN_ON_ONCE(is_ram);
103 }
104
105
106
107
108 offset = phys_addr & ~PAGE_MASK;
109 phys_addr &= PHYSICAL_PAGE_MASK;
110 size = PAGE_ALIGN(last_addr+1) - phys_addr;
111
112 retval = reserve_memtype(phys_addr, (u64)phys_addr + size,
113 prot_val, &new_prot_val);
114 if (retval) {
115 printk(KERN_ERR "ioremap reserve_memtype failed %d\n", retval);
116 return NULL;
117 }
118
119 if (prot_val != new_prot_val) {
120 if (!is_new_memtype_allowed(phys_addr, size,
121 prot_val, new_prot_val)) {
122 printk(KERN_ERR
123 "ioremap error for 0x%llx-0x%llx, requested 0x%lx, got 0x%lx\n",
124 (unsigned long long)phys_addr,
125 (unsigned long long)(phys_addr + size),
126 prot_val, new_prot_val);
127 goto err_free_memtype;
128 }
129 prot_val = new_prot_val;
130 }
131
132 switch (prot_val) {
133 case _PAGE_CACHE_UC:
134 default:
135 prot = PAGE_KERNEL_IO_NOCACHE;
136 break;
137 case _PAGE_CACHE_UC_MINUS:
138 prot = PAGE_KERNEL_IO_UC_MINUS;
139 break;
140 case _PAGE_CACHE_WC:
141 prot = PAGE_KERNEL_IO_WC;
142 break;
143 case _PAGE_CACHE_WB:
144 prot = PAGE_KERNEL_IO;
145 break;
146 }
147
148
149
150
151 area = get_vm_area_caller(size, VM_IOREMAP, caller);
152 if (!area)
153 goto err_free_memtype;
154 area->phys_addr = phys_addr;
155 vaddr = (unsigned long) area->addr;
156
157 if (kernel_map_sync_memtype(phys_addr, size, prot_val))
158 goto err_free_area;
159
160 if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot))
161 goto err_free_area;
162
163 ret_addr = (void __iomem *) (vaddr + offset);
164 mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr);
165
166
167
168
169
170 WARN_ONCE(iomem_map_sanity_check(unaligned_phys_addr, unaligned_size),
171 KERN_INFO "Info: mapping multiple BARs. Your kernel is fine.");
172
173 return ret_addr;
174err_free_area:
175 free_vm_area(area);
176err_free_memtype:
177 free_memtype(phys_addr, phys_addr + size);
178 return NULL;
179}
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size)
203{
204
205
206
207
208
209
210
211 unsigned long val = _PAGE_CACHE_UC_MINUS;
212
213 return __ioremap_caller(phys_addr, size, val,
214 __builtin_return_address(0));
215}
216EXPORT_SYMBOL(ioremap_nocache);
217
218
219
220
221
222
223
224
225
226
227
228void __iomem *ioremap_wc(resource_size_t phys_addr, unsigned long size)
229{
230 if (pat_enabled)
231 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_WC,
232 __builtin_return_address(0));
233 else
234 return ioremap_nocache(phys_addr, size);
235}
236EXPORT_SYMBOL(ioremap_wc);
237
238void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
239{
240 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_WB,
241 __builtin_return_address(0));
242}
243EXPORT_SYMBOL(ioremap_cache);
244
245void __iomem *ioremap_prot(resource_size_t phys_addr, unsigned long size,
246 unsigned long prot_val)
247{
248 return __ioremap_caller(phys_addr, size, (prot_val & _PAGE_CACHE_MASK),
249 __builtin_return_address(0));
250}
251EXPORT_SYMBOL(ioremap_prot);
252
253
254
255
256
257
258
259void iounmap(volatile void __iomem *addr)
260{
261 struct vm_struct *p, *o;
262
263 if ((void __force *)addr <= high_memory)
264 return;
265
266
267
268
269
270
271 if ((void __force *)addr >= phys_to_virt(ISA_START_ADDRESS) &&
272 (void __force *)addr < phys_to_virt(ISA_END_ADDRESS))
273 return;
274
275 addr = (volatile void __iomem *)
276 (PAGE_MASK & (unsigned long __force)addr);
277
278 mmiotrace_iounmap(addr);
279
280
281
282
283
284
285 p = find_vm_area((void __force *)addr);
286
287 if (!p) {
288 printk(KERN_ERR "iounmap: bad address %p\n", addr);
289 dump_stack();
290 return;
291 }
292
293 free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p));
294
295
296 o = remove_vm_area((void __force *)addr);
297 BUG_ON(p != o || o == NULL);
298 kfree(p);
299}
300EXPORT_SYMBOL(iounmap);
301
302
303
304
305
306void *xlate_dev_mem_ptr(unsigned long phys)
307{
308 void *addr;
309 unsigned long start = phys & PAGE_MASK;
310
311
312 if (page_is_ram(start >> PAGE_SHIFT))
313 return __va(phys);
314
315 addr = (void __force *)ioremap_cache(start, PAGE_SIZE);
316 if (addr)
317 addr = (void *)((unsigned long)addr | (phys & ~PAGE_MASK));
318
319 return addr;
320}
321
322void unxlate_dev_mem_ptr(unsigned long phys, void *addr)
323{
324 if (page_is_ram(phys >> PAGE_SHIFT))
325 return;
326
327 iounmap((void __iomem *)((unsigned long)addr & PAGE_MASK));
328 return;
329}
330
331static int __initdata early_ioremap_debug;
332
333static int __init early_ioremap_debug_setup(char *str)
334{
335 early_ioremap_debug = 1;
336
337 return 0;
338}
339early_param("early_ioremap_debug", early_ioremap_debug_setup);
340
341static __initdata int after_paging_init;
342static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __page_aligned_bss;
343
344static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
345{
346
347 pgd_t *base = __va(read_cr3());
348 pgd_t *pgd = &base[pgd_index(addr)];
349 pud_t *pud = pud_offset(pgd, addr);
350 pmd_t *pmd = pmd_offset(pud, addr);
351
352 return pmd;
353}
354
355static inline pte_t * __init early_ioremap_pte(unsigned long addr)
356{
357 return &bm_pte[pte_index(addr)];
358}
359
360bool __init is_early_ioremap_ptep(pte_t *ptep)
361{
362 return ptep >= &bm_pte[0] && ptep < &bm_pte[PAGE_SIZE/sizeof(pte_t)];
363}
364
365static unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata;
366
367void __init early_ioremap_init(void)
368{
369 pmd_t *pmd;
370 int i;
371
372 if (early_ioremap_debug)
373 printk(KERN_INFO "early_ioremap_init()\n");
374
375 for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
376 slot_virt[i] = __fix_to_virt(FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*i);
377
378 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
379 memset(bm_pte, 0, sizeof(bm_pte));
380 pmd_populate_kernel(&init_mm, pmd, bm_pte);
381
382
383
384
385
386#define __FIXADDR_TOP (-PAGE_SIZE)
387 BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
388 != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
389#undef __FIXADDR_TOP
390 if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
391 WARN_ON(1);
392 printk(KERN_WARNING "pmd %p != %p\n",
393 pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
394 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
395 fix_to_virt(FIX_BTMAP_BEGIN));
396 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END): %08lx\n",
397 fix_to_virt(FIX_BTMAP_END));
398
399 printk(KERN_WARNING "FIX_BTMAP_END: %d\n", FIX_BTMAP_END);
400 printk(KERN_WARNING "FIX_BTMAP_BEGIN: %d\n",
401 FIX_BTMAP_BEGIN);
402 }
403}
404
405void __init early_ioremap_reset(void)
406{
407 after_paging_init = 1;
408}
409
410static void __init __early_set_fixmap(enum fixed_addresses idx,
411 phys_addr_t phys, pgprot_t flags)
412{
413 unsigned long addr = __fix_to_virt(idx);
414 pte_t *pte;
415
416 if (idx >= __end_of_fixed_addresses) {
417 BUG();
418 return;
419 }
420 pte = early_ioremap_pte(addr);
421
422 if (pgprot_val(flags))
423 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
424 else
425 pte_clear(&init_mm, addr, pte);
426 __flush_tlb_one(addr);
427}
428
429static inline void __init early_set_fixmap(enum fixed_addresses idx,
430 phys_addr_t phys, pgprot_t prot)
431{
432 if (after_paging_init)
433 __set_fixmap(idx, phys, prot);
434 else
435 __early_set_fixmap(idx, phys, prot);
436}
437
438static inline void __init early_clear_fixmap(enum fixed_addresses idx)
439{
440 if (after_paging_init)
441 clear_fixmap(idx);
442 else
443 __early_set_fixmap(idx, 0, __pgprot(0));
444}
445
446static void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata;
447static unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata;
448
449void __init fixup_early_ioremap(void)
450{
451 int i;
452
453 for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
454 if (prev_map[i]) {
455 WARN_ON(1);
456 break;
457 }
458 }
459
460 early_ioremap_init();
461}
462
463static int __init check_early_ioremap_leak(void)
464{
465 int count = 0;
466 int i;
467
468 for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
469 if (prev_map[i])
470 count++;
471
472 if (!count)
473 return 0;
474 WARN(1, KERN_WARNING
475 "Debug warning: early ioremap leak of %d areas detected.\n",
476 count);
477 printk(KERN_WARNING
478 "please boot with early_ioremap_debug and report the dmesg.\n");
479
480 return 1;
481}
482late_initcall(check_early_ioremap_leak);
483
484static void __init __iomem *
485__early_ioremap(resource_size_t phys_addr, unsigned long size, pgprot_t prot)
486{
487 unsigned long offset;
488 resource_size_t last_addr;
489 unsigned int nrpages;
490 enum fixed_addresses idx;
491 int i, slot;
492
493 WARN_ON(system_state != SYSTEM_BOOTING);
494
495 slot = -1;
496 for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
497 if (!prev_map[i]) {
498 slot = i;
499 break;
500 }
501 }
502
503 if (slot < 0) {
504 printk(KERN_INFO "%s(%08llx, %08lx) not found slot\n",
505 __func__, (u64)phys_addr, size);
506 WARN_ON(1);
507 return NULL;
508 }
509
510 if (early_ioremap_debug) {
511 printk(KERN_INFO "%s(%08llx, %08lx) [%d] => ",
512 __func__, (u64)phys_addr, size, slot);
513 dump_stack();
514 }
515
516
517 last_addr = phys_addr + size - 1;
518 if (!size || last_addr < phys_addr) {
519 WARN_ON(1);
520 return NULL;
521 }
522
523 prev_size[slot] = size;
524
525
526
527 offset = phys_addr & ~PAGE_MASK;
528 phys_addr &= PAGE_MASK;
529 size = PAGE_ALIGN(last_addr + 1) - phys_addr;
530
531
532
533
534 nrpages = size >> PAGE_SHIFT;
535 if (nrpages > NR_FIX_BTMAPS) {
536 WARN_ON(1);
537 return NULL;
538 }
539
540
541
542
543 idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
544 while (nrpages > 0) {
545 early_set_fixmap(idx, phys_addr, prot);
546 phys_addr += PAGE_SIZE;
547 --idx;
548 --nrpages;
549 }
550 if (early_ioremap_debug)
551 printk(KERN_CONT "%08lx + %08lx\n", offset, slot_virt[slot]);
552
553 prev_map[slot] = (void __iomem *)(offset + slot_virt[slot]);
554 return prev_map[slot];
555}
556
557
558void __init __iomem *
559early_ioremap(resource_size_t phys_addr, unsigned long size)
560{
561 return __early_ioremap(phys_addr, size, PAGE_KERNEL_IO);
562}
563
564
565void __init __iomem *
566early_memremap(resource_size_t phys_addr, unsigned long size)
567{
568 return __early_ioremap(phys_addr, size, PAGE_KERNEL);
569}
570
571void __init early_iounmap(void __iomem *addr, unsigned long size)
572{
573 unsigned long virt_addr;
574 unsigned long offset;
575 unsigned int nrpages;
576 enum fixed_addresses idx;
577 int i, slot;
578
579 slot = -1;
580 for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
581 if (prev_map[i] == addr) {
582 slot = i;
583 break;
584 }
585 }
586
587 if (slot < 0) {
588 printk(KERN_INFO "early_iounmap(%p, %08lx) not found slot\n",
589 addr, size);
590 WARN_ON(1);
591 return;
592 }
593
594 if (prev_size[slot] != size) {
595 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d] size not consistent %08lx\n",
596 addr, size, slot, prev_size[slot]);
597 WARN_ON(1);
598 return;
599 }
600
601 if (early_ioremap_debug) {
602 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
603 size, slot);
604 dump_stack();
605 }
606
607 virt_addr = (unsigned long)addr;
608 if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
609 WARN_ON(1);
610 return;
611 }
612 offset = virt_addr & ~PAGE_MASK;
613 nrpages = PAGE_ALIGN(offset + size) >> PAGE_SHIFT;
614
615 idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
616 while (nrpages > 0) {
617 early_clear_fixmap(idx);
618 --idx;
619 --nrpages;
620 }
621 prev_map[slot] = NULL;
622}
623