1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/kernel.h>
14#include <linux/types.h>
15#include <linux/init.h>
16#include <linux/bootmem.h>
17#include <linux/mmzone.h>
18#include <linux/pci_ids.h>
19#include <linux/pci.h>
20#include <linux/bitops.h>
21#include <linux/ioport.h>
22#include <linux/suspend.h>
23#include <linux/kmemleak.h>
24#include <asm/e820.h>
25#include <asm/io.h>
26#include <asm/iommu.h>
27#include <asm/gart.h>
28#include <asm/pci-direct.h>
29#include <asm/dma.h>
30#include <asm/amd_nb.h>
31#include <asm/x86_init.h>
32
33int gart_iommu_aperture;
34int gart_iommu_aperture_disabled __initdata;
35int gart_iommu_aperture_allowed __initdata;
36
37int fallback_aper_order __initdata = 1;
38int fallback_aper_force __initdata;
39
40int fix_aperture __initdata = 1;
41
42static struct resource gart_resource = {
43 .name = "GART",
44 .flags = IORESOURCE_MEM,
45};
46
47static void __init insert_aperture_resource(u32 aper_base, u32 aper_size)
48{
49 gart_resource.start = aper_base;
50 gart_resource.end = aper_base + aper_size - 1;
51 insert_resource(&iomem_resource, &gart_resource);
52}
53
54
55
56
57static u32 __init allocate_aperture(void)
58{
59 u32 aper_size;
60 void *p;
61
62
63 if (fallback_aper_order > 5)
64 fallback_aper_order = 5;
65 aper_size = (32 * 1024 * 1024) << fallback_aper_order;
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86 p = __alloc_bootmem_nopanic(aper_size, aper_size, 512ULL<<20);
87
88
89
90
91 kmemleak_ignore(p);
92 if (!p || __pa(p)+aper_size > 0xffffffff) {
93 printk(KERN_ERR
94 "Cannot allocate aperture memory hole (%p,%uK)\n",
95 p, aper_size>>10);
96 if (p)
97 free_bootmem(__pa(p), aper_size);
98 return 0;
99 }
100 printk(KERN_INFO "Mapping aperture over %d KB of RAM @ %lx\n",
101 aper_size >> 10, __pa(p));
102 insert_aperture_resource((u32)__pa(p), aper_size);
103 register_nosave_region((u32)__pa(p) >> PAGE_SHIFT,
104 (u32)__pa(p+aper_size) >> PAGE_SHIFT);
105
106 return (u32)__pa(p);
107}
108
109
110
111static u32 __init find_cap(int bus, int slot, int func, int cap)
112{
113 int bytes;
114 u8 pos;
115
116 if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) &
117 PCI_STATUS_CAP_LIST))
118 return 0;
119
120 pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST);
121 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) {
122 u8 id;
123
124 pos &= ~3;
125 id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID);
126 if (id == 0xff)
127 break;
128 if (id == cap)
129 return pos;
130 pos = read_pci_config_byte(bus, slot, func,
131 pos+PCI_CAP_LIST_NEXT);
132 }
133 return 0;
134}
135
136
137static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order)
138{
139 u32 apsize;
140 u32 apsizereg;
141 int nbits;
142 u32 aper_low, aper_hi;
143 u64 aper;
144 u32 old_order;
145
146 printk(KERN_INFO "AGP bridge at %02x:%02x:%02x\n", bus, slot, func);
147 apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14);
148 if (apsizereg == 0xffffffff) {
149 printk(KERN_ERR "APSIZE in AGP bridge unreadable\n");
150 return 0;
151 }
152
153
154 old_order = *order;
155
156 apsize = apsizereg & 0xfff;
157
158 if (apsize & 0xff)
159 apsize |= 0xf00;
160 nbits = hweight16(apsize);
161 *order = 7 - nbits;
162 if ((int)*order < 0)
163 *order = 0;
164
165 aper_low = read_pci_config(bus, slot, func, 0x10);
166 aper_hi = read_pci_config(bus, slot, func, 0x14);
167 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32);
168
169
170
171
172
173 printk(KERN_INFO "Aperture from AGP @ %Lx old size %u MB\n",
174 aper, 32 << old_order);
175 if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) {
176 printk(KERN_INFO "Aperture size %u MB (APSIZE %x) is not right, using settings from NB\n",
177 32 << *order, apsizereg);
178 *order = old_order;
179 }
180
181 printk(KERN_INFO "Aperture from AGP @ %Lx size %u MB (APSIZE %x)\n",
182 aper, 32 << *order, apsizereg);
183
184 if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20))
185 return 0;
186 return (u32)aper;
187}
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202static u32 __init search_agp_bridge(u32 *order, int *valid_agp)
203{
204 int bus, slot, func;
205
206
207 for (bus = 0; bus < 256; bus++) {
208 for (slot = 0; slot < 32; slot++) {
209 for (func = 0; func < 8; func++) {
210 u32 class, cap;
211 u8 type;
212 class = read_pci_config(bus, slot, func,
213 PCI_CLASS_REVISION);
214 if (class == 0xffffffff)
215 break;
216
217 switch (class >> 16) {
218 case PCI_CLASS_BRIDGE_HOST:
219 case PCI_CLASS_BRIDGE_OTHER:
220
221 cap = find_cap(bus, slot, func,
222 PCI_CAP_ID_AGP);
223 if (!cap)
224 break;
225 *valid_agp = 1;
226 return read_agp(bus, slot, func, cap,
227 order);
228 }
229
230
231 type = read_pci_config_byte(bus, slot, func,
232 PCI_HEADER_TYPE);
233 if (!(type & 0x80))
234 break;
235 }
236 }
237 }
238 printk(KERN_INFO "No AGP bridge found\n");
239
240 return 0;
241}
242
243static int gart_fix_e820 __initdata = 1;
244
245static int __init parse_gart_mem(char *p)
246{
247 if (!p)
248 return -EINVAL;
249
250 if (!strncmp(p, "off", 3))
251 gart_fix_e820 = 0;
252 else if (!strncmp(p, "on", 2))
253 gart_fix_e820 = 1;
254
255 return 0;
256}
257early_param("gart_fix_e820", parse_gart_mem);
258
259void __init early_gart_iommu_check(void)
260{
261
262
263
264
265
266
267
268
269
270
271 u32 agp_aper_order = 0;
272 int i, fix, slot, valid_agp = 0;
273 u32 ctl;
274 u32 aper_size = 0, aper_order = 0, last_aper_order = 0;
275 u64 aper_base = 0, last_aper_base = 0;
276 int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0;
277
278 if (!early_pci_allowed())
279 return;
280
281
282 search_agp_bridge(&agp_aper_order, &valid_agp);
283
284 fix = 0;
285 for (i = 0; amd_nb_bus_dev_ranges[i].dev_limit; i++) {
286 int bus;
287 int dev_base, dev_limit;
288
289 bus = amd_nb_bus_dev_ranges[i].bus;
290 dev_base = amd_nb_bus_dev_ranges[i].dev_base;
291 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit;
292
293 for (slot = dev_base; slot < dev_limit; slot++) {
294 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00)))
295 continue;
296
297 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL);
298 aper_enabled = ctl & GARTEN;
299 aper_order = (ctl >> 1) & 7;
300 aper_size = (32 * 1024 * 1024) << aper_order;
301 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff;
302 aper_base <<= 25;
303
304 if (last_valid) {
305 if ((aper_order != last_aper_order) ||
306 (aper_base != last_aper_base) ||
307 (aper_enabled != last_aper_enabled)) {
308 fix = 1;
309 break;
310 }
311 }
312
313 last_aper_order = aper_order;
314 last_aper_base = aper_base;
315 last_aper_enabled = aper_enabled;
316 last_valid = 1;
317 }
318 }
319
320 if (!fix && !aper_enabled)
321 return;
322
323 if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL)
324 fix = 1;
325
326 if (gart_fix_e820 && !fix && aper_enabled) {
327 if (e820_any_mapped(aper_base, aper_base + aper_size,
328 E820_RAM)) {
329
330 printk(KERN_INFO "update e820 for GART\n");
331 e820_add_region(aper_base, aper_size, E820_RESERVED);
332 update_e820();
333 }
334 }
335
336 if (valid_agp)
337 return;
338
339
340 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) {
341 int bus;
342 int dev_base, dev_limit;
343
344 bus = amd_nb_bus_dev_ranges[i].bus;
345 dev_base = amd_nb_bus_dev_ranges[i].dev_base;
346 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit;
347
348 for (slot = dev_base; slot < dev_limit; slot++) {
349 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00)))
350 continue;
351
352 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL);
353 ctl &= ~GARTEN;
354 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl);
355 }
356 }
357
358}
359
360static int __initdata printed_gart_size_msg;
361
362int __init gart_iommu_hole_init(void)
363{
364 u32 agp_aper_base = 0, agp_aper_order = 0;
365 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0;
366 u64 aper_base, last_aper_base = 0;
367 int fix, slot, valid_agp = 0;
368 int i, node;
369
370 if (gart_iommu_aperture_disabled || !fix_aperture ||
371 !early_pci_allowed())
372 return -ENODEV;
373
374 printk(KERN_INFO "Checking aperture...\n");
375
376 if (!fallback_aper_force)
377 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp);
378
379 fix = 0;
380 node = 0;
381 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) {
382 int bus;
383 int dev_base, dev_limit;
384 u32 ctl;
385
386 bus = amd_nb_bus_dev_ranges[i].bus;
387 dev_base = amd_nb_bus_dev_ranges[i].dev_base;
388 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit;
389
390 for (slot = dev_base; slot < dev_limit; slot++) {
391 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00)))
392 continue;
393
394 iommu_detected = 1;
395 gart_iommu_aperture = 1;
396 x86_init.iommu.iommu_init = gart_iommu_init;
397
398 ctl = read_pci_config(bus, slot, 3,
399 AMD64_GARTAPERTURECTL);
400
401
402
403
404
405
406
407 ctl &= ~GARTEN;
408 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl);
409
410 aper_order = (ctl >> 1) & 7;
411 aper_size = (32 * 1024 * 1024) << aper_order;
412 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff;
413 aper_base <<= 25;
414
415 printk(KERN_INFO "Node %d: aperture @ %Lx size %u MB\n",
416 node, aper_base, aper_size >> 20);
417 node++;
418
419 if (!aperture_valid(aper_base, aper_size, 64<<20)) {
420 if (valid_agp && agp_aper_base &&
421 agp_aper_base == aper_base &&
422 agp_aper_order == aper_order) {
423
424 if (!no_iommu &&
425 max_pfn > MAX_DMA32_PFN &&
426 !printed_gart_size_msg) {
427 printk(KERN_ERR "you are using iommu with agp, but GART size is less than 64M\n");
428 printk(KERN_ERR "please increase GART size in your BIOS setup\n");
429 printk(KERN_ERR "if BIOS doesn't have that option, contact your HW vendor!\n");
430 printed_gart_size_msg = 1;
431 }
432 } else {
433 fix = 1;
434 goto out;
435 }
436 }
437
438 if ((last_aper_order && aper_order != last_aper_order) ||
439 (last_aper_base && aper_base != last_aper_base)) {
440 fix = 1;
441 goto out;
442 }
443 last_aper_order = aper_order;
444 last_aper_base = aper_base;
445 }
446 }
447
448out:
449 if (!fix && !fallback_aper_force) {
450 if (last_aper_base) {
451 unsigned long n = (32 * 1024 * 1024) << last_aper_order;
452
453 insert_aperture_resource((u32)last_aper_base, n);
454 return 1;
455 }
456 return 0;
457 }
458
459 if (!fallback_aper_force) {
460 aper_alloc = agp_aper_base;
461 aper_order = agp_aper_order;
462 }
463
464 if (aper_alloc) {
465
466 } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) ||
467 force_iommu ||
468 valid_agp ||
469 fallback_aper_force) {
470 printk(KERN_INFO
471 "Your BIOS doesn't leave a aperture memory hole\n");
472 printk(KERN_INFO
473 "Please enable the IOMMU option in the BIOS setup\n");
474 printk(KERN_INFO
475 "This costs you %d MB of RAM\n",
476 32 << fallback_aper_order);
477
478 aper_order = fallback_aper_order;
479 aper_alloc = allocate_aperture();
480 if (!aper_alloc) {
481
482
483
484
485
486
487
488
489 panic("Not enough memory for aperture");
490 }
491 } else {
492 return 0;
493 }
494
495
496 for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) {
497 int bus, dev_base, dev_limit;
498
499
500
501
502
503 u32 ctl = DISTLBWALKPRB | aper_order << 1;
504
505 bus = amd_nb_bus_dev_ranges[i].bus;
506 dev_base = amd_nb_bus_dev_ranges[i].dev_base;
507 dev_limit = amd_nb_bus_dev_ranges[i].dev_limit;
508 for (slot = dev_base; slot < dev_limit; slot++) {
509 if (!early_is_amd_nb(read_pci_config(bus, slot, 3, 0x00)))
510 continue;
511
512 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl);
513 write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25);
514 }
515 }
516
517 set_up_gart_resume(aper_order, aper_alloc);
518
519 return 1;
520}
521