1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/kernel.h>
14#include <linux/types.h>
15#include <linux/init.h>
16#include <linux/bootmem.h>
17#include <linux/mmzone.h>
18#include <linux/pci_ids.h>
19#include <linux/pci.h>
20#include <linux/bitops.h>
21#include <linux/ioport.h>
22#include <linux/suspend.h>
23#include <linux/kmemleak.h>
24#include <asm/e820.h>
25#include <asm/io.h>
26#include <asm/iommu.h>
27#include <asm/gart.h>
28#include <asm/pci-direct.h>
29#include <asm/dma.h>
30#include <asm/k8.h>
31
32int gart_iommu_aperture;
33int gart_iommu_aperture_disabled __initdata;
34int gart_iommu_aperture_allowed __initdata;
35
36int fallback_aper_order __initdata = 1;
37int fallback_aper_force __initdata;
38
39int fix_aperture __initdata = 1;
40
41struct bus_dev_range {
42 int bus;
43 int dev_base;
44 int dev_limit;
45};
46
47static struct bus_dev_range bus_dev_ranges[] __initdata = {
48 { 0x00, 0x18, 0x20},
49 { 0xff, 0x00, 0x20},
50 { 0xfe, 0x00, 0x20}
51};
52
53static struct resource gart_resource = {
54 .name = "GART",
55 .flags = IORESOURCE_MEM,
56};
57
58static void __init insert_aperture_resource(u32 aper_base, u32 aper_size)
59{
60 gart_resource.start = aper_base;
61 gart_resource.end = aper_base + aper_size - 1;
62 insert_resource(&iomem_resource, &gart_resource);
63}
64
65
66
67
68static u32 __init allocate_aperture(void)
69{
70 u32 aper_size;
71 void *p;
72
73
74 if (fallback_aper_order > 5)
75 fallback_aper_order = 5;
76 aper_size = (32 * 1024 * 1024) << fallback_aper_order;
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97 p = __alloc_bootmem_nopanic(aper_size, aper_size, 512ULL<<20);
98
99
100
101
102 kmemleak_ignore(p);
103 if (!p || __pa(p)+aper_size > 0xffffffff) {
104 printk(KERN_ERR
105 "Cannot allocate aperture memory hole (%p,%uK)\n",
106 p, aper_size>>10);
107 if (p)
108 free_bootmem(__pa(p), aper_size);
109 return 0;
110 }
111 printk(KERN_INFO "Mapping aperture over %d KB of RAM @ %lx\n",
112 aper_size >> 10, __pa(p));
113 insert_aperture_resource((u32)__pa(p), aper_size);
114 register_nosave_region((u32)__pa(p) >> PAGE_SHIFT,
115 (u32)__pa(p+aper_size) >> PAGE_SHIFT);
116
117 return (u32)__pa(p);
118}
119
120
121
122static u32 __init find_cap(int bus, int slot, int func, int cap)
123{
124 int bytes;
125 u8 pos;
126
127 if (!(read_pci_config_16(bus, slot, func, PCI_STATUS) &
128 PCI_STATUS_CAP_LIST))
129 return 0;
130
131 pos = read_pci_config_byte(bus, slot, func, PCI_CAPABILITY_LIST);
132 for (bytes = 0; bytes < 48 && pos >= 0x40; bytes++) {
133 u8 id;
134
135 pos &= ~3;
136 id = read_pci_config_byte(bus, slot, func, pos+PCI_CAP_LIST_ID);
137 if (id == 0xff)
138 break;
139 if (id == cap)
140 return pos;
141 pos = read_pci_config_byte(bus, slot, func,
142 pos+PCI_CAP_LIST_NEXT);
143 }
144 return 0;
145}
146
147
148static u32 __init read_agp(int bus, int slot, int func, int cap, u32 *order)
149{
150 u32 apsize;
151 u32 apsizereg;
152 int nbits;
153 u32 aper_low, aper_hi;
154 u64 aper;
155 u32 old_order;
156
157 printk(KERN_INFO "AGP bridge at %02x:%02x:%02x\n", bus, slot, func);
158 apsizereg = read_pci_config_16(bus, slot, func, cap + 0x14);
159 if (apsizereg == 0xffffffff) {
160 printk(KERN_ERR "APSIZE in AGP bridge unreadable\n");
161 return 0;
162 }
163
164
165 old_order = *order;
166
167 apsize = apsizereg & 0xfff;
168
169 if (apsize & 0xff)
170 apsize |= 0xf00;
171 nbits = hweight16(apsize);
172 *order = 7 - nbits;
173 if ((int)*order < 0)
174 *order = 0;
175
176 aper_low = read_pci_config(bus, slot, func, 0x10);
177 aper_hi = read_pci_config(bus, slot, func, 0x14);
178 aper = (aper_low & ~((1<<22)-1)) | ((u64)aper_hi << 32);
179
180
181
182
183
184 printk(KERN_INFO "Aperture from AGP @ %Lx old size %u MB\n",
185 aper, 32 << old_order);
186 if (aper + (32ULL<<(20 + *order)) > 0x100000000ULL) {
187 printk(KERN_INFO "Aperture size %u MB (APSIZE %x) is not right, using settings from NB\n",
188 32 << *order, apsizereg);
189 *order = old_order;
190 }
191
192 printk(KERN_INFO "Aperture from AGP @ %Lx size %u MB (APSIZE %x)\n",
193 aper, 32 << *order, apsizereg);
194
195 if (!aperture_valid(aper, (32*1024*1024) << *order, 32<<20))
196 return 0;
197 return (u32)aper;
198}
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213static u32 __init search_agp_bridge(u32 *order, int *valid_agp)
214{
215 int bus, slot, func;
216
217
218 for (bus = 0; bus < 256; bus++) {
219 for (slot = 0; slot < 32; slot++) {
220 for (func = 0; func < 8; func++) {
221 u32 class, cap;
222 u8 type;
223 class = read_pci_config(bus, slot, func,
224 PCI_CLASS_REVISION);
225 if (class == 0xffffffff)
226 break;
227
228 switch (class >> 16) {
229 case PCI_CLASS_BRIDGE_HOST:
230 case PCI_CLASS_BRIDGE_OTHER:
231
232 cap = find_cap(bus, slot, func,
233 PCI_CAP_ID_AGP);
234 if (!cap)
235 break;
236 *valid_agp = 1;
237 return read_agp(bus, slot, func, cap,
238 order);
239 }
240
241
242 type = read_pci_config_byte(bus, slot, func,
243 PCI_HEADER_TYPE);
244 if (!(type & 0x80))
245 break;
246 }
247 }
248 }
249 printk(KERN_INFO "No AGP bridge found\n");
250
251 return 0;
252}
253
254static int gart_fix_e820 __initdata = 1;
255
256static int __init parse_gart_mem(char *p)
257{
258 if (!p)
259 return -EINVAL;
260
261 if (!strncmp(p, "off", 3))
262 gart_fix_e820 = 0;
263 else if (!strncmp(p, "on", 2))
264 gart_fix_e820 = 1;
265
266 return 0;
267}
268early_param("gart_fix_e820", parse_gart_mem);
269
270void __init early_gart_iommu_check(void)
271{
272
273
274
275
276
277
278
279
280
281
282 int i, fix, slot;
283 u32 ctl;
284 u32 aper_size = 0, aper_order = 0, last_aper_order = 0;
285 u64 aper_base = 0, last_aper_base = 0;
286 int aper_enabled = 0, last_aper_enabled = 0, last_valid = 0;
287
288 if (!early_pci_allowed())
289 return;
290
291
292 fix = 0;
293 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
294 int bus;
295 int dev_base, dev_limit;
296
297 bus = bus_dev_ranges[i].bus;
298 dev_base = bus_dev_ranges[i].dev_base;
299 dev_limit = bus_dev_ranges[i].dev_limit;
300
301 for (slot = dev_base; slot < dev_limit; slot++) {
302 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
303 continue;
304
305 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL);
306 aper_enabled = ctl & AMD64_GARTEN;
307 aper_order = (ctl >> 1) & 7;
308 aper_size = (32 * 1024 * 1024) << aper_order;
309 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff;
310 aper_base <<= 25;
311
312 if (last_valid) {
313 if ((aper_order != last_aper_order) ||
314 (aper_base != last_aper_base) ||
315 (aper_enabled != last_aper_enabled)) {
316 fix = 1;
317 break;
318 }
319 }
320
321 last_aper_order = aper_order;
322 last_aper_base = aper_base;
323 last_aper_enabled = aper_enabled;
324 last_valid = 1;
325 }
326 }
327
328 if (!fix && !aper_enabled)
329 return;
330
331 if (!aper_base || !aper_size || aper_base + aper_size > 0x100000000UL)
332 fix = 1;
333
334 if (gart_fix_e820 && !fix && aper_enabled) {
335 if (e820_any_mapped(aper_base, aper_base + aper_size,
336 E820_RAM)) {
337
338 printk(KERN_INFO "update e820 for GART\n");
339 e820_add_region(aper_base, aper_size, E820_RESERVED);
340 update_e820();
341 }
342 }
343
344 if (!fix)
345 return;
346
347
348 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
349 int bus;
350 int dev_base, dev_limit;
351
352 bus = bus_dev_ranges[i].bus;
353 dev_base = bus_dev_ranges[i].dev_base;
354 dev_limit = bus_dev_ranges[i].dev_limit;
355
356 for (slot = dev_base; slot < dev_limit; slot++) {
357 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
358 continue;
359
360 ctl = read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL);
361 ctl &= ~AMD64_GARTEN;
362 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, ctl);
363 }
364 }
365
366}
367
368static int __initdata printed_gart_size_msg;
369
370void __init gart_iommu_hole_init(void)
371{
372 u32 agp_aper_base = 0, agp_aper_order = 0;
373 u32 aper_size, aper_alloc = 0, aper_order = 0, last_aper_order = 0;
374 u64 aper_base, last_aper_base = 0;
375 int fix, slot, valid_agp = 0;
376 int i, node;
377
378 if (gart_iommu_aperture_disabled || !fix_aperture ||
379 !early_pci_allowed())
380 return;
381
382 printk(KERN_INFO "Checking aperture...\n");
383
384 if (!fallback_aper_force)
385 agp_aper_base = search_agp_bridge(&agp_aper_order, &valid_agp);
386
387 fix = 0;
388 node = 0;
389 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
390 int bus;
391 int dev_base, dev_limit;
392
393 bus = bus_dev_ranges[i].bus;
394 dev_base = bus_dev_ranges[i].dev_base;
395 dev_limit = bus_dev_ranges[i].dev_limit;
396
397 for (slot = dev_base; slot < dev_limit; slot++) {
398 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
399 continue;
400
401 iommu_detected = 1;
402 gart_iommu_aperture = 1;
403
404 aper_order = (read_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL) >> 1) & 7;
405 aper_size = (32 * 1024 * 1024) << aper_order;
406 aper_base = read_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE) & 0x7fff;
407 aper_base <<= 25;
408
409 printk(KERN_INFO "Node %d: aperture @ %Lx size %u MB\n",
410 node, aper_base, aper_size >> 20);
411 node++;
412
413 if (!aperture_valid(aper_base, aper_size, 64<<20)) {
414 if (valid_agp && agp_aper_base &&
415 agp_aper_base == aper_base &&
416 agp_aper_order == aper_order) {
417
418 if (!no_iommu &&
419 max_pfn > MAX_DMA32_PFN &&
420 !printed_gart_size_msg) {
421 printk(KERN_ERR "you are using iommu with agp, but GART size is less than 64M\n");
422 printk(KERN_ERR "please increase GART size in your BIOS setup\n");
423 printk(KERN_ERR "if BIOS doesn't have that option, contact your HW vendor!\n");
424 printed_gart_size_msg = 1;
425 }
426 } else {
427 fix = 1;
428 goto out;
429 }
430 }
431
432 if ((last_aper_order && aper_order != last_aper_order) ||
433 (last_aper_base && aper_base != last_aper_base)) {
434 fix = 1;
435 goto out;
436 }
437 last_aper_order = aper_order;
438 last_aper_base = aper_base;
439 }
440 }
441
442out:
443 if (!fix && !fallback_aper_force) {
444 if (last_aper_base) {
445 unsigned long n = (32 * 1024 * 1024) << last_aper_order;
446
447 insert_aperture_resource((u32)last_aper_base, n);
448 }
449 return;
450 }
451
452 if (!fallback_aper_force) {
453 aper_alloc = agp_aper_base;
454 aper_order = agp_aper_order;
455 }
456
457 if (aper_alloc) {
458
459 } else if (swiotlb && !valid_agp) {
460
461 } else if ((!no_iommu && max_pfn > MAX_DMA32_PFN) ||
462 force_iommu ||
463 valid_agp ||
464 fallback_aper_force) {
465 printk(KERN_INFO
466 "Your BIOS doesn't leave a aperture memory hole\n");
467 printk(KERN_INFO
468 "Please enable the IOMMU option in the BIOS setup\n");
469 printk(KERN_INFO
470 "This costs you %d MB of RAM\n",
471 32 << fallback_aper_order);
472
473 aper_order = fallback_aper_order;
474 aper_alloc = allocate_aperture();
475 if (!aper_alloc) {
476
477
478
479
480
481
482
483
484 panic("Not enough memory for aperture");
485 }
486 } else {
487 return;
488 }
489
490
491 for (i = 0; i < ARRAY_SIZE(bus_dev_ranges); i++) {
492 int bus;
493 int dev_base, dev_limit;
494
495 bus = bus_dev_ranges[i].bus;
496 dev_base = bus_dev_ranges[i].dev_base;
497 dev_limit = bus_dev_ranges[i].dev_limit;
498 for (slot = dev_base; slot < dev_limit; slot++) {
499 if (!early_is_k8_nb(read_pci_config(bus, slot, 3, 0x00)))
500 continue;
501
502
503
504
505 write_pci_config(bus, slot, 3, AMD64_GARTAPERTURECTL, aper_order << 1);
506 write_pci_config(bus, slot, 3, AMD64_GARTAPERTUREBASE, aper_alloc >> 25);
507 }
508 }
509
510 set_up_gart_resume(aper_order, aper_alloc);
511}
512