1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#define DEBUG
35
36#include <linux/types.h>
37
38#include <linux/kvm_para.h>
39#include <linux/uaccess.h>
40#include <linux/module.h>
41#include <linux/mutex.h>
42#include <linux/init.h>
43#include <linux/sort.h>
44#include <linux/cpu.h>
45#include <linux/pci.h>
46#include <linux/smp.h>
47
48#include <asm/processor.h>
49#include <asm/e820.h>
50#include <asm/mtrr.h>
51#include <asm/msr.h>
52
53#include "mtrr.h"
54
55u32 num_var_ranges;
56
57unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];
58static DEFINE_MUTEX(mtrr_mutex);
59
60u64 size_or_mask, size_and_mask;
61static bool mtrr_aps_delayed_init;
62
63static struct mtrr_ops *mtrr_ops[X86_VENDOR_NUM];
64
65struct mtrr_ops *mtrr_if;
66
67static void set_mtrr(unsigned int reg, unsigned long base,
68 unsigned long size, mtrr_type type);
69
70void set_mtrr_ops(struct mtrr_ops *ops)
71{
72 if (ops->vendor && ops->vendor < X86_VENDOR_NUM)
73 mtrr_ops[ops->vendor] = ops;
74}
75
76
77static int have_wrcomb(void)
78{
79 struct pci_dev *dev;
80 u8 rev;
81
82 dev = pci_get_class(PCI_CLASS_BRIDGE_HOST << 8, NULL);
83 if (dev != NULL) {
84
85
86
87
88
89 if (dev->vendor == PCI_VENDOR_ID_SERVERWORKS &&
90 dev->device == PCI_DEVICE_ID_SERVERWORKS_LE) {
91 pci_read_config_byte(dev, PCI_CLASS_REVISION, &rev);
92 if (rev <= 5) {
93 pr_info("mtrr: Serverworks LE rev < 6 detected. Write-combining disabled.\n");
94 pci_dev_put(dev);
95 return 0;
96 }
97 }
98
99
100
101
102 if (dev->vendor == PCI_VENDOR_ID_INTEL &&
103 dev->device == PCI_DEVICE_ID_INTEL_82451NX) {
104 pr_info("mtrr: Intel 450NX MMC detected. Write-combining disabled.\n");
105 pci_dev_put(dev);
106 return 0;
107 }
108 pci_dev_put(dev);
109 }
110 return mtrr_if->have_wrcomb ? mtrr_if->have_wrcomb() : 0;
111}
112
113
114static void __init set_num_var_ranges(void)
115{
116 unsigned long config = 0, dummy;
117
118 if (use_intel())
119 rdmsr(MSR_MTRRcap, config, dummy);
120 else if (is_cpu(AMD))
121 config = 2;
122 else if (is_cpu(CYRIX) || is_cpu(CENTAUR))
123 config = 8;
124
125 num_var_ranges = config & 0xff;
126}
127
128static void __init init_table(void)
129{
130 int i, max;
131
132 max = num_var_ranges;
133 for (i = 0; i < max; i++)
134 mtrr_usage_table[i] = 1;
135}
136
137struct set_mtrr_data {
138 atomic_t count;
139 atomic_t gate;
140 unsigned long smp_base;
141 unsigned long smp_size;
142 unsigned int smp_reg;
143 mtrr_type smp_type;
144};
145
146
147
148
149
150
151static void ipi_handler(void *info)
152{
153#ifdef CONFIG_SMP
154 struct set_mtrr_data *data = info;
155 unsigned long flags;
156
157 local_irq_save(flags);
158
159 atomic_dec(&data->count);
160 while (!atomic_read(&data->gate))
161 cpu_relax();
162
163
164 if (data->smp_reg != ~0U) {
165 mtrr_if->set(data->smp_reg, data->smp_base,
166 data->smp_size, data->smp_type);
167 } else if (mtrr_aps_delayed_init) {
168
169
170
171 mtrr_if->set_all();
172 }
173
174 atomic_dec(&data->count);
175 while (atomic_read(&data->gate))
176 cpu_relax();
177
178 atomic_dec(&data->count);
179 local_irq_restore(flags);
180#endif
181}
182
183static inline int types_compatible(mtrr_type type1, mtrr_type type2)
184{
185 return type1 == MTRR_TYPE_UNCACHABLE ||
186 type2 == MTRR_TYPE_UNCACHABLE ||
187 (type1 == MTRR_TYPE_WRTHROUGH && type2 == MTRR_TYPE_WRBACK) ||
188 (type1 == MTRR_TYPE_WRBACK && type2 == MTRR_TYPE_WRTHROUGH);
189}
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231static void
232set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type)
233{
234 struct set_mtrr_data data;
235 unsigned long flags;
236
237 data.smp_reg = reg;
238 data.smp_base = base;
239 data.smp_size = size;
240 data.smp_type = type;
241 atomic_set(&data.count, num_booting_cpus() - 1);
242
243
244 smp_wmb();
245 atomic_set(&data.gate, 0);
246
247
248 if (smp_call_function(ipi_handler, &data, 0) != 0)
249 panic("mtrr: timed out waiting for other CPUs\n");
250
251 local_irq_save(flags);
252
253 while (atomic_read(&data.count))
254 cpu_relax();
255
256
257 atomic_set(&data.count, num_booting_cpus() - 1);
258 smp_wmb();
259 atomic_set(&data.gate, 1);
260
261
262
263
264
265
266
267
268
269
270 if (reg != ~0U)
271 mtrr_if->set(reg, base, size, type);
272 else if (!mtrr_aps_delayed_init)
273 mtrr_if->set_all();
274
275
276 while (atomic_read(&data.count))
277 cpu_relax();
278
279 atomic_set(&data.count, num_booting_cpus() - 1);
280 smp_wmb();
281 atomic_set(&data.gate, 0);
282
283
284
285
286
287 while (atomic_read(&data.count))
288 cpu_relax();
289
290 local_irq_restore(flags);
291}
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328int mtrr_add_page(unsigned long base, unsigned long size,
329 unsigned int type, bool increment)
330{
331 unsigned long lbase, lsize;
332 int i, replace, error;
333 mtrr_type ltype;
334
335 if (!mtrr_if)
336 return -ENXIO;
337
338 error = mtrr_if->validate_add_page(base, size, type);
339 if (error)
340 return error;
341
342 if (type >= MTRR_NUM_TYPES) {
343 pr_warning("mtrr: type: %u invalid\n", type);
344 return -EINVAL;
345 }
346
347
348 if ((type == MTRR_TYPE_WRCOMB) && !have_wrcomb()) {
349 pr_warning("mtrr: your processor doesn't support write-combining\n");
350 return -ENOSYS;
351 }
352
353 if (!size) {
354 pr_warning("mtrr: zero sized request\n");
355 return -EINVAL;
356 }
357
358 if (base & size_or_mask || size & size_or_mask) {
359 pr_warning("mtrr: base or size exceeds the MTRR width\n");
360 return -EINVAL;
361 }
362
363 error = -EINVAL;
364 replace = -1;
365
366
367 get_online_cpus();
368
369
370 mutex_lock(&mtrr_mutex);
371 for (i = 0; i < num_var_ranges; ++i) {
372 mtrr_if->get(i, &lbase, &lsize, <ype);
373 if (!lsize || base > lbase + lsize - 1 ||
374 base + size - 1 < lbase)
375 continue;
376
377
378
379
380 if (base < lbase || base + size - 1 > lbase + lsize - 1) {
381 if (base <= lbase &&
382 base + size - 1 >= lbase + lsize - 1) {
383
384 if (type == ltype) {
385 replace = replace == -1 ? i : -2;
386 continue;
387 } else if (types_compatible(type, ltype))
388 continue;
389 }
390 pr_warning("mtrr: 0x%lx000,0x%lx000 overlaps existing"
391 " 0x%lx000,0x%lx000\n", base, size, lbase,
392 lsize);
393 goto out;
394 }
395
396 if (ltype != type) {
397 if (types_compatible(type, ltype))
398 continue;
399 pr_warning("mtrr: type mismatch for %lx000,%lx000 old: %s new: %s\n",
400 base, size, mtrr_attrib_to_str(ltype),
401 mtrr_attrib_to_str(type));
402 goto out;
403 }
404 if (increment)
405 ++mtrr_usage_table[i];
406 error = i;
407 goto out;
408 }
409
410 i = mtrr_if->get_free_region(base, size, replace);
411 if (i >= 0) {
412 set_mtrr(i, base, size, type);
413 if (likely(replace < 0)) {
414 mtrr_usage_table[i] = 1;
415 } else {
416 mtrr_usage_table[i] = mtrr_usage_table[replace];
417 if (increment)
418 mtrr_usage_table[i]++;
419 if (unlikely(replace != i)) {
420 set_mtrr(replace, 0, 0, 0);
421 mtrr_usage_table[replace] = 0;
422 }
423 }
424 } else {
425 pr_info("mtrr: no more MTRRs available\n");
426 }
427 error = i;
428 out:
429 mutex_unlock(&mtrr_mutex);
430 put_online_cpus();
431 return error;
432}
433
434static int mtrr_check(unsigned long base, unsigned long size)
435{
436 if ((base & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) {
437 pr_warning("mtrr: size and base must be multiples of 4 kiB\n");
438 pr_debug("mtrr: size: 0x%lx base: 0x%lx\n", size, base);
439 dump_stack();
440 return -1;
441 }
442 return 0;
443}
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480int mtrr_add(unsigned long base, unsigned long size, unsigned int type,
481 bool increment)
482{
483 if (mtrr_check(base, size))
484 return -EINVAL;
485 return mtrr_add_page(base >> PAGE_SHIFT, size >> PAGE_SHIFT, type,
486 increment);
487}
488EXPORT_SYMBOL(mtrr_add);
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504int mtrr_del_page(int reg, unsigned long base, unsigned long size)
505{
506 int i, max;
507 mtrr_type ltype;
508 unsigned long lbase, lsize;
509 int error = -EINVAL;
510
511 if (!mtrr_if)
512 return -ENXIO;
513
514 max = num_var_ranges;
515
516 get_online_cpus();
517 mutex_lock(&mtrr_mutex);
518 if (reg < 0) {
519
520 for (i = 0; i < max; ++i) {
521 mtrr_if->get(i, &lbase, &lsize, <ype);
522 if (lbase == base && lsize == size) {
523 reg = i;
524 break;
525 }
526 }
527 if (reg < 0) {
528 pr_debug("mtrr: no MTRR for %lx000,%lx000 found\n",
529 base, size);
530 goto out;
531 }
532 }
533 if (reg >= max) {
534 pr_warning("mtrr: register: %d too big\n", reg);
535 goto out;
536 }
537 mtrr_if->get(reg, &lbase, &lsize, <ype);
538 if (lsize < 1) {
539 pr_warning("mtrr: MTRR %d not used\n", reg);
540 goto out;
541 }
542 if (mtrr_usage_table[reg] < 1) {
543 pr_warning("mtrr: reg: %d has count=0\n", reg);
544 goto out;
545 }
546 if (--mtrr_usage_table[reg] < 1)
547 set_mtrr(reg, 0, 0, 0);
548 error = reg;
549 out:
550 mutex_unlock(&mtrr_mutex);
551 put_online_cpus();
552 return error;
553}
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569int mtrr_del(int reg, unsigned long base, unsigned long size)
570{
571 if (mtrr_check(base, size))
572 return -EINVAL;
573 return mtrr_del_page(reg, base >> PAGE_SHIFT, size >> PAGE_SHIFT);
574}
575EXPORT_SYMBOL(mtrr_del);
576
577
578
579
580
581
582static void __init init_ifs(void)
583{
584#ifndef CONFIG_X86_64
585 amd_init_mtrr();
586 cyrix_init_mtrr();
587 centaur_init_mtrr();
588#endif
589}
590
591
592
593
594struct mtrr_value {
595 mtrr_type ltype;
596 unsigned long lbase;
597 unsigned long lsize;
598};
599
600static struct mtrr_value mtrr_value[MTRR_MAX_VAR_RANGES];
601
602static int mtrr_save(struct sys_device *sysdev, pm_message_t state)
603{
604 int i;
605
606 for (i = 0; i < num_var_ranges; i++) {
607 mtrr_if->get(i, &mtrr_value[i].lbase,
608 &mtrr_value[i].lsize,
609 &mtrr_value[i].ltype);
610 }
611 return 0;
612}
613
614static int mtrr_restore(struct sys_device *sysdev)
615{
616 int i;
617
618 for (i = 0; i < num_var_ranges; i++) {
619 if (mtrr_value[i].lsize) {
620 set_mtrr(i, mtrr_value[i].lbase,
621 mtrr_value[i].lsize,
622 mtrr_value[i].ltype);
623 }
624 }
625 return 0;
626}
627
628
629
630static struct sysdev_driver mtrr_sysdev_driver = {
631 .suspend = mtrr_save,
632 .resume = mtrr_restore,
633};
634
635int __initdata changed_by_mtrr_cleanup;
636
637
638
639
640
641
642
643
644void __init mtrr_bp_init(void)
645{
646 u32 phys_addr;
647
648 init_ifs();
649
650 phys_addr = 32;
651
652 if (cpu_has_mtrr) {
653 mtrr_if = &generic_mtrr_ops;
654 size_or_mask = 0xff000000;
655 size_and_mask = 0x00f00000;
656 phys_addr = 36;
657
658
659
660
661
662
663 if (cpuid_eax(0x80000000) >= 0x80000008) {
664 phys_addr = cpuid_eax(0x80000008) & 0xff;
665
666 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
667 boot_cpu_data.x86 == 0xF &&
668 boot_cpu_data.x86_model == 0x3 &&
669 (boot_cpu_data.x86_mask == 0x3 ||
670 boot_cpu_data.x86_mask == 0x4))
671 phys_addr = 36;
672
673 size_or_mask = ~((1ULL << (phys_addr - PAGE_SHIFT)) - 1);
674 size_and_mask = ~size_or_mask & 0xfffff00000ULL;
675 } else if (boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR &&
676 boot_cpu_data.x86 == 6) {
677
678
679
680
681 size_or_mask = 0xfff00000;
682 size_and_mask = 0;
683 phys_addr = 32;
684 }
685 } else {
686 switch (boot_cpu_data.x86_vendor) {
687 case X86_VENDOR_AMD:
688 if (cpu_has_k6_mtrr) {
689
690 mtrr_if = mtrr_ops[X86_VENDOR_AMD];
691 size_or_mask = 0xfff00000;
692 size_and_mask = 0;
693 }
694 break;
695 case X86_VENDOR_CENTAUR:
696 if (cpu_has_centaur_mcr) {
697 mtrr_if = mtrr_ops[X86_VENDOR_CENTAUR];
698 size_or_mask = 0xfff00000;
699 size_and_mask = 0;
700 }
701 break;
702 case X86_VENDOR_CYRIX:
703 if (cpu_has_cyrix_arr) {
704 mtrr_if = mtrr_ops[X86_VENDOR_CYRIX];
705 size_or_mask = 0xfff00000;
706 size_and_mask = 0;
707 }
708 break;
709 default:
710 break;
711 }
712 }
713
714 if (mtrr_if) {
715 set_num_var_ranges();
716 init_table();
717 if (use_intel()) {
718 get_mtrr_state();
719
720 if (mtrr_cleanup(phys_addr)) {
721 changed_by_mtrr_cleanup = 1;
722 mtrr_if->set_all();
723 }
724 }
725 }
726}
727
728void mtrr_ap_init(void)
729{
730 if (!use_intel() || mtrr_aps_delayed_init)
731 return;
732
733
734
735
736
737
738
739
740
741
742
743
744
745 set_mtrr(~0U, 0, 0, 0);
746}
747
748
749
750
751void mtrr_save_state(void)
752{
753 smp_call_function_single(0, mtrr_save_fixed_ranges, NULL, 1);
754}
755
756void set_mtrr_aps_delayed_init(void)
757{
758 if (!use_intel())
759 return;
760
761 mtrr_aps_delayed_init = true;
762}
763
764
765
766
767void mtrr_aps_init(void)
768{
769 if (!use_intel())
770 return;
771
772 set_mtrr(~0U, 0, 0, 0);
773 mtrr_aps_delayed_init = false;
774}
775
776void mtrr_bp_restore(void)
777{
778 if (!use_intel())
779 return;
780
781 mtrr_if->set_all();
782}
783
784static int __init mtrr_init_finialize(void)
785{
786 if (!mtrr_if)
787 return 0;
788
789 if (use_intel()) {
790 if (!changed_by_mtrr_cleanup)
791 mtrr_state_warn();
792 return 0;
793 }
794
795
796
797
798
799
800
801
802
803 sysdev_driver_register(&cpu_sysdev_class, &mtrr_sysdev_driver);
804
805 return 0;
806}
807subsys_initcall(mtrr_init_finialize);
808