1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#define DEBUG
35
36#include <linux/types.h>
37
38#include <linux/stop_machine.h>
39#include <linux/kvm_para.h>
40#include <linux/uaccess.h>
41#include <linux/module.h>
42#include <linux/mutex.h>
43#include <linux/init.h>
44#include <linux/sort.h>
45#include <linux/cpu.h>
46#include <linux/pci.h>
47#include <linux/smp.h>
48#include <linux/syscore_ops.h>
49
50#include <asm/cpufeature.h>
51#include <asm/e820.h>
52#include <asm/mtrr.h>
53#include <asm/msr.h>
54#include <asm/pat.h>
55
56#include "mtrr.h"
57
58
59#define MTRR_TO_PHYS_WC_OFFSET 1000
60
61u32 num_var_ranges;
62static bool __mtrr_enabled;
63
64static bool mtrr_enabled(void)
65{
66 return __mtrr_enabled;
67}
68
69unsigned int mtrr_usage_table[MTRR_MAX_VAR_RANGES];
70static DEFINE_MUTEX(mtrr_mutex);
71
72u64 size_or_mask, size_and_mask;
73static bool mtrr_aps_delayed_init;
74
75static const struct mtrr_ops *mtrr_ops[X86_VENDOR_NUM];
76
77const struct mtrr_ops *mtrr_if;
78
79static void set_mtrr(unsigned int reg, unsigned long base,
80 unsigned long size, mtrr_type type);
81
82void set_mtrr_ops(const struct mtrr_ops *ops)
83{
84 if (ops->vendor && ops->vendor < X86_VENDOR_NUM)
85 mtrr_ops[ops->vendor] = ops;
86}
87
88
89static int have_wrcomb(void)
90{
91 struct pci_dev *dev;
92
93 dev = pci_get_class(PCI_CLASS_BRIDGE_HOST << 8, NULL);
94 if (dev != NULL) {
95
96
97
98
99
100 if (dev->vendor == PCI_VENDOR_ID_SERVERWORKS &&
101 dev->device == PCI_DEVICE_ID_SERVERWORKS_LE &&
102 dev->revision <= 5) {
103 pr_info("mtrr: Serverworks LE rev < 6 detected. Write-combining disabled.\n");
104 pci_dev_put(dev);
105 return 0;
106 }
107
108
109
110
111 if (dev->vendor == PCI_VENDOR_ID_INTEL &&
112 dev->device == PCI_DEVICE_ID_INTEL_82451NX) {
113 pr_info("mtrr: Intel 450NX MMC detected. Write-combining disabled.\n");
114 pci_dev_put(dev);
115 return 0;
116 }
117 pci_dev_put(dev);
118 }
119 return mtrr_if->have_wrcomb ? mtrr_if->have_wrcomb() : 0;
120}
121
122
123static void __init set_num_var_ranges(void)
124{
125 unsigned long config = 0, dummy;
126
127 if (use_intel())
128 rdmsr(MSR_MTRRcap, config, dummy);
129 else if (is_cpu(AMD))
130 config = 2;
131 else if (is_cpu(CYRIX) || is_cpu(CENTAUR))
132 config = 8;
133
134 num_var_ranges = config & 0xff;
135}
136
137static void __init init_table(void)
138{
139 int i, max;
140
141 max = num_var_ranges;
142 for (i = 0; i < max; i++)
143 mtrr_usage_table[i] = 1;
144}
145
146struct set_mtrr_data {
147 unsigned long smp_base;
148 unsigned long smp_size;
149 unsigned int smp_reg;
150 mtrr_type smp_type;
151};
152
153
154
155
156
157
158
159
160static int mtrr_rendezvous_handler(void *info)
161{
162 struct set_mtrr_data *data = info;
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177 if (data->smp_reg != ~0U) {
178 mtrr_if->set(data->smp_reg, data->smp_base,
179 data->smp_size, data->smp_type);
180 } else if (mtrr_aps_delayed_init || !cpu_online(smp_processor_id())) {
181 mtrr_if->set_all();
182 }
183 return 0;
184}
185
186static inline int types_compatible(mtrr_type type1, mtrr_type type2)
187{
188 return type1 == MTRR_TYPE_UNCACHABLE ||
189 type2 == MTRR_TYPE_UNCACHABLE ||
190 (type1 == MTRR_TYPE_WRTHROUGH && type2 == MTRR_TYPE_WRBACK) ||
191 (type1 == MTRR_TYPE_WRBACK && type2 == MTRR_TYPE_WRTHROUGH);
192}
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228static void
229set_mtrr(unsigned int reg, unsigned long base, unsigned long size, mtrr_type type)
230{
231 struct set_mtrr_data data = { .smp_reg = reg,
232 .smp_base = base,
233 .smp_size = size,
234 .smp_type = type
235 };
236
237 stop_machine(mtrr_rendezvous_handler, &data, cpu_online_mask);
238}
239
240static void set_mtrr_from_inactive_cpu(unsigned int reg, unsigned long base,
241 unsigned long size, mtrr_type type)
242{
243 struct set_mtrr_data data = { .smp_reg = reg,
244 .smp_base = base,
245 .smp_size = size,
246 .smp_type = type
247 };
248
249 stop_machine_from_inactive_cpu(mtrr_rendezvous_handler, &data,
250 cpu_callout_mask);
251}
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288int mtrr_add_page(unsigned long base, unsigned long size,
289 unsigned int type, bool increment)
290{
291 unsigned long lbase, lsize;
292 int i, replace, error;
293 mtrr_type ltype;
294
295 if (!mtrr_enabled())
296 return -ENXIO;
297
298 error = mtrr_if->validate_add_page(base, size, type);
299 if (error)
300 return error;
301
302 if (type >= MTRR_NUM_TYPES) {
303 pr_warn("mtrr: type: %u invalid\n", type);
304 return -EINVAL;
305 }
306
307
308 if ((type == MTRR_TYPE_WRCOMB) && !have_wrcomb()) {
309 pr_warn("mtrr: your processor doesn't support write-combining\n");
310 return -ENOSYS;
311 }
312
313 if (!size) {
314 pr_warn("mtrr: zero sized request\n");
315 return -EINVAL;
316 }
317
318 if ((base | (base + size - 1)) >>
319 (boot_cpu_data.x86_phys_bits - PAGE_SHIFT)) {
320 pr_warn("mtrr: base or size exceeds the MTRR width\n");
321 return -EINVAL;
322 }
323
324 error = -EINVAL;
325 replace = -1;
326
327
328 get_online_cpus();
329
330
331 mutex_lock(&mtrr_mutex);
332 for (i = 0; i < num_var_ranges; ++i) {
333 mtrr_if->get(i, &lbase, &lsize, <ype);
334 if (!lsize || base > lbase + lsize - 1 ||
335 base + size - 1 < lbase)
336 continue;
337
338
339
340
341 if (base < lbase || base + size - 1 > lbase + lsize - 1) {
342 if (base <= lbase &&
343 base + size - 1 >= lbase + lsize - 1) {
344
345 if (type == ltype) {
346 replace = replace == -1 ? i : -2;
347 continue;
348 } else if (types_compatible(type, ltype))
349 continue;
350 }
351 pr_warn("mtrr: 0x%lx000,0x%lx000 overlaps existing"
352 " 0x%lx000,0x%lx000\n", base, size, lbase,
353 lsize);
354 goto out;
355 }
356
357 if (ltype != type) {
358 if (types_compatible(type, ltype))
359 continue;
360 pr_warn("mtrr: type mismatch for %lx000,%lx000 old: %s new: %s\n",
361 base, size, mtrr_attrib_to_str(ltype),
362 mtrr_attrib_to_str(type));
363 goto out;
364 }
365 if (increment)
366 ++mtrr_usage_table[i];
367 error = i;
368 goto out;
369 }
370
371 i = mtrr_if->get_free_region(base, size, replace);
372 if (i >= 0) {
373 set_mtrr(i, base, size, type);
374 if (likely(replace < 0)) {
375 mtrr_usage_table[i] = 1;
376 } else {
377 mtrr_usage_table[i] = mtrr_usage_table[replace];
378 if (increment)
379 mtrr_usage_table[i]++;
380 if (unlikely(replace != i)) {
381 set_mtrr(replace, 0, 0, 0);
382 mtrr_usage_table[replace] = 0;
383 }
384 }
385 } else {
386 pr_info("mtrr: no more MTRRs available\n");
387 }
388 error = i;
389 out:
390 mutex_unlock(&mtrr_mutex);
391 put_online_cpus();
392 return error;
393}
394
395static int mtrr_check(unsigned long base, unsigned long size)
396{
397 if ((base & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) {
398 pr_warn("mtrr: size and base must be multiples of 4 kiB\n");
399 pr_debug("mtrr: size: 0x%lx base: 0x%lx\n", size, base);
400 dump_stack();
401 return -1;
402 }
403 return 0;
404}
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441int mtrr_add(unsigned long base, unsigned long size, unsigned int type,
442 bool increment)
443{
444 if (!mtrr_enabled())
445 return -ENODEV;
446 if (mtrr_check(base, size))
447 return -EINVAL;
448 return mtrr_add_page(base >> PAGE_SHIFT, size >> PAGE_SHIFT, type,
449 increment);
450}
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466int mtrr_del_page(int reg, unsigned long base, unsigned long size)
467{
468 int i, max;
469 mtrr_type ltype;
470 unsigned long lbase, lsize;
471 int error = -EINVAL;
472
473 if (!mtrr_enabled())
474 return -ENODEV;
475
476 max = num_var_ranges;
477
478 get_online_cpus();
479 mutex_lock(&mtrr_mutex);
480 if (reg < 0) {
481
482 for (i = 0; i < max; ++i) {
483 mtrr_if->get(i, &lbase, &lsize, <ype);
484 if (lbase == base && lsize == size) {
485 reg = i;
486 break;
487 }
488 }
489 if (reg < 0) {
490 pr_debug("mtrr: no MTRR for %lx000,%lx000 found\n",
491 base, size);
492 goto out;
493 }
494 }
495 if (reg >= max) {
496 pr_warn("mtrr: register: %d too big\n", reg);
497 goto out;
498 }
499 mtrr_if->get(reg, &lbase, &lsize, <ype);
500 if (lsize < 1) {
501 pr_warn("mtrr: MTRR %d not used\n", reg);
502 goto out;
503 }
504 if (mtrr_usage_table[reg] < 1) {
505 pr_warn("mtrr: reg: %d has count=0\n", reg);
506 goto out;
507 }
508 if (--mtrr_usage_table[reg] < 1)
509 set_mtrr(reg, 0, 0, 0);
510 error = reg;
511 out:
512 mutex_unlock(&mtrr_mutex);
513 put_online_cpus();
514 return error;
515}
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531int mtrr_del(int reg, unsigned long base, unsigned long size)
532{
533 if (!mtrr_enabled())
534 return -ENODEV;
535 if (mtrr_check(base, size))
536 return -EINVAL;
537 return mtrr_del_page(reg, base >> PAGE_SHIFT, size >> PAGE_SHIFT);
538}
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555int arch_phys_wc_add(unsigned long base, unsigned long size)
556{
557 int ret;
558
559 if (pat_enabled() || !mtrr_enabled())
560 return 0;
561
562 ret = mtrr_add(base, size, MTRR_TYPE_WRCOMB, true);
563 if (ret < 0) {
564 pr_warn("Failed to add WC MTRR for [%p-%p]; performance may suffer.",
565 (void *)base, (void *)(base + size - 1));
566 return ret;
567 }
568 return ret + MTRR_TO_PHYS_WC_OFFSET;
569}
570EXPORT_SYMBOL(arch_phys_wc_add);
571
572
573
574
575
576
577
578
579
580
581void arch_phys_wc_del(int handle)
582{
583 if (handle >= 1) {
584 WARN_ON(handle < MTRR_TO_PHYS_WC_OFFSET);
585 mtrr_del(handle - MTRR_TO_PHYS_WC_OFFSET, 0, 0);
586 }
587}
588EXPORT_SYMBOL(arch_phys_wc_del);
589
590
591
592
593
594
595
596
597
598
599
600
601int arch_phys_wc_index(int handle)
602{
603 if (handle < MTRR_TO_PHYS_WC_OFFSET)
604 return -1;
605 else
606 return handle - MTRR_TO_PHYS_WC_OFFSET;
607}
608EXPORT_SYMBOL_GPL(arch_phys_wc_index);
609
610
611
612
613
614
615static void __init init_ifs(void)
616{
617#ifndef CONFIG_X86_64
618 amd_init_mtrr();
619 cyrix_init_mtrr();
620 centaur_init_mtrr();
621#endif
622}
623
624
625
626
627struct mtrr_value {
628 mtrr_type ltype;
629 unsigned long lbase;
630 unsigned long lsize;
631};
632
633static struct mtrr_value mtrr_value[MTRR_MAX_VAR_RANGES];
634
635static int mtrr_save(void)
636{
637 int i;
638
639 for (i = 0; i < num_var_ranges; i++) {
640 mtrr_if->get(i, &mtrr_value[i].lbase,
641 &mtrr_value[i].lsize,
642 &mtrr_value[i].ltype);
643 }
644 return 0;
645}
646
647static void mtrr_restore(void)
648{
649 int i;
650
651 for (i = 0; i < num_var_ranges; i++) {
652 if (mtrr_value[i].lsize) {
653 set_mtrr(i, mtrr_value[i].lbase,
654 mtrr_value[i].lsize,
655 mtrr_value[i].ltype);
656 }
657 }
658}
659
660
661
662static struct syscore_ops mtrr_syscore_ops = {
663 .suspend = mtrr_save,
664 .resume = mtrr_restore,
665};
666
667int __initdata changed_by_mtrr_cleanup;
668
669#define SIZE_OR_MASK_BITS(n) (~((1ULL << ((n) - PAGE_SHIFT)) - 1))
670
671
672
673
674
675
676
677void __init mtrr_bp_init(void)
678{
679 u32 phys_addr;
680
681 init_ifs();
682
683 phys_addr = 32;
684
685 if (boot_cpu_has(X86_FEATURE_MTRR)) {
686 mtrr_if = &generic_mtrr_ops;
687 size_or_mask = SIZE_OR_MASK_BITS(36);
688 size_and_mask = 0x00f00000;
689 phys_addr = 36;
690
691
692
693
694
695
696 if (cpuid_eax(0x80000000) >= 0x80000008) {
697 phys_addr = cpuid_eax(0x80000008) & 0xff;
698
699 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
700 boot_cpu_data.x86 == 0xF &&
701 boot_cpu_data.x86_model == 0x3 &&
702 (boot_cpu_data.x86_mask == 0x3 ||
703 boot_cpu_data.x86_mask == 0x4))
704 phys_addr = 36;
705
706 size_or_mask = SIZE_OR_MASK_BITS(phys_addr);
707 size_and_mask = ~size_or_mask & 0xfffff00000ULL;
708 } else if (boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR &&
709 boot_cpu_data.x86 == 6) {
710
711
712
713
714 size_or_mask = SIZE_OR_MASK_BITS(32);
715 size_and_mask = 0;
716 phys_addr = 32;
717 }
718 } else {
719 switch (boot_cpu_data.x86_vendor) {
720 case X86_VENDOR_AMD:
721 if (cpu_feature_enabled(X86_FEATURE_K6_MTRR)) {
722
723 mtrr_if = mtrr_ops[X86_VENDOR_AMD];
724 size_or_mask = SIZE_OR_MASK_BITS(32);
725 size_and_mask = 0;
726 }
727 break;
728 case X86_VENDOR_CENTAUR:
729 if (cpu_feature_enabled(X86_FEATURE_CENTAUR_MCR)) {
730 mtrr_if = mtrr_ops[X86_VENDOR_CENTAUR];
731 size_or_mask = SIZE_OR_MASK_BITS(32);
732 size_and_mask = 0;
733 }
734 break;
735 case X86_VENDOR_CYRIX:
736 if (cpu_feature_enabled(X86_FEATURE_CYRIX_ARR)) {
737 mtrr_if = mtrr_ops[X86_VENDOR_CYRIX];
738 size_or_mask = SIZE_OR_MASK_BITS(32);
739 size_and_mask = 0;
740 }
741 break;
742 default:
743 break;
744 }
745 }
746
747 if (mtrr_if) {
748 __mtrr_enabled = true;
749 set_num_var_ranges();
750 init_table();
751 if (use_intel()) {
752
753 __mtrr_enabled = get_mtrr_state();
754
755 if (mtrr_cleanup(phys_addr)) {
756 changed_by_mtrr_cleanup = 1;
757 mtrr_if->set_all();
758 }
759 }
760 }
761
762 if (!mtrr_enabled())
763 pr_info("MTRR: Disabled\n");
764}
765
766void mtrr_ap_init(void)
767{
768 if (!mtrr_enabled())
769 return;
770
771 if (!use_intel() || mtrr_aps_delayed_init)
772 return;
773
774
775
776
777
778
779
780
781
782
783
784
785
786 set_mtrr_from_inactive_cpu(~0U, 0, 0, 0);
787}
788
789
790
791
792void mtrr_save_state(void)
793{
794 int first_cpu;
795
796 if (!mtrr_enabled())
797 return;
798
799 get_online_cpus();
800 first_cpu = cpumask_first(cpu_online_mask);
801 smp_call_function_single(first_cpu, mtrr_save_fixed_ranges, NULL, 1);
802 put_online_cpus();
803}
804
805void set_mtrr_aps_delayed_init(void)
806{
807 if (!mtrr_enabled())
808 return;
809 if (!use_intel())
810 return;
811
812 mtrr_aps_delayed_init = true;
813}
814
815
816
817
818void mtrr_aps_init(void)
819{
820 if (!use_intel() || !mtrr_enabled())
821 return;
822
823
824
825
826
827
828 if (!mtrr_aps_delayed_init)
829 return;
830
831 set_mtrr(~0U, 0, 0, 0);
832 mtrr_aps_delayed_init = false;
833}
834
835void mtrr_bp_restore(void)
836{
837 if (!use_intel() || !mtrr_enabled())
838 return;
839
840 mtrr_if->set_all();
841}
842
843static int __init mtrr_init_finialize(void)
844{
845 if (!mtrr_enabled())
846 return 0;
847
848 if (use_intel()) {
849 if (!changed_by_mtrr_cleanup)
850 mtrr_state_warn();
851 return 0;
852 }
853
854
855
856
857
858
859
860
861
862 register_syscore_ops(&mtrr_syscore_ops);
863
864 return 0;
865}
866subsys_initcall(mtrr_init_finialize);
867