1
2
3
4
5
6
7
8
9
10#include <linux/seq_file.h>
11#include <linux/bootmem.h>
12#include <linux/debugfs.h>
13#include <linux/ioport.h>
14#include <linux/kernel.h>
15#include <linux/pfn_t.h>
16#include <linux/slab.h>
17#include <linux/mm.h>
18#include <linux/fs.h>
19#include <linux/rbtree.h>
20
21#include <asm/cacheflush.h>
22#include <asm/processor.h>
23#include <asm/tlbflush.h>
24#include <asm/x86_init.h>
25#include <asm/pgtable.h>
26#include <asm/fcntl.h>
27#include <asm/e820/api.h>
28#include <asm/mtrr.h>
29#include <asm/page.h>
30#include <asm/msr.h>
31#include <asm/pat.h>
32#include <asm/io.h>
33
34#include "pat_internal.h"
35#include "mm_internal.h"
36
37#undef pr_fmt
38#define pr_fmt(fmt) "" fmt
39
40static bool __read_mostly boot_cpu_done;
41static bool __read_mostly pat_disabled = !IS_ENABLED(CONFIG_X86_PAT);
42static bool __read_mostly pat_initialized;
43static bool __read_mostly init_cm_done;
44
45void pat_disable(const char *reason)
46{
47 if (pat_disabled)
48 return;
49
50 if (boot_cpu_done) {
51 WARN_ONCE(1, "x86/PAT: PAT cannot be disabled after initialization\n");
52 return;
53 }
54
55 pat_disabled = true;
56 pr_info("x86/PAT: %s\n", reason);
57}
58
59static int __init nopat(char *str)
60{
61 pat_disable("PAT support disabled.");
62 return 0;
63}
64early_param("nopat", nopat);
65
66bool pat_enabled(void)
67{
68 return pat_initialized;
69}
70EXPORT_SYMBOL_GPL(pat_enabled);
71
72int pat_debug_enable;
73
74static int __init pat_debug_setup(char *str)
75{
76 pat_debug_enable = 1;
77 return 0;
78}
79__setup("debugpat", pat_debug_setup);
80
81#ifdef CONFIG_X86_PAT
82
83
84
85
86
87
88
89
90
91
92
93
94
95#define _PGMT_WB 0
96#define _PGMT_WC (1UL << PG_arch_1)
97#define _PGMT_UC_MINUS (1UL << PG_uncached)
98#define _PGMT_WT (1UL << PG_uncached | 1UL << PG_arch_1)
99#define _PGMT_MASK (1UL << PG_uncached | 1UL << PG_arch_1)
100#define _PGMT_CLEAR_MASK (~_PGMT_MASK)
101
102static inline enum page_cache_mode get_page_memtype(struct page *pg)
103{
104 unsigned long pg_flags = pg->flags & _PGMT_MASK;
105
106 if (pg_flags == _PGMT_WB)
107 return _PAGE_CACHE_MODE_WB;
108 else if (pg_flags == _PGMT_WC)
109 return _PAGE_CACHE_MODE_WC;
110 else if (pg_flags == _PGMT_UC_MINUS)
111 return _PAGE_CACHE_MODE_UC_MINUS;
112 else
113 return _PAGE_CACHE_MODE_WT;
114}
115
116static inline void set_page_memtype(struct page *pg,
117 enum page_cache_mode memtype)
118{
119 unsigned long memtype_flags;
120 unsigned long old_flags;
121 unsigned long new_flags;
122
123 switch (memtype) {
124 case _PAGE_CACHE_MODE_WC:
125 memtype_flags = _PGMT_WC;
126 break;
127 case _PAGE_CACHE_MODE_UC_MINUS:
128 memtype_flags = _PGMT_UC_MINUS;
129 break;
130 case _PAGE_CACHE_MODE_WT:
131 memtype_flags = _PGMT_WT;
132 break;
133 case _PAGE_CACHE_MODE_WB:
134 default:
135 memtype_flags = _PGMT_WB;
136 break;
137 }
138
139 do {
140 old_flags = pg->flags;
141 new_flags = (old_flags & _PGMT_CLEAR_MASK) | memtype_flags;
142 } while (cmpxchg(&pg->flags, old_flags, new_flags) != old_flags);
143}
144#else
145static inline enum page_cache_mode get_page_memtype(struct page *pg)
146{
147 return -1;
148}
149static inline void set_page_memtype(struct page *pg,
150 enum page_cache_mode memtype)
151{
152}
153#endif
154
155enum {
156 PAT_UC = 0,
157 PAT_WC = 1,
158 PAT_WT = 4,
159 PAT_WP = 5,
160 PAT_WB = 6,
161 PAT_UC_MINUS = 7,
162};
163
164#define CM(c) (_PAGE_CACHE_MODE_ ## c)
165
166static enum page_cache_mode pat_get_cache_mode(unsigned pat_val, char *msg)
167{
168 enum page_cache_mode cache;
169 char *cache_mode;
170
171 switch (pat_val) {
172 case PAT_UC: cache = CM(UC); cache_mode = "UC "; break;
173 case PAT_WC: cache = CM(WC); cache_mode = "WC "; break;
174 case PAT_WT: cache = CM(WT); cache_mode = "WT "; break;
175 case PAT_WP: cache = CM(WP); cache_mode = "WP "; break;
176 case PAT_WB: cache = CM(WB); cache_mode = "WB "; break;
177 case PAT_UC_MINUS: cache = CM(UC_MINUS); cache_mode = "UC- "; break;
178 default: cache = CM(WB); cache_mode = "WB "; break;
179 }
180
181 memcpy(msg, cache_mode, 4);
182
183 return cache;
184}
185
186#undef CM
187
188
189
190
191
192
193static void __init_cache_modes(u64 pat)
194{
195 enum page_cache_mode cache;
196 char pat_msg[33];
197 int i;
198
199 pat_msg[32] = 0;
200 for (i = 7; i >= 0; i--) {
201 cache = pat_get_cache_mode((pat >> (i * 8)) & 7,
202 pat_msg + 4 * i);
203 update_cache_mode_entry(i, cache);
204 }
205 pr_info("x86/PAT: Configuration [0-7]: %s\n", pat_msg);
206
207 init_cm_done = true;
208}
209
210#define PAT(x, y) ((u64)PAT_ ## y << ((x)*8))
211
212static void pat_bsp_init(u64 pat)
213{
214 u64 tmp_pat;
215
216 if (!boot_cpu_has(X86_FEATURE_PAT)) {
217 pat_disable("PAT not supported by CPU.");
218 return;
219 }
220
221 rdmsrl(MSR_IA32_CR_PAT, tmp_pat);
222 if (!tmp_pat) {
223 pat_disable("PAT MSR is 0, disabled.");
224 return;
225 }
226
227 wrmsrl(MSR_IA32_CR_PAT, pat);
228 pat_initialized = true;
229
230 __init_cache_modes(pat);
231}
232
233static void pat_ap_init(u64 pat)
234{
235 if (!boot_cpu_has(X86_FEATURE_PAT)) {
236
237
238
239
240 panic("x86/PAT: PAT enabled, but not supported by secondary CPU\n");
241 }
242
243 wrmsrl(MSR_IA32_CR_PAT, pat);
244}
245
246void init_cache_modes(void)
247{
248 u64 pat = 0;
249
250 if (init_cm_done)
251 return;
252
253 if (boot_cpu_has(X86_FEATURE_PAT)) {
254
255
256
257
258
259
260
261
262
263 rdmsrl(MSR_IA32_CR_PAT, pat);
264 }
265
266 if (!pat) {
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285 pat = PAT(0, WB) | PAT(1, WT) | PAT(2, UC_MINUS) | PAT(3, UC) |
286 PAT(4, WB) | PAT(5, WT) | PAT(6, UC_MINUS) | PAT(7, UC);
287 }
288
289 __init_cache_modes(pat);
290}
291
292
293
294
295
296
297
298
299
300
301
302void pat_init(void)
303{
304 u64 pat;
305 struct cpuinfo_x86 *c = &boot_cpu_data;
306
307 if (pat_disabled)
308 return;
309
310 if ((c->x86_vendor == X86_VENDOR_INTEL) &&
311 (((c->x86 == 0x6) && (c->x86_model <= 0xd)) ||
312 ((c->x86 == 0xf) && (c->x86_model <= 0x6)))) {
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333 pat = PAT(0, WB) | PAT(1, WC) | PAT(2, UC_MINUS) | PAT(3, UC) |
334 PAT(4, WB) | PAT(5, WC) | PAT(6, UC_MINUS) | PAT(7, UC);
335 } else {
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362 pat = PAT(0, WB) | PAT(1, WC) | PAT(2, UC_MINUS) | PAT(3, UC) |
363 PAT(4, WB) | PAT(5, WP) | PAT(6, UC_MINUS) | PAT(7, WT);
364 }
365
366 if (!boot_cpu_done) {
367 pat_bsp_init(pat);
368 boot_cpu_done = true;
369 } else {
370 pat_ap_init(pat);
371 }
372}
373
374#undef PAT
375
376static DEFINE_SPINLOCK(memtype_lock);
377
378
379
380
381
382
383
384
385static unsigned long pat_x_mtrr_type(u64 start, u64 end,
386 enum page_cache_mode req_type)
387{
388
389
390
391
392 if (req_type == _PAGE_CACHE_MODE_WB) {
393 u8 mtrr_type, uniform;
394
395 mtrr_type = mtrr_type_lookup(start, end, &uniform);
396 if (mtrr_type != MTRR_TYPE_WRBACK)
397 return _PAGE_CACHE_MODE_UC_MINUS;
398
399 return _PAGE_CACHE_MODE_WB;
400 }
401
402 return req_type;
403}
404
405struct pagerange_state {
406 unsigned long cur_pfn;
407 int ram;
408 int not_ram;
409};
410
411static int
412pagerange_is_ram_callback(unsigned long initial_pfn, unsigned long total_nr_pages, void *arg)
413{
414 struct pagerange_state *state = arg;
415
416 state->not_ram |= initial_pfn > state->cur_pfn;
417 state->ram |= total_nr_pages > 0;
418 state->cur_pfn = initial_pfn + total_nr_pages;
419
420 return state->ram && state->not_ram;
421}
422
423static int pat_pagerange_is_ram(resource_size_t start, resource_size_t end)
424{
425 int ret = 0;
426 unsigned long start_pfn = start >> PAGE_SHIFT;
427 unsigned long end_pfn = (end + PAGE_SIZE - 1) >> PAGE_SHIFT;
428 struct pagerange_state state = {start_pfn, 0, 0};
429
430
431
432
433
434
435
436
437 if (start_pfn < ISA_END_ADDRESS >> PAGE_SHIFT)
438 start_pfn = ISA_END_ADDRESS >> PAGE_SHIFT;
439
440 if (start_pfn < end_pfn) {
441 ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn,
442 &state, pagerange_is_ram_callback);
443 }
444
445 return (ret > 0) ? -1 : (state.ram ? 1 : 0);
446}
447
448
449
450
451
452
453
454
455
456
457
458
459static int reserve_ram_pages_type(u64 start, u64 end,
460 enum page_cache_mode req_type,
461 enum page_cache_mode *new_type)
462{
463 struct page *page;
464 u64 pfn;
465
466 if (req_type == _PAGE_CACHE_MODE_WP) {
467 if (new_type)
468 *new_type = _PAGE_CACHE_MODE_UC_MINUS;
469 return -EINVAL;
470 }
471
472 if (req_type == _PAGE_CACHE_MODE_UC) {
473
474 WARN_ON_ONCE(1);
475 req_type = _PAGE_CACHE_MODE_UC_MINUS;
476 }
477
478 for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
479 enum page_cache_mode type;
480
481 page = pfn_to_page(pfn);
482 type = get_page_memtype(page);
483 if (type != _PAGE_CACHE_MODE_WB) {
484 pr_info("x86/PAT: reserve_ram_pages_type failed [mem %#010Lx-%#010Lx], track 0x%x, req 0x%x\n",
485 start, end - 1, type, req_type);
486 if (new_type)
487 *new_type = type;
488
489 return -EBUSY;
490 }
491 }
492
493 if (new_type)
494 *new_type = req_type;
495
496 for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
497 page = pfn_to_page(pfn);
498 set_page_memtype(page, req_type);
499 }
500 return 0;
501}
502
503static int free_ram_pages_type(u64 start, u64 end)
504{
505 struct page *page;
506 u64 pfn;
507
508 for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
509 page = pfn_to_page(pfn);
510 set_page_memtype(page, _PAGE_CACHE_MODE_WB);
511 }
512 return 0;
513}
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528int reserve_memtype(u64 start, u64 end, enum page_cache_mode req_type,
529 enum page_cache_mode *new_type)
530{
531 struct memtype *new;
532 enum page_cache_mode actual_type;
533 int is_range_ram;
534 int err = 0;
535
536 BUG_ON(start >= end);
537
538 if (!pat_enabled()) {
539
540 if (new_type)
541 *new_type = req_type;
542 return 0;
543 }
544
545
546 if (x86_platform.is_untracked_pat_range(start, end)) {
547 if (new_type)
548 *new_type = _PAGE_CACHE_MODE_WB;
549 return 0;
550 }
551
552
553
554
555
556
557
558 actual_type = pat_x_mtrr_type(start, end, req_type);
559
560 if (new_type)
561 *new_type = actual_type;
562
563 is_range_ram = pat_pagerange_is_ram(start, end);
564 if (is_range_ram == 1) {
565
566 err = reserve_ram_pages_type(start, end, req_type, new_type);
567
568 return err;
569 } else if (is_range_ram < 0) {
570 return -EINVAL;
571 }
572
573 new = kzalloc(sizeof(struct memtype), GFP_KERNEL);
574 if (!new)
575 return -ENOMEM;
576
577 new->start = start;
578 new->end = end;
579 new->type = actual_type;
580
581 spin_lock(&memtype_lock);
582
583 err = rbt_memtype_check_insert(new, new_type);
584 if (err) {
585 pr_info("x86/PAT: reserve_memtype failed [mem %#010Lx-%#010Lx], track %s, req %s\n",
586 start, end - 1,
587 cattr_name(new->type), cattr_name(req_type));
588 kfree(new);
589 spin_unlock(&memtype_lock);
590
591 return err;
592 }
593
594 spin_unlock(&memtype_lock);
595
596 dprintk("reserve_memtype added [mem %#010Lx-%#010Lx], track %s, req %s, ret %s\n",
597 start, end - 1, cattr_name(new->type), cattr_name(req_type),
598 new_type ? cattr_name(*new_type) : "-");
599
600 return err;
601}
602
603int free_memtype(u64 start, u64 end)
604{
605 int err = -EINVAL;
606 int is_range_ram;
607 struct memtype *entry;
608
609 if (!pat_enabled())
610 return 0;
611
612
613 if (x86_platform.is_untracked_pat_range(start, end))
614 return 0;
615
616 is_range_ram = pat_pagerange_is_ram(start, end);
617 if (is_range_ram == 1) {
618
619 err = free_ram_pages_type(start, end);
620
621 return err;
622 } else if (is_range_ram < 0) {
623 return -EINVAL;
624 }
625
626 spin_lock(&memtype_lock);
627 entry = rbt_memtype_erase(start, end);
628 spin_unlock(&memtype_lock);
629
630 if (IS_ERR(entry)) {
631 pr_info("x86/PAT: %s:%d freeing invalid memtype [mem %#010Lx-%#010Lx]\n",
632 current->comm, current->pid, start, end - 1);
633 return -EINVAL;
634 }
635
636 kfree(entry);
637
638 dprintk("free_memtype request [mem %#010Lx-%#010Lx]\n", start, end - 1);
639
640 return 0;
641}
642
643
644
645
646
647
648
649
650
651
652
653static enum page_cache_mode lookup_memtype(u64 paddr)
654{
655 enum page_cache_mode rettype = _PAGE_CACHE_MODE_WB;
656 struct memtype *entry;
657
658 if (x86_platform.is_untracked_pat_range(paddr, paddr + PAGE_SIZE))
659 return rettype;
660
661 if (pat_pagerange_is_ram(paddr, paddr + PAGE_SIZE)) {
662 struct page *page;
663
664 page = pfn_to_page(paddr >> PAGE_SHIFT);
665 return get_page_memtype(page);
666 }
667
668 spin_lock(&memtype_lock);
669
670 entry = rbt_memtype_lookup(paddr);
671 if (entry != NULL)
672 rettype = entry->type;
673 else
674 rettype = _PAGE_CACHE_MODE_UC_MINUS;
675
676 spin_unlock(&memtype_lock);
677 return rettype;
678}
679
680
681
682
683
684
685
686
687
688
689bool pat_pfn_immune_to_uc_mtrr(unsigned long pfn)
690{
691 enum page_cache_mode cm = lookup_memtype(PFN_PHYS(pfn));
692
693 return cm == _PAGE_CACHE_MODE_UC ||
694 cm == _PAGE_CACHE_MODE_UC_MINUS ||
695 cm == _PAGE_CACHE_MODE_WC;
696}
697EXPORT_SYMBOL_GPL(pat_pfn_immune_to_uc_mtrr);
698
699
700
701
702
703
704
705
706
707
708
709int io_reserve_memtype(resource_size_t start, resource_size_t end,
710 enum page_cache_mode *type)
711{
712 resource_size_t size = end - start;
713 enum page_cache_mode req_type = *type;
714 enum page_cache_mode new_type;
715 int ret;
716
717 WARN_ON_ONCE(iomem_map_sanity_check(start, size));
718
719 ret = reserve_memtype(start, end, req_type, &new_type);
720 if (ret)
721 goto out_err;
722
723 if (!is_new_memtype_allowed(start, size, req_type, new_type))
724 goto out_free;
725
726 if (kernel_map_sync_memtype(start, size, new_type) < 0)
727 goto out_free;
728
729 *type = new_type;
730 return 0;
731
732out_free:
733 free_memtype(start, end);
734 ret = -EBUSY;
735out_err:
736 return ret;
737}
738
739
740
741
742
743
744void io_free_memtype(resource_size_t start, resource_size_t end)
745{
746 free_memtype(start, end);
747}
748
749int arch_io_reserve_memtype_wc(resource_size_t start, resource_size_t size)
750{
751 enum page_cache_mode type = _PAGE_CACHE_MODE_WC;
752
753 return io_reserve_memtype(start, start + size, &type);
754}
755EXPORT_SYMBOL(arch_io_reserve_memtype_wc);
756
757void arch_io_free_memtype_wc(resource_size_t start, resource_size_t size)
758{
759 io_free_memtype(start, start + size);
760}
761EXPORT_SYMBOL(arch_io_free_memtype_wc);
762
763pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
764 unsigned long size, pgprot_t vma_prot)
765{
766 if (!phys_mem_access_encrypted(pfn << PAGE_SHIFT, size))
767 vma_prot = pgprot_decrypted(vma_prot);
768
769 return vma_prot;
770}
771
772#ifdef CONFIG_STRICT_DEVMEM
773
774static inline int range_is_allowed(unsigned long pfn, unsigned long size)
775{
776 return 1;
777}
778#else
779
780static inline int range_is_allowed(unsigned long pfn, unsigned long size)
781{
782 u64 from = ((u64)pfn) << PAGE_SHIFT;
783 u64 to = from + size;
784 u64 cursor = from;
785
786 if (!pat_enabled())
787 return 1;
788
789 while (cursor < to) {
790 if (!devmem_is_allowed(pfn))
791 return 0;
792 cursor += PAGE_SIZE;
793 pfn++;
794 }
795 return 1;
796}
797#endif
798
799int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
800 unsigned long size, pgprot_t *vma_prot)
801{
802 enum page_cache_mode pcm = _PAGE_CACHE_MODE_WB;
803
804 if (!range_is_allowed(pfn, size))
805 return 0;
806
807 if (file->f_flags & O_DSYNC)
808 pcm = _PAGE_CACHE_MODE_UC_MINUS;
809
810 *vma_prot = __pgprot((pgprot_val(*vma_prot) & ~_PAGE_CACHE_MASK) |
811 cachemode2protval(pcm));
812 return 1;
813}
814
815
816
817
818
819int kernel_map_sync_memtype(u64 base, unsigned long size,
820 enum page_cache_mode pcm)
821{
822 unsigned long id_sz;
823
824 if (base > __pa(high_memory-1))
825 return 0;
826
827
828
829
830
831 if (!page_is_ram(base >> PAGE_SHIFT))
832 return 0;
833
834 id_sz = (__pa(high_memory-1) <= base + size) ?
835 __pa(high_memory) - base :
836 size;
837
838 if (ioremap_change_attr((unsigned long)__va(base), id_sz, pcm) < 0) {
839 pr_info("x86/PAT: %s:%d ioremap_change_attr failed %s for [mem %#010Lx-%#010Lx]\n",
840 current->comm, current->pid,
841 cattr_name(pcm),
842 base, (unsigned long long)(base + size-1));
843 return -EINVAL;
844 }
845 return 0;
846}
847
848
849
850
851
852
853static int reserve_pfn_range(u64 paddr, unsigned long size, pgprot_t *vma_prot,
854 int strict_prot)
855{
856 int is_ram = 0;
857 int ret;
858 enum page_cache_mode want_pcm = pgprot2cachemode(*vma_prot);
859 enum page_cache_mode pcm = want_pcm;
860
861 is_ram = pat_pagerange_is_ram(paddr, paddr + size);
862
863
864
865
866
867
868 if (is_ram) {
869 if (!pat_enabled())
870 return 0;
871
872 pcm = lookup_memtype(paddr);
873 if (want_pcm != pcm) {
874 pr_warn("x86/PAT: %s:%d map pfn RAM range req %s for [mem %#010Lx-%#010Lx], got %s\n",
875 current->comm, current->pid,
876 cattr_name(want_pcm),
877 (unsigned long long)paddr,
878 (unsigned long long)(paddr + size - 1),
879 cattr_name(pcm));
880 *vma_prot = __pgprot((pgprot_val(*vma_prot) &
881 (~_PAGE_CACHE_MASK)) |
882 cachemode2protval(pcm));
883 }
884 return 0;
885 }
886
887 ret = reserve_memtype(paddr, paddr + size, want_pcm, &pcm);
888 if (ret)
889 return ret;
890
891 if (pcm != want_pcm) {
892 if (strict_prot ||
893 !is_new_memtype_allowed(paddr, size, want_pcm, pcm)) {
894 free_memtype(paddr, paddr + size);
895 pr_err("x86/PAT: %s:%d map pfn expected mapping type %s for [mem %#010Lx-%#010Lx], got %s\n",
896 current->comm, current->pid,
897 cattr_name(want_pcm),
898 (unsigned long long)paddr,
899 (unsigned long long)(paddr + size - 1),
900 cattr_name(pcm));
901 return -EINVAL;
902 }
903
904
905
906
907 *vma_prot = __pgprot((pgprot_val(*vma_prot) &
908 (~_PAGE_CACHE_MASK)) |
909 cachemode2protval(pcm));
910 }
911
912 if (kernel_map_sync_memtype(paddr, size, pcm) < 0) {
913 free_memtype(paddr, paddr + size);
914 return -EINVAL;
915 }
916 return 0;
917}
918
919
920
921
922
923static void free_pfn_range(u64 paddr, unsigned long size)
924{
925 int is_ram;
926
927 is_ram = pat_pagerange_is_ram(paddr, paddr + size);
928 if (is_ram == 0)
929 free_memtype(paddr, paddr + size);
930}
931
932
933
934
935
936
937
938
939int track_pfn_copy(struct vm_area_struct *vma)
940{
941 resource_size_t paddr;
942 unsigned long prot;
943 unsigned long vma_size = vma->vm_end - vma->vm_start;
944 pgprot_t pgprot;
945
946 if (vma->vm_flags & VM_PAT) {
947
948
949
950
951 if (follow_phys(vma, vma->vm_start, 0, &prot, &paddr)) {
952 WARN_ON_ONCE(1);
953 return -EINVAL;
954 }
955 pgprot = __pgprot(prot);
956 return reserve_pfn_range(paddr, vma_size, &pgprot, 1);
957 }
958
959 return 0;
960}
961
962
963
964
965
966
967
968int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
969 unsigned long pfn, unsigned long addr, unsigned long size)
970{
971 resource_size_t paddr = (resource_size_t)pfn << PAGE_SHIFT;
972 enum page_cache_mode pcm;
973
974
975 if (!vma || (addr == vma->vm_start
976 && size == (vma->vm_end - vma->vm_start))) {
977 int ret;
978
979 ret = reserve_pfn_range(paddr, size, prot, 0);
980 if (ret == 0 && vma)
981 vma->vm_flags |= VM_PAT;
982 return ret;
983 }
984
985 if (!pat_enabled())
986 return 0;
987
988
989
990
991
992 pcm = lookup_memtype(paddr);
993
994
995 while (size > PAGE_SIZE) {
996 size -= PAGE_SIZE;
997 paddr += PAGE_SIZE;
998 if (pcm != lookup_memtype(paddr))
999 return -EINVAL;
1000 }
1001
1002 *prot = __pgprot((pgprot_val(*prot) & (~_PAGE_CACHE_MASK)) |
1003 cachemode2protval(pcm));
1004
1005 return 0;
1006}
1007
1008void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot, pfn_t pfn)
1009{
1010 enum page_cache_mode pcm;
1011
1012 if (!pat_enabled())
1013 return;
1014
1015
1016 pcm = lookup_memtype(pfn_t_to_phys(pfn));
1017 *prot = __pgprot((pgprot_val(*prot) & (~_PAGE_CACHE_MASK)) |
1018 cachemode2protval(pcm));
1019}
1020
1021
1022
1023
1024
1025
1026void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
1027 unsigned long size)
1028{
1029 resource_size_t paddr;
1030 unsigned long prot;
1031
1032 if (vma && !(vma->vm_flags & VM_PAT))
1033 return;
1034
1035
1036 paddr = (resource_size_t)pfn << PAGE_SHIFT;
1037 if (!paddr && !size) {
1038 if (follow_phys(vma, vma->vm_start, 0, &prot, &paddr)) {
1039 WARN_ON_ONCE(1);
1040 return;
1041 }
1042
1043 size = vma->vm_end - vma->vm_start;
1044 }
1045 free_pfn_range(paddr, size);
1046 if (vma)
1047 vma->vm_flags &= ~VM_PAT;
1048}
1049
1050
1051
1052
1053
1054
1055void untrack_pfn_moved(struct vm_area_struct *vma)
1056{
1057 vma->vm_flags &= ~VM_PAT;
1058}
1059
1060pgprot_t pgprot_writecombine(pgprot_t prot)
1061{
1062 return __pgprot(pgprot_val(prot) |
1063 cachemode2protval(_PAGE_CACHE_MODE_WC));
1064}
1065EXPORT_SYMBOL_GPL(pgprot_writecombine);
1066
1067pgprot_t pgprot_writethrough(pgprot_t prot)
1068{
1069 return __pgprot(pgprot_val(prot) |
1070 cachemode2protval(_PAGE_CACHE_MODE_WT));
1071}
1072EXPORT_SYMBOL_GPL(pgprot_writethrough);
1073
1074#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_X86_PAT)
1075
1076static struct memtype *memtype_get_idx(loff_t pos)
1077{
1078 struct memtype *print_entry;
1079 int ret;
1080
1081 print_entry = kzalloc(sizeof(struct memtype), GFP_KERNEL);
1082 if (!print_entry)
1083 return NULL;
1084
1085 spin_lock(&memtype_lock);
1086 ret = rbt_memtype_copy_nth_element(print_entry, pos);
1087 spin_unlock(&memtype_lock);
1088
1089 if (!ret) {
1090 return print_entry;
1091 } else {
1092 kfree(print_entry);
1093 return NULL;
1094 }
1095}
1096
1097static void *memtype_seq_start(struct seq_file *seq, loff_t *pos)
1098{
1099 if (*pos == 0) {
1100 ++*pos;
1101 seq_puts(seq, "PAT memtype list:\n");
1102 }
1103
1104 return memtype_get_idx(*pos);
1105}
1106
1107static void *memtype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1108{
1109 ++*pos;
1110 return memtype_get_idx(*pos);
1111}
1112
1113static void memtype_seq_stop(struct seq_file *seq, void *v)
1114{
1115}
1116
1117static int memtype_seq_show(struct seq_file *seq, void *v)
1118{
1119 struct memtype *print_entry = (struct memtype *)v;
1120
1121 seq_printf(seq, "%s @ 0x%Lx-0x%Lx\n", cattr_name(print_entry->type),
1122 print_entry->start, print_entry->end);
1123 kfree(print_entry);
1124
1125 return 0;
1126}
1127
1128static const struct seq_operations memtype_seq_ops = {
1129 .start = memtype_seq_start,
1130 .next = memtype_seq_next,
1131 .stop = memtype_seq_stop,
1132 .show = memtype_seq_show,
1133};
1134
1135static int memtype_seq_open(struct inode *inode, struct file *file)
1136{
1137 return seq_open(file, &memtype_seq_ops);
1138}
1139
1140static const struct file_operations memtype_fops = {
1141 .open = memtype_seq_open,
1142 .read = seq_read,
1143 .llseek = seq_lseek,
1144 .release = seq_release,
1145};
1146
1147static int __init pat_memtype_list_init(void)
1148{
1149 if (pat_enabled()) {
1150 debugfs_create_file("pat_memtype_list", S_IRUSR,
1151 arch_debugfs_dir, NULL, &memtype_fops);
1152 }
1153 return 0;
1154}
1155
1156late_initcall(pat_memtype_list_init);
1157
1158#endif
1159