1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41#include <linux/sched.h>
42#include <linux/highmem.h>
43#include <linux/debugfs.h>
44#include <linux/bug.h>
45#include <linux/vmalloc.h>
46#include <linux/module.h>
47#include <linux/gfp.h>
48#include <linux/memblock.h>
49
50#include <asm/pgtable.h>
51#include <asm/tlbflush.h>
52#include <asm/fixmap.h>
53#include <asm/mmu_context.h>
54#include <asm/setup.h>
55#include <asm/paravirt.h>
56#include <asm/e820.h>
57#include <asm/linkage.h>
58#include <asm/page.h>
59#include <asm/init.h>
60#include <asm/pat.h>
61
62#include <asm/xen/hypercall.h>
63#include <asm/xen/hypervisor.h>
64
65#include <xen/xen.h>
66#include <xen/page.h>
67#include <xen/interface/xen.h>
68#include <xen/interface/hvm/hvm_op.h>
69#include <xen/interface/version.h>
70#include <xen/interface/memory.h>
71#include <xen/hvc-console.h>
72
73#include "multicalls.h"
74#include "mmu.h"
75#include "debugfs.h"
76
77#define MMU_UPDATE_HISTO 30
78
79
80
81
82
83
84DEFINE_SPINLOCK(xen_reservation_lock);
85
86#ifdef CONFIG_XEN_DEBUG_FS
87
88static struct {
89 u32 pgd_update;
90 u32 pgd_update_pinned;
91 u32 pgd_update_batched;
92
93 u32 pud_update;
94 u32 pud_update_pinned;
95 u32 pud_update_batched;
96
97 u32 pmd_update;
98 u32 pmd_update_pinned;
99 u32 pmd_update_batched;
100
101 u32 pte_update;
102 u32 pte_update_pinned;
103 u32 pte_update_batched;
104
105 u32 mmu_update;
106 u32 mmu_update_extended;
107 u32 mmu_update_histo[MMU_UPDATE_HISTO];
108
109 u32 prot_commit;
110 u32 prot_commit_batched;
111
112 u32 set_pte_at;
113 u32 set_pte_at_batched;
114 u32 set_pte_at_pinned;
115 u32 set_pte_at_current;
116 u32 set_pte_at_kernel;
117} mmu_stats;
118
119static u8 zero_stats;
120
121static inline void check_zero(void)
122{
123 if (unlikely(zero_stats)) {
124 memset(&mmu_stats, 0, sizeof(mmu_stats));
125 zero_stats = 0;
126 }
127}
128
129#define ADD_STATS(elem, val) \
130 do { check_zero(); mmu_stats.elem += (val); } while(0)
131
132#else
133
134#define ADD_STATS(elem, val) do { (void)(val); } while(0)
135
136#endif
137
138
139
140
141
142
143
144#define LEVEL1_IDENT_ENTRIES (PTRS_PER_PTE * 4)
145static RESERVE_BRK_ARRAY(pte_t, level1_ident_pgt, LEVEL1_IDENT_ENTRIES);
146
147#ifdef CONFIG_X86_64
148
149static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss;
150#endif
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166DEFINE_PER_CPU(unsigned long, xen_cr3);
167DEFINE_PER_CPU(unsigned long, xen_current_cr3);
168
169
170
171
172
173
174#define USER_LIMIT ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK)
175
176unsigned long arbitrary_virt_to_mfn(void *vaddr)
177{
178 xmaddr_t maddr = arbitrary_virt_to_machine(vaddr);
179
180 return PFN_DOWN(maddr.maddr);
181}
182
183xmaddr_t arbitrary_virt_to_machine(void *vaddr)
184{
185 unsigned long address = (unsigned long)vaddr;
186 unsigned int level;
187 pte_t *pte;
188 unsigned offset;
189
190
191
192
193
194 if (virt_addr_valid(vaddr))
195 return virt_to_machine(vaddr);
196
197
198
199 pte = lookup_address(address, &level);
200 BUG_ON(pte == NULL);
201 offset = address & ~PAGE_MASK;
202 return XMADDR(((phys_addr_t)pte_mfn(*pte) << PAGE_SHIFT) + offset);
203}
204EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine);
205
206void make_lowmem_page_readonly(void *vaddr)
207{
208 pte_t *pte, ptev;
209 unsigned long address = (unsigned long)vaddr;
210 unsigned int level;
211
212 pte = lookup_address(address, &level);
213 if (pte == NULL)
214 return;
215
216 ptev = pte_wrprotect(*pte);
217
218 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
219 BUG();
220}
221
222void make_lowmem_page_readwrite(void *vaddr)
223{
224 pte_t *pte, ptev;
225 unsigned long address = (unsigned long)vaddr;
226 unsigned int level;
227
228 pte = lookup_address(address, &level);
229 if (pte == NULL)
230 return;
231
232 ptev = pte_mkwrite(*pte);
233
234 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
235 BUG();
236}
237
238
239static bool xen_page_pinned(void *ptr)
240{
241 struct page *page = virt_to_page(ptr);
242
243 return PagePinned(page);
244}
245
246static bool xen_iomap_pte(pte_t pte)
247{
248 return pte_flags(pte) & _PAGE_IOMAP;
249}
250
251void xen_set_domain_pte(pte_t *ptep, pte_t pteval, unsigned domid)
252{
253 struct multicall_space mcs;
254 struct mmu_update *u;
255
256 mcs = xen_mc_entry(sizeof(*u));
257 u = mcs.args;
258
259
260 u->ptr = arbitrary_virt_to_machine(ptep).maddr;
261 u->val = pte_val_ma(pteval);
262
263 MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, domid);
264
265 xen_mc_issue(PARAVIRT_LAZY_MMU);
266}
267EXPORT_SYMBOL_GPL(xen_set_domain_pte);
268
269static void xen_set_iomap_pte(pte_t *ptep, pte_t pteval)
270{
271 xen_set_domain_pte(ptep, pteval, DOMID_IO);
272}
273
274static void xen_extend_mmu_update(const struct mmu_update *update)
275{
276 struct multicall_space mcs;
277 struct mmu_update *u;
278
279 mcs = xen_mc_extend_args(__HYPERVISOR_mmu_update, sizeof(*u));
280
281 if (mcs.mc != NULL) {
282 ADD_STATS(mmu_update_extended, 1);
283 ADD_STATS(mmu_update_histo[mcs.mc->args[1]], -1);
284
285 mcs.mc->args[1]++;
286
287 if (mcs.mc->args[1] < MMU_UPDATE_HISTO)
288 ADD_STATS(mmu_update_histo[mcs.mc->args[1]], 1);
289 else
290 ADD_STATS(mmu_update_histo[0], 1);
291 } else {
292 ADD_STATS(mmu_update, 1);
293 mcs = __xen_mc_entry(sizeof(*u));
294 MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
295 ADD_STATS(mmu_update_histo[1], 1);
296 }
297
298 u = mcs.args;
299 *u = *update;
300}
301
302void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val)
303{
304 struct mmu_update u;
305
306 preempt_disable();
307
308 xen_mc_batch();
309
310
311 u.ptr = arbitrary_virt_to_machine(ptr).maddr;
312 u.val = pmd_val_ma(val);
313 xen_extend_mmu_update(&u);
314
315 ADD_STATS(pmd_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
316
317 xen_mc_issue(PARAVIRT_LAZY_MMU);
318
319 preempt_enable();
320}
321
322void xen_set_pmd(pmd_t *ptr, pmd_t val)
323{
324 ADD_STATS(pmd_update, 1);
325
326
327
328 if (!xen_page_pinned(ptr)) {
329 *ptr = val;
330 return;
331 }
332
333 ADD_STATS(pmd_update_pinned, 1);
334
335 xen_set_pmd_hyper(ptr, val);
336}
337
338
339
340
341
342void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags)
343{
344 set_pte_vaddr(vaddr, mfn_pte(mfn, flags));
345}
346
347void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
348 pte_t *ptep, pte_t pteval)
349{
350 if (xen_iomap_pte(pteval)) {
351 xen_set_iomap_pte(ptep, pteval);
352 goto out;
353 }
354
355 ADD_STATS(set_pte_at, 1);
356
357 ADD_STATS(set_pte_at_current, mm == current->mm);
358 ADD_STATS(set_pte_at_kernel, mm == &init_mm);
359
360 if (mm == current->mm || mm == &init_mm) {
361 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) {
362 struct multicall_space mcs;
363 mcs = xen_mc_entry(0);
364
365 MULTI_update_va_mapping(mcs.mc, addr, pteval, 0);
366 ADD_STATS(set_pte_at_batched, 1);
367 xen_mc_issue(PARAVIRT_LAZY_MMU);
368 goto out;
369 } else
370 if (HYPERVISOR_update_va_mapping(addr, pteval, 0) == 0)
371 goto out;
372 }
373 xen_set_pte(ptep, pteval);
374
375out: return;
376}
377
378pte_t xen_ptep_modify_prot_start(struct mm_struct *mm,
379 unsigned long addr, pte_t *ptep)
380{
381
382 return *ptep;
383}
384
385void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
386 pte_t *ptep, pte_t pte)
387{
388 struct mmu_update u;
389
390 xen_mc_batch();
391
392 u.ptr = arbitrary_virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD;
393 u.val = pte_val_ma(pte);
394 xen_extend_mmu_update(&u);
395
396 ADD_STATS(prot_commit, 1);
397 ADD_STATS(prot_commit_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
398
399 xen_mc_issue(PARAVIRT_LAZY_MMU);
400}
401
402
403static pteval_t pte_mfn_to_pfn(pteval_t val)
404{
405 if (val & _PAGE_PRESENT) {
406 unsigned long mfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
407 pteval_t flags = val & PTE_FLAGS_MASK;
408 val = ((pteval_t)mfn_to_pfn(mfn) << PAGE_SHIFT) | flags;
409 }
410
411 return val;
412}
413
414static pteval_t pte_pfn_to_mfn(pteval_t val)
415{
416 if (val & _PAGE_PRESENT) {
417 unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
418 pteval_t flags = val & PTE_FLAGS_MASK;
419 unsigned long mfn = pfn_to_mfn(pfn);
420
421
422
423
424
425
426
427 if (unlikely(mfn == INVALID_P2M_ENTRY)) {
428 mfn = 0;
429 flags = 0;
430 }
431
432 val = ((pteval_t)mfn << PAGE_SHIFT) | flags;
433 }
434
435 return val;
436}
437
438static pteval_t iomap_pte(pteval_t val)
439{
440 if (val & _PAGE_PRESENT) {
441 unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
442 pteval_t flags = val & PTE_FLAGS_MASK;
443
444
445
446 val = ((pteval_t)pfn << PAGE_SHIFT) | flags;
447 }
448
449 return val;
450}
451
452pteval_t xen_pte_val(pte_t pte)
453{
454 pteval_t pteval = pte.pte;
455
456
457 if ((pteval & (_PAGE_PAT | _PAGE_PCD | _PAGE_PWT)) == _PAGE_PAT) {
458 WARN_ON(!pat_enabled);
459 pteval = (pteval & ~_PAGE_PAT) | _PAGE_PWT;
460 }
461
462 if (xen_initial_domain() && (pteval & _PAGE_IOMAP))
463 return pteval;
464
465 return pte_mfn_to_pfn(pteval);
466}
467PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val);
468
469pgdval_t xen_pgd_val(pgd_t pgd)
470{
471 return pte_mfn_to_pfn(pgd.pgd);
472}
473PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val);
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493void xen_set_pat(u64 pat)
494{
495
496
497 WARN_ON(pat != 0x0007010600070106ull);
498}
499
500pte_t xen_make_pte(pteval_t pte)
501{
502 phys_addr_t addr = (pte & PTE_PFN_MASK);
503
504
505
506
507
508
509
510
511
512 if (pat_enabled && !WARN_ON(pte & _PAGE_PAT)) {
513 if ((pte & (_PAGE_PCD | _PAGE_PWT)) == _PAGE_PWT)
514 pte = (pte & ~(_PAGE_PCD | _PAGE_PWT)) | _PAGE_PAT;
515 }
516
517
518
519
520
521
522
523 if (unlikely(pte & _PAGE_IOMAP) &&
524 (xen_initial_domain() || addr >= ISA_END_ADDRESS)) {
525 pte = iomap_pte(pte);
526 } else {
527 pte &= ~_PAGE_IOMAP;
528 pte = pte_pfn_to_mfn(pte);
529 }
530
531 return native_make_pte(pte);
532}
533PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte);
534
535pgd_t xen_make_pgd(pgdval_t pgd)
536{
537 pgd = pte_pfn_to_mfn(pgd);
538 return native_make_pgd(pgd);
539}
540PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd);
541
542pmdval_t xen_pmd_val(pmd_t pmd)
543{
544 return pte_mfn_to_pfn(pmd.pmd);
545}
546PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val);
547
548void xen_set_pud_hyper(pud_t *ptr, pud_t val)
549{
550 struct mmu_update u;
551
552 preempt_disable();
553
554 xen_mc_batch();
555
556
557 u.ptr = arbitrary_virt_to_machine(ptr).maddr;
558 u.val = pud_val_ma(val);
559 xen_extend_mmu_update(&u);
560
561 ADD_STATS(pud_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
562
563 xen_mc_issue(PARAVIRT_LAZY_MMU);
564
565 preempt_enable();
566}
567
568void xen_set_pud(pud_t *ptr, pud_t val)
569{
570 ADD_STATS(pud_update, 1);
571
572
573
574 if (!xen_page_pinned(ptr)) {
575 *ptr = val;
576 return;
577 }
578
579 ADD_STATS(pud_update_pinned, 1);
580
581 xen_set_pud_hyper(ptr, val);
582}
583
584void xen_set_pte(pte_t *ptep, pte_t pte)
585{
586 if (xen_iomap_pte(pte)) {
587 xen_set_iomap_pte(ptep, pte);
588 return;
589 }
590
591 ADD_STATS(pte_update, 1);
592
593 ADD_STATS(pte_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
594
595#ifdef CONFIG_X86_PAE
596 ptep->pte_high = pte.pte_high;
597 smp_wmb();
598 ptep->pte_low = pte.pte_low;
599#else
600 *ptep = pte;
601#endif
602}
603
604#ifdef CONFIG_X86_PAE
605void xen_set_pte_atomic(pte_t *ptep, pte_t pte)
606{
607 if (xen_iomap_pte(pte)) {
608 xen_set_iomap_pte(ptep, pte);
609 return;
610 }
611
612 set_64bit((u64 *)ptep, native_pte_val(pte));
613}
614
615void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
616{
617 ptep->pte_low = 0;
618 smp_wmb();
619 ptep->pte_high = 0;
620}
621
622void xen_pmd_clear(pmd_t *pmdp)
623{
624 set_pmd(pmdp, __pmd(0));
625}
626#endif
627
628pmd_t xen_make_pmd(pmdval_t pmd)
629{
630 pmd = pte_pfn_to_mfn(pmd);
631 return native_make_pmd(pmd);
632}
633PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd);
634
635#if PAGETABLE_LEVELS == 4
636pudval_t xen_pud_val(pud_t pud)
637{
638 return pte_mfn_to_pfn(pud.pud);
639}
640PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val);
641
642pud_t xen_make_pud(pudval_t pud)
643{
644 pud = pte_pfn_to_mfn(pud);
645
646 return native_make_pud(pud);
647}
648PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud);
649
650pgd_t *xen_get_user_pgd(pgd_t *pgd)
651{
652 pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK);
653 unsigned offset = pgd - pgd_page;
654 pgd_t *user_ptr = NULL;
655
656 if (offset < pgd_index(USER_LIMIT)) {
657 struct page *page = virt_to_page(pgd_page);
658 user_ptr = (pgd_t *)page->private;
659 if (user_ptr)
660 user_ptr += offset;
661 }
662
663 return user_ptr;
664}
665
666static void __xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
667{
668 struct mmu_update u;
669
670 u.ptr = virt_to_machine(ptr).maddr;
671 u.val = pgd_val_ma(val);
672 xen_extend_mmu_update(&u);
673}
674
675
676
677
678
679
680
681
682void __init xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
683{
684 preempt_disable();
685
686 xen_mc_batch();
687
688 __xen_set_pgd_hyper(ptr, val);
689
690 xen_mc_issue(PARAVIRT_LAZY_MMU);
691
692 preempt_enable();
693}
694
695void xen_set_pgd(pgd_t *ptr, pgd_t val)
696{
697 pgd_t *user_ptr = xen_get_user_pgd(ptr);
698
699 ADD_STATS(pgd_update, 1);
700
701
702
703 if (!xen_page_pinned(ptr)) {
704 *ptr = val;
705 if (user_ptr) {
706 WARN_ON(xen_page_pinned(user_ptr));
707 *user_ptr = val;
708 }
709 return;
710 }
711
712 ADD_STATS(pgd_update_pinned, 1);
713 ADD_STATS(pgd_update_batched, paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU);
714
715
716
717 xen_mc_batch();
718
719 __xen_set_pgd_hyper(ptr, val);
720 if (user_ptr)
721 __xen_set_pgd_hyper(user_ptr, val);
722
723 xen_mc_issue(PARAVIRT_LAZY_MMU);
724}
725#endif
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742static int __xen_pgd_walk(struct mm_struct *mm, pgd_t *pgd,
743 int (*func)(struct mm_struct *mm, struct page *,
744 enum pt_level),
745 unsigned long limit)
746{
747 int flush = 0;
748 unsigned hole_low, hole_high;
749 unsigned pgdidx_limit, pudidx_limit, pmdidx_limit;
750 unsigned pgdidx, pudidx, pmdidx;
751
752
753 limit--;
754 BUG_ON(limit >= FIXADDR_TOP);
755
756 if (xen_feature(XENFEAT_auto_translated_physmap))
757 return 0;
758
759
760
761
762
763
764 hole_low = pgd_index(USER_LIMIT);
765 hole_high = pgd_index(PAGE_OFFSET);
766
767 pgdidx_limit = pgd_index(limit);
768#if PTRS_PER_PUD > 1
769 pudidx_limit = pud_index(limit);
770#else
771 pudidx_limit = 0;
772#endif
773#if PTRS_PER_PMD > 1
774 pmdidx_limit = pmd_index(limit);
775#else
776 pmdidx_limit = 0;
777#endif
778
779 for (pgdidx = 0; pgdidx <= pgdidx_limit; pgdidx++) {
780 pud_t *pud;
781
782 if (pgdidx >= hole_low && pgdidx < hole_high)
783 continue;
784
785 if (!pgd_val(pgd[pgdidx]))
786 continue;
787
788 pud = pud_offset(&pgd[pgdidx], 0);
789
790 if (PTRS_PER_PUD > 1)
791 flush |= (*func)(mm, virt_to_page(pud), PT_PUD);
792
793 for (pudidx = 0; pudidx < PTRS_PER_PUD; pudidx++) {
794 pmd_t *pmd;
795
796 if (pgdidx == pgdidx_limit &&
797 pudidx > pudidx_limit)
798 goto out;
799
800 if (pud_none(pud[pudidx]))
801 continue;
802
803 pmd = pmd_offset(&pud[pudidx], 0);
804
805 if (PTRS_PER_PMD > 1)
806 flush |= (*func)(mm, virt_to_page(pmd), PT_PMD);
807
808 for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++) {
809 struct page *pte;
810
811 if (pgdidx == pgdidx_limit &&
812 pudidx == pudidx_limit &&
813 pmdidx > pmdidx_limit)
814 goto out;
815
816 if (pmd_none(pmd[pmdidx]))
817 continue;
818
819 pte = pmd_page(pmd[pmdidx]);
820 flush |= (*func)(mm, pte, PT_PTE);
821 }
822 }
823 }
824
825out:
826
827
828 flush |= (*func)(mm, virt_to_page(pgd), PT_PGD);
829
830 return flush;
831}
832
833static int xen_pgd_walk(struct mm_struct *mm,
834 int (*func)(struct mm_struct *mm, struct page *,
835 enum pt_level),
836 unsigned long limit)
837{
838 return __xen_pgd_walk(mm, mm->pgd, func, limit);
839}
840
841
842
843static spinlock_t *xen_pte_lock(struct page *page, struct mm_struct *mm)
844{
845 spinlock_t *ptl = NULL;
846
847#if USE_SPLIT_PTLOCKS
848 ptl = __pte_lockptr(page);
849 spin_lock_nest_lock(ptl, &mm->page_table_lock);
850#endif
851
852 return ptl;
853}
854
855static void xen_pte_unlock(void *v)
856{
857 spinlock_t *ptl = v;
858 spin_unlock(ptl);
859}
860
861static void xen_do_pin(unsigned level, unsigned long pfn)
862{
863 struct mmuext_op *op;
864 struct multicall_space mcs;
865
866 mcs = __xen_mc_entry(sizeof(*op));
867 op = mcs.args;
868 op->cmd = level;
869 op->arg1.mfn = pfn_to_mfn(pfn);
870 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
871}
872
873static int xen_pin_page(struct mm_struct *mm, struct page *page,
874 enum pt_level level)
875{
876 unsigned pgfl = TestSetPagePinned(page);
877 int flush;
878
879 if (pgfl)
880 flush = 0;
881 else if (PageHighMem(page))
882
883
884 flush = 1;
885 else {
886 void *pt = lowmem_page_address(page);
887 unsigned long pfn = page_to_pfn(page);
888 struct multicall_space mcs = __xen_mc_entry(0);
889 spinlock_t *ptl;
890
891 flush = 0;
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913 ptl = NULL;
914 if (level == PT_PTE)
915 ptl = xen_pte_lock(page, mm);
916
917 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
918 pfn_pte(pfn, PAGE_KERNEL_RO),
919 level == PT_PGD ? UVMF_TLB_FLUSH : 0);
920
921 if (ptl) {
922 xen_do_pin(MMUEXT_PIN_L1_TABLE, pfn);
923
924
925
926 xen_mc_callback(xen_pte_unlock, ptl);
927 }
928 }
929
930 return flush;
931}
932
933
934
935
936static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd)
937{
938 xen_mc_batch();
939
940 if (__xen_pgd_walk(mm, pgd, xen_pin_page, USER_LIMIT)) {
941
942 xen_mc_issue(0);
943
944 kmap_flush_unused();
945
946 xen_mc_batch();
947 }
948
949#ifdef CONFIG_X86_64
950 {
951 pgd_t *user_pgd = xen_get_user_pgd(pgd);
952
953 xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(pgd)));
954
955 if (user_pgd) {
956 xen_pin_page(mm, virt_to_page(user_pgd), PT_PGD);
957 xen_do_pin(MMUEXT_PIN_L4_TABLE,
958 PFN_DOWN(__pa(user_pgd)));
959 }
960 }
961#else
962#ifdef CONFIG_X86_PAE
963
964 xen_pin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),
965 PT_PMD);
966#endif
967 xen_do_pin(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(pgd)));
968#endif
969 xen_mc_issue(0);
970}
971
972static void xen_pgd_pin(struct mm_struct *mm)
973{
974 __xen_pgd_pin(mm, mm->pgd);
975}
976
977
978
979
980
981
982
983
984
985
986
987void xen_mm_pin_all(void)
988{
989 struct page *page;
990
991 spin_lock(&pgd_lock);
992
993 list_for_each_entry(page, &pgd_list, lru) {
994 if (!PagePinned(page)) {
995 __xen_pgd_pin(&init_mm, (pgd_t *)page_address(page));
996 SetPageSavePinned(page);
997 }
998 }
999
1000 spin_unlock(&pgd_lock);
1001}
1002
1003
1004
1005
1006
1007
1008static __init int xen_mark_pinned(struct mm_struct *mm, struct page *page,
1009 enum pt_level level)
1010{
1011 SetPagePinned(page);
1012 return 0;
1013}
1014
1015static void __init xen_mark_init_mm_pinned(void)
1016{
1017 xen_pgd_walk(&init_mm, xen_mark_pinned, FIXADDR_TOP);
1018}
1019
1020static int xen_unpin_page(struct mm_struct *mm, struct page *page,
1021 enum pt_level level)
1022{
1023 unsigned pgfl = TestClearPagePinned(page);
1024
1025 if (pgfl && !PageHighMem(page)) {
1026 void *pt = lowmem_page_address(page);
1027 unsigned long pfn = page_to_pfn(page);
1028 spinlock_t *ptl = NULL;
1029 struct multicall_space mcs;
1030
1031
1032
1033
1034
1035
1036
1037
1038 if (level == PT_PTE) {
1039 ptl = xen_pte_lock(page, mm);
1040
1041 if (ptl)
1042 xen_do_pin(MMUEXT_UNPIN_TABLE, pfn);
1043 }
1044
1045 mcs = __xen_mc_entry(0);
1046
1047 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
1048 pfn_pte(pfn, PAGE_KERNEL),
1049 level == PT_PGD ? UVMF_TLB_FLUSH : 0);
1050
1051 if (ptl) {
1052
1053 xen_mc_callback(xen_pte_unlock, ptl);
1054 }
1055 }
1056
1057 return 0;
1058}
1059
1060
1061static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd)
1062{
1063 xen_mc_batch();
1064
1065 xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
1066
1067#ifdef CONFIG_X86_64
1068 {
1069 pgd_t *user_pgd = xen_get_user_pgd(pgd);
1070
1071 if (user_pgd) {
1072 xen_do_pin(MMUEXT_UNPIN_TABLE,
1073 PFN_DOWN(__pa(user_pgd)));
1074 xen_unpin_page(mm, virt_to_page(user_pgd), PT_PGD);
1075 }
1076 }
1077#endif
1078
1079#ifdef CONFIG_X86_PAE
1080
1081 xen_unpin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),
1082 PT_PMD);
1083#endif
1084
1085 __xen_pgd_walk(mm, pgd, xen_unpin_page, USER_LIMIT);
1086
1087 xen_mc_issue(0);
1088}
1089
1090static void xen_pgd_unpin(struct mm_struct *mm)
1091{
1092 __xen_pgd_unpin(mm, mm->pgd);
1093}
1094
1095
1096
1097
1098
1099void xen_mm_unpin_all(void)
1100{
1101 struct page *page;
1102
1103 spin_lock(&pgd_lock);
1104
1105 list_for_each_entry(page, &pgd_list, lru) {
1106 if (PageSavePinned(page)) {
1107 BUG_ON(!PagePinned(page));
1108 __xen_pgd_unpin(&init_mm, (pgd_t *)page_address(page));
1109 ClearPageSavePinned(page);
1110 }
1111 }
1112
1113 spin_unlock(&pgd_lock);
1114}
1115
1116void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next)
1117{
1118 spin_lock(&next->page_table_lock);
1119 xen_pgd_pin(next);
1120 spin_unlock(&next->page_table_lock);
1121}
1122
1123void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
1124{
1125 spin_lock(&mm->page_table_lock);
1126 xen_pgd_pin(mm);
1127 spin_unlock(&mm->page_table_lock);
1128}
1129
1130
1131#ifdef CONFIG_SMP
1132
1133
1134static void drop_other_mm_ref(void *info)
1135{
1136 struct mm_struct *mm = info;
1137 struct mm_struct *active_mm;
1138
1139 active_mm = percpu_read(cpu_tlbstate.active_mm);
1140
1141 if (active_mm == mm)
1142 leave_mm(smp_processor_id());
1143
1144
1145
1146 if (percpu_read(xen_current_cr3) == __pa(mm->pgd))
1147 load_cr3(swapper_pg_dir);
1148}
1149
1150static void xen_drop_mm_ref(struct mm_struct *mm)
1151{
1152 cpumask_var_t mask;
1153 unsigned cpu;
1154
1155 if (current->active_mm == mm) {
1156 if (current->mm == mm)
1157 load_cr3(swapper_pg_dir);
1158 else
1159 leave_mm(smp_processor_id());
1160 }
1161
1162
1163 if (!alloc_cpumask_var(&mask, GFP_ATOMIC)) {
1164 for_each_online_cpu(cpu) {
1165 if (!cpumask_test_cpu(cpu, mm_cpumask(mm))
1166 && per_cpu(xen_current_cr3, cpu) != __pa(mm->pgd))
1167 continue;
1168 smp_call_function_single(cpu, drop_other_mm_ref, mm, 1);
1169 }
1170 return;
1171 }
1172 cpumask_copy(mask, mm_cpumask(mm));
1173
1174
1175
1176
1177
1178
1179 for_each_online_cpu(cpu) {
1180 if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd))
1181 cpumask_set_cpu(cpu, mask);
1182 }
1183
1184 if (!cpumask_empty(mask))
1185 smp_call_function_many(mask, drop_other_mm_ref, mm, 1);
1186 free_cpumask_var(mask);
1187}
1188#else
1189static void xen_drop_mm_ref(struct mm_struct *mm)
1190{
1191 if (current->active_mm == mm)
1192 load_cr3(swapper_pg_dir);
1193}
1194#endif
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210void xen_exit_mmap(struct mm_struct *mm)
1211{
1212 get_cpu();
1213 xen_drop_mm_ref(mm);
1214 put_cpu();
1215
1216 spin_lock(&mm->page_table_lock);
1217
1218
1219 if (xen_page_pinned(mm->pgd))
1220 xen_pgd_unpin(mm);
1221
1222 spin_unlock(&mm->page_table_lock);
1223}
1224
1225static __init void xen_pagetable_setup_start(pgd_t *base)
1226{
1227}
1228
1229static void xen_post_allocator_init(void);
1230
1231static __init void xen_pagetable_setup_done(pgd_t *base)
1232{
1233 xen_setup_shared_info();
1234 xen_post_allocator_init();
1235}
1236
1237static void xen_write_cr2(unsigned long cr2)
1238{
1239 percpu_read(xen_vcpu)->arch.cr2 = cr2;
1240}
1241
1242static unsigned long xen_read_cr2(void)
1243{
1244 return percpu_read(xen_vcpu)->arch.cr2;
1245}
1246
1247unsigned long xen_read_cr2_direct(void)
1248{
1249 return percpu_read(xen_vcpu_info.arch.cr2);
1250}
1251
1252static void xen_flush_tlb(void)
1253{
1254 struct mmuext_op *op;
1255 struct multicall_space mcs;
1256
1257 preempt_disable();
1258
1259 mcs = xen_mc_entry(sizeof(*op));
1260
1261 op = mcs.args;
1262 op->cmd = MMUEXT_TLB_FLUSH_LOCAL;
1263 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1264
1265 xen_mc_issue(PARAVIRT_LAZY_MMU);
1266
1267 preempt_enable();
1268}
1269
1270static void xen_flush_tlb_single(unsigned long addr)
1271{
1272 struct mmuext_op *op;
1273 struct multicall_space mcs;
1274
1275 preempt_disable();
1276
1277 mcs = xen_mc_entry(sizeof(*op));
1278 op = mcs.args;
1279 op->cmd = MMUEXT_INVLPG_LOCAL;
1280 op->arg1.linear_addr = addr & PAGE_MASK;
1281 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1282
1283 xen_mc_issue(PARAVIRT_LAZY_MMU);
1284
1285 preempt_enable();
1286}
1287
1288static void xen_flush_tlb_others(const struct cpumask *cpus,
1289 struct mm_struct *mm, unsigned long va)
1290{
1291 struct {
1292 struct mmuext_op op;
1293 DECLARE_BITMAP(mask, NR_CPUS);
1294 } *args;
1295 struct multicall_space mcs;
1296
1297 if (cpumask_empty(cpus))
1298 return;
1299
1300 mcs = xen_mc_entry(sizeof(*args));
1301 args = mcs.args;
1302 args->op.arg2.vcpumask = to_cpumask(args->mask);
1303
1304
1305 cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
1306 cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
1307
1308 if (va == TLB_FLUSH_ALL) {
1309 args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
1310 } else {
1311 args->op.cmd = MMUEXT_INVLPG_MULTI;
1312 args->op.arg1.linear_addr = va;
1313 }
1314
1315 MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
1316
1317 xen_mc_issue(PARAVIRT_LAZY_MMU);
1318}
1319
1320static unsigned long xen_read_cr3(void)
1321{
1322 return percpu_read(xen_cr3);
1323}
1324
1325static void set_current_cr3(void *v)
1326{
1327 percpu_write(xen_current_cr3, (unsigned long)v);
1328}
1329
1330static void __xen_write_cr3(bool kernel, unsigned long cr3)
1331{
1332 struct mmuext_op *op;
1333 struct multicall_space mcs;
1334 unsigned long mfn;
1335
1336 if (cr3)
1337 mfn = pfn_to_mfn(PFN_DOWN(cr3));
1338 else
1339 mfn = 0;
1340
1341 WARN_ON(mfn == 0 && kernel);
1342
1343 mcs = __xen_mc_entry(sizeof(*op));
1344
1345 op = mcs.args;
1346 op->cmd = kernel ? MMUEXT_NEW_BASEPTR : MMUEXT_NEW_USER_BASEPTR;
1347 op->arg1.mfn = mfn;
1348
1349 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1350
1351 if (kernel) {
1352 percpu_write(xen_cr3, cr3);
1353
1354
1355
1356 xen_mc_callback(set_current_cr3, (void *)cr3);
1357 }
1358}
1359
1360static void xen_write_cr3(unsigned long cr3)
1361{
1362 BUG_ON(preemptible());
1363
1364 xen_mc_batch();
1365
1366
1367
1368 percpu_write(xen_cr3, cr3);
1369
1370 __xen_write_cr3(true, cr3);
1371
1372#ifdef CONFIG_X86_64
1373 {
1374 pgd_t *user_pgd = xen_get_user_pgd(__va(cr3));
1375 if (user_pgd)
1376 __xen_write_cr3(false, __pa(user_pgd));
1377 else
1378 __xen_write_cr3(false, 0);
1379 }
1380#endif
1381
1382 xen_mc_issue(PARAVIRT_LAZY_CPU);
1383}
1384
1385static int xen_pgd_alloc(struct mm_struct *mm)
1386{
1387 pgd_t *pgd = mm->pgd;
1388 int ret = 0;
1389
1390 BUG_ON(PagePinned(virt_to_page(pgd)));
1391
1392#ifdef CONFIG_X86_64
1393 {
1394 struct page *page = virt_to_page(pgd);
1395 pgd_t *user_pgd;
1396
1397 BUG_ON(page->private != 0);
1398
1399 ret = -ENOMEM;
1400
1401 user_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
1402 page->private = (unsigned long)user_pgd;
1403
1404 if (user_pgd != NULL) {
1405 user_pgd[pgd_index(VSYSCALL_START)] =
1406 __pgd(__pa(level3_user_vsyscall) | _PAGE_TABLE);
1407 ret = 0;
1408 }
1409
1410 BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd))));
1411 }
1412#endif
1413
1414 return ret;
1415}
1416
1417static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd)
1418{
1419#ifdef CONFIG_X86_64
1420 pgd_t *user_pgd = xen_get_user_pgd(pgd);
1421
1422 if (user_pgd)
1423 free_page((unsigned long)user_pgd);
1424#endif
1425}
1426
1427static __init pte_t mask_rw_pte(pte_t *ptep, pte_t pte)
1428{
1429 unsigned long pfn = pte_pfn(pte);
1430
1431#ifdef CONFIG_X86_32
1432
1433 if (pte_val_ma(*ptep) & _PAGE_PRESENT)
1434 pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) &
1435 pte_val_ma(pte));
1436#endif
1437
1438
1439
1440
1441
1442
1443 if (!is_early_ioremap_ptep(ptep) &&
1444 pfn >= e820_table_start && pfn < e820_table_end)
1445 pte = pte_wrprotect(pte);
1446
1447 return pte;
1448}
1449
1450
1451
1452static __init void xen_set_pte_init(pte_t *ptep, pte_t pte)
1453{
1454 pte = mask_rw_pte(ptep, pte);
1455
1456 xen_set_pte(ptep, pte);
1457}
1458
1459static void pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
1460{
1461 struct mmuext_op op;
1462 op.cmd = cmd;
1463 op.arg1.mfn = pfn_to_mfn(pfn);
1464 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF))
1465 BUG();
1466}
1467
1468
1469
1470static __init void xen_alloc_pte_init(struct mm_struct *mm, unsigned long pfn)
1471{
1472#ifdef CONFIG_FLATMEM
1473 BUG_ON(mem_map);
1474#endif
1475 make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
1476 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
1477}
1478
1479
1480static __init void xen_alloc_pmd_init(struct mm_struct *mm, unsigned long pfn)
1481{
1482#ifdef CONFIG_FLATMEM
1483 BUG_ON(mem_map);
1484#endif
1485 make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
1486}
1487
1488
1489
1490static __init void xen_release_pte_init(unsigned long pfn)
1491{
1492 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
1493 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
1494}
1495
1496static __init void xen_release_pmd_init(unsigned long pfn)
1497{
1498 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
1499}
1500
1501
1502
1503static void xen_alloc_ptpage(struct mm_struct *mm, unsigned long pfn, unsigned level)
1504{
1505 struct page *page = pfn_to_page(pfn);
1506
1507 if (PagePinned(virt_to_page(mm->pgd))) {
1508 SetPagePinned(page);
1509
1510 if (!PageHighMem(page)) {
1511 make_lowmem_page_readonly(__va(PFN_PHYS((unsigned long)pfn)));
1512 if (level == PT_PTE && USE_SPLIT_PTLOCKS)
1513 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
1514 } else {
1515
1516
1517 kmap_flush_unused();
1518 }
1519 }
1520}
1521
1522static void xen_alloc_pte(struct mm_struct *mm, unsigned long pfn)
1523{
1524 xen_alloc_ptpage(mm, pfn, PT_PTE);
1525}
1526
1527static void xen_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
1528{
1529 xen_alloc_ptpage(mm, pfn, PT_PMD);
1530}
1531
1532
1533static void xen_release_ptpage(unsigned long pfn, unsigned level)
1534{
1535 struct page *page = pfn_to_page(pfn);
1536
1537 if (PagePinned(page)) {
1538 if (!PageHighMem(page)) {
1539 if (level == PT_PTE && USE_SPLIT_PTLOCKS)
1540 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
1541 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
1542 }
1543 ClearPagePinned(page);
1544 }
1545}
1546
1547static void xen_release_pte(unsigned long pfn)
1548{
1549 xen_release_ptpage(pfn, PT_PTE);
1550}
1551
1552static void xen_release_pmd(unsigned long pfn)
1553{
1554 xen_release_ptpage(pfn, PT_PMD);
1555}
1556
1557#if PAGETABLE_LEVELS == 4
1558static void xen_alloc_pud(struct mm_struct *mm, unsigned long pfn)
1559{
1560 xen_alloc_ptpage(mm, pfn, PT_PUD);
1561}
1562
1563static void xen_release_pud(unsigned long pfn)
1564{
1565 xen_release_ptpage(pfn, PT_PUD);
1566}
1567#endif
1568
1569void __init xen_reserve_top(void)
1570{
1571#ifdef CONFIG_X86_32
1572 unsigned long top = HYPERVISOR_VIRT_START;
1573 struct xen_platform_parameters pp;
1574
1575 if (HYPERVISOR_xen_version(XENVER_platform_parameters, &pp) == 0)
1576 top = pp.virt_start;
1577
1578 reserve_top_address(-top);
1579#endif
1580}
1581
1582
1583
1584
1585
1586static void *__ka(phys_addr_t paddr)
1587{
1588#ifdef CONFIG_X86_64
1589 return (void *)(paddr + __START_KERNEL_map);
1590#else
1591 return __va(paddr);
1592#endif
1593}
1594
1595
1596static unsigned long m2p(phys_addr_t maddr)
1597{
1598 phys_addr_t paddr;
1599
1600 maddr &= PTE_PFN_MASK;
1601 paddr = mfn_to_pfn(maddr >> PAGE_SHIFT) << PAGE_SHIFT;
1602
1603 return paddr;
1604}
1605
1606
1607static void *m2v(phys_addr_t maddr)
1608{
1609 return __ka(m2p(maddr));
1610}
1611
1612
1613static void set_page_prot(void *addr, pgprot_t prot)
1614{
1615 unsigned long pfn = __pa(addr) >> PAGE_SHIFT;
1616 pte_t pte = pfn_pte(pfn, prot);
1617
1618 if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, 0))
1619 BUG();
1620}
1621
1622static __init void xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn)
1623{
1624 unsigned pmdidx, pteidx;
1625 unsigned ident_pte;
1626 unsigned long pfn;
1627
1628 level1_ident_pgt = extend_brk(sizeof(pte_t) * LEVEL1_IDENT_ENTRIES,
1629 PAGE_SIZE);
1630
1631 ident_pte = 0;
1632 pfn = 0;
1633 for (pmdidx = 0; pmdidx < PTRS_PER_PMD && pfn < max_pfn; pmdidx++) {
1634 pte_t *pte_page;
1635
1636
1637 if (pmd_present(pmd[pmdidx]))
1638 pte_page = m2v(pmd[pmdidx].pmd);
1639 else {
1640
1641 if (ident_pte == LEVEL1_IDENT_ENTRIES)
1642 break;
1643
1644 pte_page = &level1_ident_pgt[ident_pte];
1645 ident_pte += PTRS_PER_PTE;
1646
1647 pmd[pmdidx] = __pmd(__pa(pte_page) | _PAGE_TABLE);
1648 }
1649
1650
1651 for (pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) {
1652 pte_t pte;
1653
1654 if (pfn > max_pfn_mapped)
1655 max_pfn_mapped = pfn;
1656
1657 if (!pte_none(pte_page[pteidx]))
1658 continue;
1659
1660 pte = pfn_pte(pfn, PAGE_KERNEL_EXEC);
1661 pte_page[pteidx] = pte;
1662 }
1663 }
1664
1665 for (pteidx = 0; pteidx < ident_pte; pteidx += PTRS_PER_PTE)
1666 set_page_prot(&level1_ident_pgt[pteidx], PAGE_KERNEL_RO);
1667
1668 set_page_prot(pmd, PAGE_KERNEL_RO);
1669}
1670
1671void __init xen_setup_machphys_mapping(void)
1672{
1673 struct xen_machphys_mapping mapping;
1674 unsigned long machine_to_phys_nr_ents;
1675
1676 if (HYPERVISOR_memory_op(XENMEM_machphys_mapping, &mapping) == 0) {
1677 machine_to_phys_mapping = (unsigned long *)mapping.v_start;
1678 machine_to_phys_nr_ents = mapping.max_mfn + 1;
1679 } else {
1680 machine_to_phys_nr_ents = MACH2PHYS_NR_ENTRIES;
1681 }
1682 machine_to_phys_order = fls(machine_to_phys_nr_ents - 1);
1683}
1684
1685#ifdef CONFIG_X86_64
1686static void convert_pfn_mfn(void *v)
1687{
1688 pte_t *pte = v;
1689 int i;
1690
1691
1692
1693 for (i = 0; i < PTRS_PER_PTE; i++)
1694 pte[i] = xen_make_pte(pte[i].pte);
1695}
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708__init pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd,
1709 unsigned long max_pfn)
1710{
1711 pud_t *l3;
1712 pmd_t *l2;
1713
1714
1715 init_level4_pgt[0] = __pgd(0);
1716
1717
1718 convert_pfn_mfn(init_level4_pgt);
1719 convert_pfn_mfn(level3_ident_pgt);
1720 convert_pfn_mfn(level3_kernel_pgt);
1721
1722 l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd);
1723 l2 = m2v(l3[pud_index(__START_KERNEL_map)].pud);
1724
1725 memcpy(level2_ident_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
1726 memcpy(level2_kernel_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
1727
1728 l3 = m2v(pgd[pgd_index(__START_KERNEL_map + PMD_SIZE)].pgd);
1729 l2 = m2v(l3[pud_index(__START_KERNEL_map + PMD_SIZE)].pud);
1730 memcpy(level2_fixmap_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
1731
1732
1733 xen_map_identity_early(level2_ident_pgt, max_pfn);
1734
1735
1736 set_page_prot(init_level4_pgt, PAGE_KERNEL_RO);
1737 set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO);
1738 set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO);
1739 set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO);
1740 set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
1741 set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO);
1742
1743
1744 pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE,
1745 PFN_DOWN(__pa_symbol(init_level4_pgt)));
1746
1747
1748 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
1749
1750
1751 pgd = init_level4_pgt;
1752
1753
1754
1755
1756
1757
1758 xen_mc_batch();
1759 __xen_write_cr3(true, __pa(pgd));
1760 xen_mc_issue(PARAVIRT_LAZY_CPU);
1761
1762 memblock_x86_reserve_range(__pa(xen_start_info->pt_base),
1763 __pa(xen_start_info->pt_base +
1764 xen_start_info->nr_pt_frames * PAGE_SIZE),
1765 "XEN PAGETABLES");
1766
1767 return pgd;
1768}
1769#else
1770static RESERVE_BRK_ARRAY(pmd_t, initial_kernel_pmd, PTRS_PER_PMD);
1771static RESERVE_BRK_ARRAY(pmd_t, swapper_kernel_pmd, PTRS_PER_PMD);
1772
1773static __init void xen_write_cr3_init(unsigned long cr3)
1774{
1775 unsigned long pfn = PFN_DOWN(__pa(swapper_pg_dir));
1776
1777 BUG_ON(read_cr3() != __pa(initial_page_table));
1778 BUG_ON(cr3 != __pa(swapper_pg_dir));
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790 swapper_kernel_pmd =
1791 extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE);
1792 memcpy(swapper_kernel_pmd, initial_kernel_pmd,
1793 sizeof(pmd_t) * PTRS_PER_PMD);
1794 swapper_pg_dir[KERNEL_PGD_BOUNDARY] =
1795 __pgd(__pa(swapper_kernel_pmd) | _PAGE_PRESENT);
1796 set_page_prot(swapper_kernel_pmd, PAGE_KERNEL_RO);
1797
1798 set_page_prot(swapper_pg_dir, PAGE_KERNEL_RO);
1799 xen_write_cr3(cr3);
1800 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, pfn);
1801
1802 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE,
1803 PFN_DOWN(__pa(initial_page_table)));
1804 set_page_prot(initial_page_table, PAGE_KERNEL);
1805 set_page_prot(initial_kernel_pmd, PAGE_KERNEL);
1806
1807 pv_mmu_ops.write_cr3 = &xen_write_cr3;
1808}
1809
1810__init pgd_t *xen_setup_kernel_pagetable(pgd_t *pgd,
1811 unsigned long max_pfn)
1812{
1813 pmd_t *kernel_pmd;
1814
1815 initial_kernel_pmd =
1816 extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE);
1817
1818 max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->pt_base) +
1819 xen_start_info->nr_pt_frames * PAGE_SIZE +
1820 512*1024);
1821
1822 kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd);
1823 memcpy(initial_kernel_pmd, kernel_pmd, sizeof(pmd_t) * PTRS_PER_PMD);
1824
1825 xen_map_identity_early(initial_kernel_pmd, max_pfn);
1826
1827 memcpy(initial_page_table, pgd, sizeof(pgd_t) * PTRS_PER_PGD);
1828 initial_page_table[KERNEL_PGD_BOUNDARY] =
1829 __pgd(__pa(initial_kernel_pmd) | _PAGE_PRESENT);
1830
1831 set_page_prot(initial_kernel_pmd, PAGE_KERNEL_RO);
1832 set_page_prot(initial_page_table, PAGE_KERNEL_RO);
1833 set_page_prot(empty_zero_page, PAGE_KERNEL_RO);
1834
1835 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
1836
1837 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE,
1838 PFN_DOWN(__pa(initial_page_table)));
1839 xen_write_cr3(__pa(initial_page_table));
1840
1841 memblock_x86_reserve_range(__pa(xen_start_info->pt_base),
1842 __pa(xen_start_info->pt_base +
1843 xen_start_info->nr_pt_frames * PAGE_SIZE),
1844 "XEN PAGETABLES");
1845
1846 return initial_page_table;
1847}
1848#endif
1849
1850static unsigned char dummy_mapping[PAGE_SIZE] __page_aligned_bss;
1851
1852static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot)
1853{
1854 pte_t pte;
1855
1856 phys >>= PAGE_SHIFT;
1857
1858 switch (idx) {
1859 case FIX_BTMAP_END ... FIX_BTMAP_BEGIN:
1860#ifdef CONFIG_X86_F00F_BUG
1861 case FIX_F00F_IDT:
1862#endif
1863#ifdef CONFIG_X86_32
1864 case FIX_WP_TEST:
1865 case FIX_VDSO:
1866# ifdef CONFIG_HIGHMEM
1867 case FIX_KMAP_BEGIN ... FIX_KMAP_END:
1868# endif
1869#else
1870 case VSYSCALL_LAST_PAGE ... VSYSCALL_FIRST_PAGE:
1871#endif
1872 case FIX_TEXT_POKE0:
1873 case FIX_TEXT_POKE1:
1874
1875 pte = pfn_pte(phys, prot);
1876 break;
1877
1878#ifdef CONFIG_X86_LOCAL_APIC
1879 case FIX_APIC_BASE:
1880 pte = pfn_pte(PFN_DOWN(__pa(dummy_mapping)), PAGE_KERNEL);
1881 break;
1882#endif
1883
1884#ifdef CONFIG_X86_IO_APIC
1885 case FIX_IO_APIC_BASE_0 ... FIX_IO_APIC_BASE_END:
1886
1887
1888
1889
1890 pte = pfn_pte(PFN_DOWN(__pa(dummy_mapping)), PAGE_KERNEL);
1891 break;
1892#endif
1893
1894 case FIX_PARAVIRT_BOOTMAP:
1895
1896
1897 pte = mfn_pte(phys, prot);
1898 break;
1899
1900 default:
1901
1902 pte = mfn_pte(phys, __pgprot(pgprot_val(prot) | _PAGE_IOMAP));
1903 break;
1904 }
1905
1906 __native_set_fixmap(idx, pte);
1907
1908#ifdef CONFIG_X86_64
1909
1910
1911 if (idx >= VSYSCALL_LAST_PAGE && idx <= VSYSCALL_FIRST_PAGE) {
1912 unsigned long vaddr = __fix_to_virt(idx);
1913 set_pte_vaddr_pud(level3_user_vsyscall, vaddr, pte);
1914 }
1915#endif
1916}
1917
1918__init void xen_ident_map_ISA(void)
1919{
1920 unsigned long pa;
1921
1922
1923
1924
1925
1926 if (!xen_initial_domain())
1927 return;
1928
1929 xen_raw_printk("Xen: setup ISA identity maps\n");
1930
1931 for (pa = ISA_START_ADDRESS; pa < ISA_END_ADDRESS; pa += PAGE_SIZE) {
1932 pte_t pte = mfn_pte(PFN_DOWN(pa), PAGE_KERNEL_IO);
1933
1934 if (HYPERVISOR_update_va_mapping(PAGE_OFFSET + pa, pte, 0))
1935 BUG();
1936 }
1937
1938 xen_flush_tlb();
1939}
1940
1941static __init void xen_post_allocator_init(void)
1942{
1943 pv_mmu_ops.set_pte = xen_set_pte;
1944 pv_mmu_ops.set_pmd = xen_set_pmd;
1945 pv_mmu_ops.set_pud = xen_set_pud;
1946#if PAGETABLE_LEVELS == 4
1947 pv_mmu_ops.set_pgd = xen_set_pgd;
1948#endif
1949
1950
1951
1952 pv_mmu_ops.alloc_pte = xen_alloc_pte;
1953 pv_mmu_ops.alloc_pmd = xen_alloc_pmd;
1954 pv_mmu_ops.release_pte = xen_release_pte;
1955 pv_mmu_ops.release_pmd = xen_release_pmd;
1956#if PAGETABLE_LEVELS == 4
1957 pv_mmu_ops.alloc_pud = xen_alloc_pud;
1958 pv_mmu_ops.release_pud = xen_release_pud;
1959#endif
1960
1961#ifdef CONFIG_X86_64
1962 SetPagePinned(virt_to_page(level3_user_vsyscall));
1963#endif
1964 xen_mark_init_mm_pinned();
1965}
1966
1967static void xen_leave_lazy_mmu(void)
1968{
1969 preempt_disable();
1970 xen_mc_flush();
1971 paravirt_leave_lazy_mmu();
1972 preempt_enable();
1973}
1974
1975static const struct pv_mmu_ops xen_mmu_ops __initdata = {
1976 .read_cr2 = xen_read_cr2,
1977 .write_cr2 = xen_write_cr2,
1978
1979 .read_cr3 = xen_read_cr3,
1980#ifdef CONFIG_X86_32
1981 .write_cr3 = xen_write_cr3_init,
1982#else
1983 .write_cr3 = xen_write_cr3,
1984#endif
1985
1986 .flush_tlb_user = xen_flush_tlb,
1987 .flush_tlb_kernel = xen_flush_tlb,
1988 .flush_tlb_single = xen_flush_tlb_single,
1989 .flush_tlb_others = xen_flush_tlb_others,
1990
1991 .pte_update = paravirt_nop,
1992 .pte_update_defer = paravirt_nop,
1993
1994 .pgd_alloc = xen_pgd_alloc,
1995 .pgd_free = xen_pgd_free,
1996
1997 .alloc_pte = xen_alloc_pte_init,
1998 .release_pte = xen_release_pte_init,
1999 .alloc_pmd = xen_alloc_pmd_init,
2000 .release_pmd = xen_release_pmd_init,
2001
2002 .set_pte = xen_set_pte_init,
2003 .set_pte_at = xen_set_pte_at,
2004 .set_pmd = xen_set_pmd_hyper,
2005
2006 .ptep_modify_prot_start = __ptep_modify_prot_start,
2007 .ptep_modify_prot_commit = __ptep_modify_prot_commit,
2008
2009 .pte_val = PV_CALLEE_SAVE(xen_pte_val),
2010 .pgd_val = PV_CALLEE_SAVE(xen_pgd_val),
2011
2012 .make_pte = PV_CALLEE_SAVE(xen_make_pte),
2013 .make_pgd = PV_CALLEE_SAVE(xen_make_pgd),
2014
2015#ifdef CONFIG_X86_PAE
2016 .set_pte_atomic = xen_set_pte_atomic,
2017 .pte_clear = xen_pte_clear,
2018 .pmd_clear = xen_pmd_clear,
2019#endif
2020 .set_pud = xen_set_pud_hyper,
2021
2022 .make_pmd = PV_CALLEE_SAVE(xen_make_pmd),
2023 .pmd_val = PV_CALLEE_SAVE(xen_pmd_val),
2024
2025#if PAGETABLE_LEVELS == 4
2026 .pud_val = PV_CALLEE_SAVE(xen_pud_val),
2027 .make_pud = PV_CALLEE_SAVE(xen_make_pud),
2028 .set_pgd = xen_set_pgd_hyper,
2029
2030 .alloc_pud = xen_alloc_pmd_init,
2031 .release_pud = xen_release_pmd_init,
2032#endif
2033
2034 .activate_mm = xen_activate_mm,
2035 .dup_mmap = xen_dup_mmap,
2036 .exit_mmap = xen_exit_mmap,
2037
2038 .lazy_mode = {
2039 .enter = paravirt_enter_lazy_mmu,
2040 .leave = xen_leave_lazy_mmu,
2041 },
2042
2043 .set_fixmap = xen_set_fixmap,
2044};
2045
2046void __init xen_init_mmu_ops(void)
2047{
2048 x86_init.paging.pagetable_setup_start = xen_pagetable_setup_start;
2049 x86_init.paging.pagetable_setup_done = xen_pagetable_setup_done;
2050 pv_mmu_ops = xen_mmu_ops;
2051
2052 memset(dummy_mapping, 0xff, PAGE_SIZE);
2053}
2054
2055
2056#define MAX_CONTIG_ORDER 9
2057static unsigned long discontig_frames[1<<MAX_CONTIG_ORDER];
2058
2059#define VOID_PTE (mfn_pte(0, __pgprot(0)))
2060static void xen_zap_pfn_range(unsigned long vaddr, unsigned int order,
2061 unsigned long *in_frames,
2062 unsigned long *out_frames)
2063{
2064 int i;
2065 struct multicall_space mcs;
2066
2067 xen_mc_batch();
2068 for (i = 0; i < (1UL<<order); i++, vaddr += PAGE_SIZE) {
2069 mcs = __xen_mc_entry(0);
2070
2071 if (in_frames)
2072 in_frames[i] = virt_to_mfn(vaddr);
2073
2074 MULTI_update_va_mapping(mcs.mc, vaddr, VOID_PTE, 0);
2075 set_phys_to_machine(virt_to_pfn(vaddr), INVALID_P2M_ENTRY);
2076
2077 if (out_frames)
2078 out_frames[i] = virt_to_pfn(vaddr);
2079 }
2080 xen_mc_issue(0);
2081}
2082
2083
2084
2085
2086
2087
2088static void xen_remap_exchanged_ptes(unsigned long vaddr, int order,
2089 unsigned long *mfns,
2090 unsigned long first_mfn)
2091{
2092 unsigned i, limit;
2093 unsigned long mfn;
2094
2095 xen_mc_batch();
2096
2097 limit = 1u << order;
2098 for (i = 0; i < limit; i++, vaddr += PAGE_SIZE) {
2099 struct multicall_space mcs;
2100 unsigned flags;
2101
2102 mcs = __xen_mc_entry(0);
2103 if (mfns)
2104 mfn = mfns[i];
2105 else
2106 mfn = first_mfn + i;
2107
2108 if (i < (limit - 1))
2109 flags = 0;
2110 else {
2111 if (order == 0)
2112 flags = UVMF_INVLPG | UVMF_ALL;
2113 else
2114 flags = UVMF_TLB_FLUSH | UVMF_ALL;
2115 }
2116
2117 MULTI_update_va_mapping(mcs.mc, vaddr,
2118 mfn_pte(mfn, PAGE_KERNEL), flags);
2119
2120 set_phys_to_machine(virt_to_pfn(vaddr), mfn);
2121 }
2122
2123 xen_mc_issue(0);
2124}
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134static int xen_exchange_memory(unsigned long extents_in, unsigned int order_in,
2135 unsigned long *pfns_in,
2136 unsigned long extents_out,
2137 unsigned int order_out,
2138 unsigned long *mfns_out,
2139 unsigned int address_bits)
2140{
2141 long rc;
2142 int success;
2143
2144 struct xen_memory_exchange exchange = {
2145 .in = {
2146 .nr_extents = extents_in,
2147 .extent_order = order_in,
2148 .extent_start = pfns_in,
2149 .domid = DOMID_SELF
2150 },
2151 .out = {
2152 .nr_extents = extents_out,
2153 .extent_order = order_out,
2154 .extent_start = mfns_out,
2155 .address_bits = address_bits,
2156 .domid = DOMID_SELF
2157 }
2158 };
2159
2160 BUG_ON(extents_in << order_in != extents_out << order_out);
2161
2162 rc = HYPERVISOR_memory_op(XENMEM_exchange, &exchange);
2163 success = (exchange.nr_exchanged == extents_in);
2164
2165 BUG_ON(!success && ((exchange.nr_exchanged != 0) || (rc == 0)));
2166 BUG_ON(success && (rc != 0));
2167
2168 return success;
2169}
2170
2171int xen_create_contiguous_region(unsigned long vstart, unsigned int order,
2172 unsigned int address_bits)
2173{
2174 unsigned long *in_frames = discontig_frames, out_frame;
2175 unsigned long flags;
2176 int success;
2177
2178
2179
2180
2181
2182
2183
2184 if (xen_feature(XENFEAT_auto_translated_physmap))
2185 return 0;
2186
2187 if (unlikely(order > MAX_CONTIG_ORDER))
2188 return -ENOMEM;
2189
2190 memset((void *) vstart, 0, PAGE_SIZE << order);
2191
2192 spin_lock_irqsave(&xen_reservation_lock, flags);
2193
2194
2195 xen_zap_pfn_range(vstart, order, in_frames, NULL);
2196
2197
2198 out_frame = virt_to_pfn(vstart);
2199 success = xen_exchange_memory(1UL << order, 0, in_frames,
2200 1, order, &out_frame,
2201 address_bits);
2202
2203
2204 if (success)
2205 xen_remap_exchanged_ptes(vstart, order, NULL, out_frame);
2206 else
2207 xen_remap_exchanged_ptes(vstart, order, in_frames, 0);
2208
2209 spin_unlock_irqrestore(&xen_reservation_lock, flags);
2210
2211 return success ? 0 : -ENOMEM;
2212}
2213EXPORT_SYMBOL_GPL(xen_create_contiguous_region);
2214
2215void xen_destroy_contiguous_region(unsigned long vstart, unsigned int order)
2216{
2217 unsigned long *out_frames = discontig_frames, in_frame;
2218 unsigned long flags;
2219 int success;
2220
2221 if (xen_feature(XENFEAT_auto_translated_physmap))
2222 return;
2223
2224 if (unlikely(order > MAX_CONTIG_ORDER))
2225 return;
2226
2227 memset((void *) vstart, 0, PAGE_SIZE << order);
2228
2229 spin_lock_irqsave(&xen_reservation_lock, flags);
2230
2231
2232 in_frame = virt_to_mfn(vstart);
2233
2234
2235 xen_zap_pfn_range(vstart, order, NULL, out_frames);
2236
2237
2238 success = xen_exchange_memory(1, order, &in_frame, 1UL << order,
2239 0, out_frames, 0);
2240
2241
2242 if (success)
2243 xen_remap_exchanged_ptes(vstart, order, out_frames, 0);
2244 else
2245 xen_remap_exchanged_ptes(vstart, order, NULL, in_frame);
2246
2247 spin_unlock_irqrestore(&xen_reservation_lock, flags);
2248}
2249EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region);
2250
2251#ifdef CONFIG_XEN_PVHVM
2252static void xen_hvm_exit_mmap(struct mm_struct *mm)
2253{
2254 struct xen_hvm_pagetable_dying a;
2255 int rc;
2256
2257 a.domid = DOMID_SELF;
2258 a.gpa = __pa(mm->pgd);
2259 rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a);
2260 WARN_ON_ONCE(rc < 0);
2261}
2262
2263static int is_pagetable_dying_supported(void)
2264{
2265 struct xen_hvm_pagetable_dying a;
2266 int rc = 0;
2267
2268 a.domid = DOMID_SELF;
2269 a.gpa = 0x00;
2270 rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a);
2271 if (rc < 0) {
2272 printk(KERN_DEBUG "HVMOP_pagetable_dying not supported\n");
2273 return 0;
2274 }
2275 return 1;
2276}
2277
2278void __init xen_hvm_init_mmu_ops(void)
2279{
2280 if (is_pagetable_dying_supported())
2281 pv_mmu_ops.exit_mmap = xen_hvm_exit_mmap;
2282}
2283#endif
2284
2285#define REMAP_BATCH_SIZE 16
2286
2287struct remap_data {
2288 unsigned long mfn;
2289 pgprot_t prot;
2290 struct mmu_update *mmu_update;
2291};
2292
2293static int remap_area_mfn_pte_fn(pte_t *ptep, pgtable_t token,
2294 unsigned long addr, void *data)
2295{
2296 struct remap_data *rmd = data;
2297 pte_t pte = pte_mkspecial(pfn_pte(rmd->mfn++, rmd->prot));
2298
2299 rmd->mmu_update->ptr = arbitrary_virt_to_machine(ptep).maddr;
2300 rmd->mmu_update->val = pte_val_ma(pte);
2301 rmd->mmu_update++;
2302
2303 return 0;
2304}
2305
2306int xen_remap_domain_mfn_range(struct vm_area_struct *vma,
2307 unsigned long addr,
2308 unsigned long mfn, int nr,
2309 pgprot_t prot, unsigned domid)
2310{
2311 struct remap_data rmd;
2312 struct mmu_update mmu_update[REMAP_BATCH_SIZE];
2313 int batch;
2314 unsigned long range;
2315 int err = 0;
2316
2317 prot = __pgprot(pgprot_val(prot) | _PAGE_IOMAP);
2318
2319 BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_RESERVED | VM_IO)) ==
2320 (VM_PFNMAP | VM_RESERVED | VM_IO)));
2321
2322 rmd.mfn = mfn;
2323 rmd.prot = prot;
2324
2325 while (nr) {
2326 batch = min(REMAP_BATCH_SIZE, nr);
2327 range = (unsigned long)batch << PAGE_SHIFT;
2328
2329 rmd.mmu_update = mmu_update;
2330 err = apply_to_page_range(vma->vm_mm, addr, range,
2331 remap_area_mfn_pte_fn, &rmd);
2332 if (err)
2333 goto out;
2334
2335 err = -EFAULT;
2336 if (HYPERVISOR_mmu_update(mmu_update, batch, NULL, domid) < 0)
2337 goto out;
2338
2339 nr -= batch;
2340 addr += range;
2341 }
2342
2343 err = 0;
2344out:
2345
2346 flush_tlb_all();
2347
2348 return err;
2349}
2350EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_range);
2351
2352#ifdef CONFIG_XEN_DEBUG_FS
2353
2354static struct dentry *d_mmu_debug;
2355
2356static int __init xen_mmu_debugfs(void)
2357{
2358 struct dentry *d_xen = xen_init_debugfs();
2359
2360 if (d_xen == NULL)
2361 return -ENOMEM;
2362
2363 d_mmu_debug = debugfs_create_dir("mmu", d_xen);
2364
2365 debugfs_create_u8("zero_stats", 0644, d_mmu_debug, &zero_stats);
2366
2367 debugfs_create_u32("pgd_update", 0444, d_mmu_debug, &mmu_stats.pgd_update);
2368 debugfs_create_u32("pgd_update_pinned", 0444, d_mmu_debug,
2369 &mmu_stats.pgd_update_pinned);
2370 debugfs_create_u32("pgd_update_batched", 0444, d_mmu_debug,
2371 &mmu_stats.pgd_update_pinned);
2372
2373 debugfs_create_u32("pud_update", 0444, d_mmu_debug, &mmu_stats.pud_update);
2374 debugfs_create_u32("pud_update_pinned", 0444, d_mmu_debug,
2375 &mmu_stats.pud_update_pinned);
2376 debugfs_create_u32("pud_update_batched", 0444, d_mmu_debug,
2377 &mmu_stats.pud_update_pinned);
2378
2379 debugfs_create_u32("pmd_update", 0444, d_mmu_debug, &mmu_stats.pmd_update);
2380 debugfs_create_u32("pmd_update_pinned", 0444, d_mmu_debug,
2381 &mmu_stats.pmd_update_pinned);
2382 debugfs_create_u32("pmd_update_batched", 0444, d_mmu_debug,
2383 &mmu_stats.pmd_update_pinned);
2384
2385 debugfs_create_u32("pte_update", 0444, d_mmu_debug, &mmu_stats.pte_update);
2386
2387
2388 debugfs_create_u32("pte_update_batched", 0444, d_mmu_debug,
2389 &mmu_stats.pte_update_pinned);
2390
2391 debugfs_create_u32("mmu_update", 0444, d_mmu_debug, &mmu_stats.mmu_update);
2392 debugfs_create_u32("mmu_update_extended", 0444, d_mmu_debug,
2393 &mmu_stats.mmu_update_extended);
2394 xen_debugfs_create_u32_array("mmu_update_histo", 0444, d_mmu_debug,
2395 mmu_stats.mmu_update_histo, 20);
2396
2397 debugfs_create_u32("set_pte_at", 0444, d_mmu_debug, &mmu_stats.set_pte_at);
2398 debugfs_create_u32("set_pte_at_batched", 0444, d_mmu_debug,
2399 &mmu_stats.set_pte_at_batched);
2400 debugfs_create_u32("set_pte_at_current", 0444, d_mmu_debug,
2401 &mmu_stats.set_pte_at_current);
2402 debugfs_create_u32("set_pte_at_kernel", 0444, d_mmu_debug,
2403 &mmu_stats.set_pte_at_kernel);
2404
2405 debugfs_create_u32("prot_commit", 0444, d_mmu_debug, &mmu_stats.prot_commit);
2406 debugfs_create_u32("prot_commit_batched", 0444, d_mmu_debug,
2407 &mmu_stats.prot_commit_batched);
2408
2409 return 0;
2410}
2411fs_initcall(xen_mmu_debugfs);
2412
2413#endif
2414