1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41#include <linux/sched.h>
42#include <linux/highmem.h>
43#include <linux/debugfs.h>
44#include <linux/bug.h>
45#include <linux/vmalloc.h>
46#include <linux/module.h>
47#include <linux/gfp.h>
48#include <linux/memblock.h>
49#include <linux/seq_file.h>
50#include <linux/crash_dump.h>
51
52#include <trace/events/xen.h>
53
54#include <asm/pgtable.h>
55#include <asm/tlbflush.h>
56#include <asm/fixmap.h>
57#include <asm/mmu_context.h>
58#include <asm/setup.h>
59#include <asm/paravirt.h>
60#include <asm/e820.h>
61#include <asm/linkage.h>
62#include <asm/page.h>
63#include <asm/init.h>
64#include <asm/pat.h>
65#include <asm/smp.h>
66
67#include <asm/xen/hypercall.h>
68#include <asm/xen/hypervisor.h>
69
70#include <xen/xen.h>
71#include <xen/page.h>
72#include <xen/interface/xen.h>
73#include <xen/interface/hvm/hvm_op.h>
74#include <xen/interface/version.h>
75#include <xen/interface/memory.h>
76#include <xen/hvc-console.h>
77
78#include "multicalls.h"
79#include "mmu.h"
80#include "debugfs.h"
81
82
83
84
85
86DEFINE_SPINLOCK(xen_reservation_lock);
87
88#ifdef CONFIG_X86_32
89
90
91
92
93
94#define LEVEL1_IDENT_ENTRIES (PTRS_PER_PTE * 4)
95static RESERVE_BRK_ARRAY(pte_t, level1_ident_pgt, LEVEL1_IDENT_ENTRIES);
96#endif
97#ifdef CONFIG_X86_64
98
99static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss;
100#endif
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116DEFINE_PER_CPU(unsigned long, xen_cr3);
117DEFINE_PER_CPU(unsigned long, xen_current_cr3);
118
119
120
121
122
123
124#define USER_LIMIT ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK)
125
126unsigned long arbitrary_virt_to_mfn(void *vaddr)
127{
128 xmaddr_t maddr = arbitrary_virt_to_machine(vaddr);
129
130 return PFN_DOWN(maddr.maddr);
131}
132
133xmaddr_t arbitrary_virt_to_machine(void *vaddr)
134{
135 unsigned long address = (unsigned long)vaddr;
136 unsigned int level;
137 pte_t *pte;
138 unsigned offset;
139
140
141
142
143
144 if (virt_addr_valid(vaddr))
145 return virt_to_machine(vaddr);
146
147
148
149 pte = lookup_address(address, &level);
150 BUG_ON(pte == NULL);
151 offset = address & ~PAGE_MASK;
152 return XMADDR(((phys_addr_t)pte_mfn(*pte) << PAGE_SHIFT) + offset);
153}
154EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine);
155
156void make_lowmem_page_readonly(void *vaddr)
157{
158 pte_t *pte, ptev;
159 unsigned long address = (unsigned long)vaddr;
160 unsigned int level;
161
162 pte = lookup_address(address, &level);
163 if (pte == NULL)
164 return;
165
166 ptev = pte_wrprotect(*pte);
167
168 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
169 BUG();
170}
171
172void make_lowmem_page_readwrite(void *vaddr)
173{
174 pte_t *pte, ptev;
175 unsigned long address = (unsigned long)vaddr;
176 unsigned int level;
177
178 pte = lookup_address(address, &level);
179 if (pte == NULL)
180 return;
181
182 ptev = pte_mkwrite(*pte);
183
184 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
185 BUG();
186}
187
188
189static bool xen_page_pinned(void *ptr)
190{
191 struct page *page = virt_to_page(ptr);
192
193 return PagePinned(page);
194}
195
196void xen_set_domain_pte(pte_t *ptep, pte_t pteval, unsigned domid)
197{
198 struct multicall_space mcs;
199 struct mmu_update *u;
200
201 trace_xen_mmu_set_domain_pte(ptep, pteval, domid);
202
203 mcs = xen_mc_entry(sizeof(*u));
204 u = mcs.args;
205
206
207 u->ptr = virt_to_machine(ptep).maddr;
208 u->val = pte_val_ma(pteval);
209
210 MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, domid);
211
212 xen_mc_issue(PARAVIRT_LAZY_MMU);
213}
214EXPORT_SYMBOL_GPL(xen_set_domain_pte);
215
216static void xen_extend_mmu_update(const struct mmu_update *update)
217{
218 struct multicall_space mcs;
219 struct mmu_update *u;
220
221 mcs = xen_mc_extend_args(__HYPERVISOR_mmu_update, sizeof(*u));
222
223 if (mcs.mc != NULL) {
224 mcs.mc->args[1]++;
225 } else {
226 mcs = __xen_mc_entry(sizeof(*u));
227 MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
228 }
229
230 u = mcs.args;
231 *u = *update;
232}
233
234static void xen_extend_mmuext_op(const struct mmuext_op *op)
235{
236 struct multicall_space mcs;
237 struct mmuext_op *u;
238
239 mcs = xen_mc_extend_args(__HYPERVISOR_mmuext_op, sizeof(*u));
240
241 if (mcs.mc != NULL) {
242 mcs.mc->args[1]++;
243 } else {
244 mcs = __xen_mc_entry(sizeof(*u));
245 MULTI_mmuext_op(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
246 }
247
248 u = mcs.args;
249 *u = *op;
250}
251
252static void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val)
253{
254 struct mmu_update u;
255
256 preempt_disable();
257
258 xen_mc_batch();
259
260
261 u.ptr = arbitrary_virt_to_machine(ptr).maddr;
262 u.val = pmd_val_ma(val);
263 xen_extend_mmu_update(&u);
264
265 xen_mc_issue(PARAVIRT_LAZY_MMU);
266
267 preempt_enable();
268}
269
270static void xen_set_pmd(pmd_t *ptr, pmd_t val)
271{
272 trace_xen_mmu_set_pmd(ptr, val);
273
274
275
276 if (!xen_page_pinned(ptr)) {
277 *ptr = val;
278 return;
279 }
280
281 xen_set_pmd_hyper(ptr, val);
282}
283
284
285
286
287
288void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags)
289{
290 set_pte_vaddr(vaddr, mfn_pte(mfn, flags));
291}
292
293static bool xen_batched_set_pte(pte_t *ptep, pte_t pteval)
294{
295 struct mmu_update u;
296
297 if (paravirt_get_lazy_mode() != PARAVIRT_LAZY_MMU)
298 return false;
299
300 xen_mc_batch();
301
302 u.ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE;
303 u.val = pte_val_ma(pteval);
304 xen_extend_mmu_update(&u);
305
306 xen_mc_issue(PARAVIRT_LAZY_MMU);
307
308 return true;
309}
310
311static inline void __xen_set_pte(pte_t *ptep, pte_t pteval)
312{
313 if (!xen_batched_set_pte(ptep, pteval)) {
314
315
316
317
318
319
320
321 struct mmu_update u;
322
323 u.ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE;
324 u.val = pte_val_ma(pteval);
325 HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF);
326 }
327}
328
329static void xen_set_pte(pte_t *ptep, pte_t pteval)
330{
331 trace_xen_mmu_set_pte(ptep, pteval);
332 __xen_set_pte(ptep, pteval);
333}
334
335static void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
336 pte_t *ptep, pte_t pteval)
337{
338 trace_xen_mmu_set_pte_at(mm, addr, ptep, pteval);
339 __xen_set_pte(ptep, pteval);
340}
341
342pte_t xen_ptep_modify_prot_start(struct mm_struct *mm,
343 unsigned long addr, pte_t *ptep)
344{
345
346 trace_xen_mmu_ptep_modify_prot_start(mm, addr, ptep, *ptep);
347 return *ptep;
348}
349
350void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
351 pte_t *ptep, pte_t pte)
352{
353 struct mmu_update u;
354
355 trace_xen_mmu_ptep_modify_prot_commit(mm, addr, ptep, pte);
356 xen_mc_batch();
357
358 u.ptr = virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD;
359 u.val = pte_val_ma(pte);
360 xen_extend_mmu_update(&u);
361
362 xen_mc_issue(PARAVIRT_LAZY_MMU);
363}
364
365
366static pteval_t pte_mfn_to_pfn(pteval_t val)
367{
368 if (val & _PAGE_PRESENT) {
369 unsigned long mfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
370 unsigned long pfn = mfn_to_pfn(mfn);
371
372 pteval_t flags = val & PTE_FLAGS_MASK;
373 if (unlikely(pfn == ~0))
374 val = flags & ~_PAGE_PRESENT;
375 else
376 val = ((pteval_t)pfn << PAGE_SHIFT) | flags;
377 }
378
379 return val;
380}
381
382static pteval_t pte_pfn_to_mfn(pteval_t val)
383{
384 if (val & _PAGE_PRESENT) {
385 unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
386 pteval_t flags = val & PTE_FLAGS_MASK;
387 unsigned long mfn;
388
389 if (!xen_feature(XENFEAT_auto_translated_physmap))
390 mfn = get_phys_to_machine(pfn);
391 else
392 mfn = pfn;
393
394
395
396
397
398
399 if (unlikely(mfn == INVALID_P2M_ENTRY)) {
400 mfn = 0;
401 flags = 0;
402 } else {
403
404
405
406
407
408 mfn &= ~FOREIGN_FRAME_BIT;
409 if (mfn & IDENTITY_FRAME_BIT) {
410 mfn &= ~IDENTITY_FRAME_BIT;
411 flags |= _PAGE_IOMAP;
412 }
413 }
414 val = ((pteval_t)mfn << PAGE_SHIFT) | flags;
415 }
416
417 return val;
418}
419
420static pteval_t iomap_pte(pteval_t val)
421{
422 if (val & _PAGE_PRESENT) {
423 unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
424 pteval_t flags = val & PTE_FLAGS_MASK;
425
426
427
428 val = ((pteval_t)pfn << PAGE_SHIFT) | flags;
429 }
430
431 return val;
432}
433
434static pteval_t xen_pte_val(pte_t pte)
435{
436 pteval_t pteval = pte.pte;
437#if 0
438
439 if ((pteval & (_PAGE_PAT | _PAGE_PCD | _PAGE_PWT)) == _PAGE_PAT) {
440 WARN_ON(!pat_enabled);
441 pteval = (pteval & ~_PAGE_PAT) | _PAGE_PWT;
442 }
443#endif
444 if (xen_initial_domain() && (pteval & _PAGE_IOMAP))
445 return pteval;
446
447 return pte_mfn_to_pfn(pteval);
448}
449PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val);
450
451static pgdval_t xen_pgd_val(pgd_t pgd)
452{
453 return pte_mfn_to_pfn(pgd.pgd);
454}
455PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val);
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475void xen_set_pat(u64 pat)
476{
477
478
479 WARN_ON(pat != 0x0007010600070106ull);
480}
481
482static pte_t xen_make_pte(pteval_t pte)
483{
484 phys_addr_t addr = (pte & PTE_PFN_MASK);
485#if 0
486
487
488
489
490
491
492
493
494 if (pat_enabled && !WARN_ON(pte & _PAGE_PAT)) {
495 if ((pte & (_PAGE_PCD | _PAGE_PWT)) == _PAGE_PWT)
496 pte = (pte & ~(_PAGE_PCD | _PAGE_PWT)) | _PAGE_PAT;
497 }
498#endif
499
500
501
502
503
504
505 if (unlikely(pte & _PAGE_IOMAP) &&
506 (xen_initial_domain() || addr >= ISA_END_ADDRESS)) {
507 pte = iomap_pte(pte);
508 } else {
509 pte &= ~_PAGE_IOMAP;
510 pte = pte_pfn_to_mfn(pte);
511 }
512
513 return native_make_pte(pte);
514}
515PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte);
516
517static pgd_t xen_make_pgd(pgdval_t pgd)
518{
519 pgd = pte_pfn_to_mfn(pgd);
520 return native_make_pgd(pgd);
521}
522PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd);
523
524static pmdval_t xen_pmd_val(pmd_t pmd)
525{
526 return pte_mfn_to_pfn(pmd.pmd);
527}
528PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val);
529
530static void xen_set_pud_hyper(pud_t *ptr, pud_t val)
531{
532 struct mmu_update u;
533
534 preempt_disable();
535
536 xen_mc_batch();
537
538
539 u.ptr = arbitrary_virt_to_machine(ptr).maddr;
540 u.val = pud_val_ma(val);
541 xen_extend_mmu_update(&u);
542
543 xen_mc_issue(PARAVIRT_LAZY_MMU);
544
545 preempt_enable();
546}
547
548static void xen_set_pud(pud_t *ptr, pud_t val)
549{
550 trace_xen_mmu_set_pud(ptr, val);
551
552
553
554 if (!xen_page_pinned(ptr)) {
555 *ptr = val;
556 return;
557 }
558
559 xen_set_pud_hyper(ptr, val);
560}
561
562#ifdef CONFIG_X86_PAE
563static void xen_set_pte_atomic(pte_t *ptep, pte_t pte)
564{
565 trace_xen_mmu_set_pte_atomic(ptep, pte);
566 set_64bit((u64 *)ptep, native_pte_val(pte));
567}
568
569static void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
570{
571 trace_xen_mmu_pte_clear(mm, addr, ptep);
572 if (!xen_batched_set_pte(ptep, native_make_pte(0)))
573 native_pte_clear(mm, addr, ptep);
574}
575
576static void xen_pmd_clear(pmd_t *pmdp)
577{
578 trace_xen_mmu_pmd_clear(pmdp);
579 set_pmd(pmdp, __pmd(0));
580}
581#endif
582
583static pmd_t xen_make_pmd(pmdval_t pmd)
584{
585 pmd = pte_pfn_to_mfn(pmd);
586 return native_make_pmd(pmd);
587}
588PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd);
589
590#if PAGETABLE_LEVELS == 4
591static pudval_t xen_pud_val(pud_t pud)
592{
593 return pte_mfn_to_pfn(pud.pud);
594}
595PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val);
596
597static pud_t xen_make_pud(pudval_t pud)
598{
599 pud = pte_pfn_to_mfn(pud);
600
601 return native_make_pud(pud);
602}
603PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud);
604
605static pgd_t *xen_get_user_pgd(pgd_t *pgd)
606{
607 pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK);
608 unsigned offset = pgd - pgd_page;
609 pgd_t *user_ptr = NULL;
610
611 if (offset < pgd_index(USER_LIMIT)) {
612 struct page *page = virt_to_page(pgd_page);
613 user_ptr = (pgd_t *)page->private;
614 if (user_ptr)
615 user_ptr += offset;
616 }
617
618 return user_ptr;
619}
620
621static void __xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
622{
623 struct mmu_update u;
624
625 u.ptr = virt_to_machine(ptr).maddr;
626 u.val = pgd_val_ma(val);
627 xen_extend_mmu_update(&u);
628}
629
630
631
632
633
634
635
636
637static void __init xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
638{
639 preempt_disable();
640
641 xen_mc_batch();
642
643 __xen_set_pgd_hyper(ptr, val);
644
645 xen_mc_issue(PARAVIRT_LAZY_MMU);
646
647 preempt_enable();
648}
649
650static void xen_set_pgd(pgd_t *ptr, pgd_t val)
651{
652 pgd_t *user_ptr = xen_get_user_pgd(ptr);
653
654 trace_xen_mmu_set_pgd(ptr, user_ptr, val);
655
656
657
658 if (!xen_page_pinned(ptr)) {
659 *ptr = val;
660 if (user_ptr) {
661 WARN_ON(xen_page_pinned(user_ptr));
662 *user_ptr = val;
663 }
664 return;
665 }
666
667
668
669 xen_mc_batch();
670
671 __xen_set_pgd_hyper(ptr, val);
672 if (user_ptr)
673 __xen_set_pgd_hyper(user_ptr, val);
674
675 xen_mc_issue(PARAVIRT_LAZY_MMU);
676}
677#endif
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694static int __xen_pgd_walk(struct mm_struct *mm, pgd_t *pgd,
695 int (*func)(struct mm_struct *mm, struct page *,
696 enum pt_level),
697 unsigned long limit)
698{
699 int flush = 0;
700 unsigned hole_low, hole_high;
701 unsigned pgdidx_limit, pudidx_limit, pmdidx_limit;
702 unsigned pgdidx, pudidx, pmdidx;
703
704
705 limit--;
706 BUG_ON(limit >= FIXADDR_TOP);
707
708 if (xen_feature(XENFEAT_auto_translated_physmap))
709 return 0;
710
711
712
713
714
715
716 hole_low = pgd_index(USER_LIMIT);
717 hole_high = pgd_index(PAGE_OFFSET);
718
719 pgdidx_limit = pgd_index(limit);
720#if PTRS_PER_PUD > 1
721 pudidx_limit = pud_index(limit);
722#else
723 pudidx_limit = 0;
724#endif
725#if PTRS_PER_PMD > 1
726 pmdidx_limit = pmd_index(limit);
727#else
728 pmdidx_limit = 0;
729#endif
730
731 for (pgdidx = 0; pgdidx <= pgdidx_limit; pgdidx++) {
732 pud_t *pud;
733
734 if (pgdidx >= hole_low && pgdidx < hole_high)
735 continue;
736
737 if (!pgd_val(pgd[pgdidx]))
738 continue;
739
740 pud = pud_offset(&pgd[pgdidx], 0);
741
742 if (PTRS_PER_PUD > 1)
743 flush |= (*func)(mm, virt_to_page(pud), PT_PUD);
744
745 for (pudidx = 0; pudidx < PTRS_PER_PUD; pudidx++) {
746 pmd_t *pmd;
747
748 if (pgdidx == pgdidx_limit &&
749 pudidx > pudidx_limit)
750 goto out;
751
752 if (pud_none(pud[pudidx]))
753 continue;
754
755 pmd = pmd_offset(&pud[pudidx], 0);
756
757 if (PTRS_PER_PMD > 1)
758 flush |= (*func)(mm, virt_to_page(pmd), PT_PMD);
759
760 for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++) {
761 struct page *pte;
762
763 if (pgdidx == pgdidx_limit &&
764 pudidx == pudidx_limit &&
765 pmdidx > pmdidx_limit)
766 goto out;
767
768 if (pmd_none(pmd[pmdidx]))
769 continue;
770
771 pte = pmd_page(pmd[pmdidx]);
772 flush |= (*func)(mm, pte, PT_PTE);
773 }
774 }
775 }
776
777out:
778
779
780 flush |= (*func)(mm, virt_to_page(pgd), PT_PGD);
781
782 return flush;
783}
784
785static int xen_pgd_walk(struct mm_struct *mm,
786 int (*func)(struct mm_struct *mm, struct page *,
787 enum pt_level),
788 unsigned long limit)
789{
790 return __xen_pgd_walk(mm, mm->pgd, func, limit);
791}
792
793
794
795static spinlock_t *xen_pte_lock(struct page *page, struct mm_struct *mm)
796{
797 spinlock_t *ptl = NULL;
798
799#if USE_SPLIT_PTLOCKS
800 ptl = __pte_lockptr(page);
801 spin_lock_nest_lock(ptl, &mm->page_table_lock);
802#endif
803
804 return ptl;
805}
806
807static void xen_pte_unlock(void *v)
808{
809 spinlock_t *ptl = v;
810 spin_unlock(ptl);
811}
812
813static void xen_do_pin(unsigned level, unsigned long pfn)
814{
815 struct mmuext_op op;
816
817 op.cmd = level;
818 op.arg1.mfn = pfn_to_mfn(pfn);
819
820 xen_extend_mmuext_op(&op);
821}
822
823static int xen_pin_page(struct mm_struct *mm, struct page *page,
824 enum pt_level level)
825{
826 unsigned pgfl = TestSetPagePinned(page);
827 int flush;
828
829 if (pgfl)
830 flush = 0;
831 else if (PageHighMem(page))
832
833
834 flush = 1;
835 else {
836 void *pt = lowmem_page_address(page);
837 unsigned long pfn = page_to_pfn(page);
838 struct multicall_space mcs = __xen_mc_entry(0);
839 spinlock_t *ptl;
840
841 flush = 0;
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863 ptl = NULL;
864 if (level == PT_PTE)
865 ptl = xen_pte_lock(page, mm);
866
867 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
868 pfn_pte(pfn, PAGE_KERNEL_RO),
869 level == PT_PGD ? UVMF_TLB_FLUSH : 0);
870
871 if (ptl) {
872 xen_do_pin(MMUEXT_PIN_L1_TABLE, pfn);
873
874
875
876 xen_mc_callback(xen_pte_unlock, ptl);
877 }
878 }
879
880 return flush;
881}
882
883
884
885
886static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd)
887{
888 trace_xen_mmu_pgd_pin(mm, pgd);
889
890 xen_mc_batch();
891
892 if (__xen_pgd_walk(mm, pgd, xen_pin_page, USER_LIMIT)) {
893
894 xen_mc_issue(0);
895
896 kmap_flush_unused();
897
898 xen_mc_batch();
899 }
900
901#ifdef CONFIG_X86_64
902 {
903 pgd_t *user_pgd = xen_get_user_pgd(pgd);
904
905 xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(pgd)));
906
907 if (user_pgd) {
908 xen_pin_page(mm, virt_to_page(user_pgd), PT_PGD);
909 xen_do_pin(MMUEXT_PIN_L4_TABLE,
910 PFN_DOWN(__pa(user_pgd)));
911 }
912 }
913#else
914#ifdef CONFIG_X86_PAE
915
916 xen_pin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),
917 PT_PMD);
918#endif
919 xen_do_pin(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(pgd)));
920#endif
921 xen_mc_issue(0);
922}
923
924static void xen_pgd_pin(struct mm_struct *mm)
925{
926 __xen_pgd_pin(mm, mm->pgd);
927}
928
929
930
931
932
933
934
935
936
937
938
939void xen_mm_pin_all(void)
940{
941 struct page *page;
942
943 spin_lock(&pgd_lock);
944
945 list_for_each_entry(page, &pgd_list, lru) {
946 if (!PagePinned(page)) {
947 __xen_pgd_pin(&init_mm, (pgd_t *)page_address(page));
948 SetPageSavePinned(page);
949 }
950 }
951
952 spin_unlock(&pgd_lock);
953}
954
955
956
957
958
959
960static int __init xen_mark_pinned(struct mm_struct *mm, struct page *page,
961 enum pt_level level)
962{
963 SetPagePinned(page);
964 return 0;
965}
966
967static void __init xen_mark_init_mm_pinned(void)
968{
969 xen_pgd_walk(&init_mm, xen_mark_pinned, FIXADDR_TOP);
970}
971
972static int xen_unpin_page(struct mm_struct *mm, struct page *page,
973 enum pt_level level)
974{
975 unsigned pgfl = TestClearPagePinned(page);
976
977 if (pgfl && !PageHighMem(page)) {
978 void *pt = lowmem_page_address(page);
979 unsigned long pfn = page_to_pfn(page);
980 spinlock_t *ptl = NULL;
981 struct multicall_space mcs;
982
983
984
985
986
987
988
989
990 if (level == PT_PTE) {
991 ptl = xen_pte_lock(page, mm);
992
993 if (ptl)
994 xen_do_pin(MMUEXT_UNPIN_TABLE, pfn);
995 }
996
997 mcs = __xen_mc_entry(0);
998
999 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
1000 pfn_pte(pfn, PAGE_KERNEL),
1001 level == PT_PGD ? UVMF_TLB_FLUSH : 0);
1002
1003 if (ptl) {
1004
1005 xen_mc_callback(xen_pte_unlock, ptl);
1006 }
1007 }
1008
1009 return 0;
1010}
1011
1012
1013static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd)
1014{
1015 trace_xen_mmu_pgd_unpin(mm, pgd);
1016
1017 xen_mc_batch();
1018
1019 xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
1020
1021#ifdef CONFIG_X86_64
1022 {
1023 pgd_t *user_pgd = xen_get_user_pgd(pgd);
1024
1025 if (user_pgd) {
1026 xen_do_pin(MMUEXT_UNPIN_TABLE,
1027 PFN_DOWN(__pa(user_pgd)));
1028 xen_unpin_page(mm, virt_to_page(user_pgd), PT_PGD);
1029 }
1030 }
1031#endif
1032
1033#ifdef CONFIG_X86_PAE
1034
1035 xen_unpin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),
1036 PT_PMD);
1037#endif
1038
1039 __xen_pgd_walk(mm, pgd, xen_unpin_page, USER_LIMIT);
1040
1041 xen_mc_issue(0);
1042}
1043
1044static void xen_pgd_unpin(struct mm_struct *mm)
1045{
1046 __xen_pgd_unpin(mm, mm->pgd);
1047}
1048
1049
1050
1051
1052
1053void xen_mm_unpin_all(void)
1054{
1055 struct page *page;
1056
1057 spin_lock(&pgd_lock);
1058
1059 list_for_each_entry(page, &pgd_list, lru) {
1060 if (PageSavePinned(page)) {
1061 BUG_ON(!PagePinned(page));
1062 __xen_pgd_unpin(&init_mm, (pgd_t *)page_address(page));
1063 ClearPageSavePinned(page);
1064 }
1065 }
1066
1067 spin_unlock(&pgd_lock);
1068}
1069
1070static void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next)
1071{
1072 spin_lock(&next->page_table_lock);
1073 xen_pgd_pin(next);
1074 spin_unlock(&next->page_table_lock);
1075}
1076
1077static void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
1078{
1079 spin_lock(&mm->page_table_lock);
1080 xen_pgd_pin(mm);
1081 spin_unlock(&mm->page_table_lock);
1082}
1083
1084
1085#ifdef CONFIG_SMP
1086
1087
1088static void drop_other_mm_ref(void *info)
1089{
1090 struct mm_struct *mm = info;
1091 struct mm_struct *active_mm;
1092
1093 active_mm = this_cpu_read(cpu_tlbstate.active_mm);
1094
1095 if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
1096 leave_mm(smp_processor_id());
1097
1098
1099
1100 if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))
1101 load_cr3(swapper_pg_dir);
1102}
1103
1104static void xen_drop_mm_ref(struct mm_struct *mm)
1105{
1106 cpumask_var_t mask;
1107 unsigned cpu;
1108
1109 if (current->active_mm == mm) {
1110 if (current->mm == mm)
1111 load_cr3(swapper_pg_dir);
1112 else
1113 leave_mm(smp_processor_id());
1114 }
1115
1116
1117 if (!alloc_cpumask_var(&mask, GFP_ATOMIC)) {
1118 for_each_online_cpu(cpu) {
1119 if (!cpumask_test_cpu(cpu, mm_cpumask(mm))
1120 && per_cpu(xen_current_cr3, cpu) != __pa(mm->pgd))
1121 continue;
1122 smp_call_function_single(cpu, drop_other_mm_ref, mm, 1);
1123 }
1124 return;
1125 }
1126 cpumask_copy(mask, mm_cpumask(mm));
1127
1128
1129
1130
1131
1132
1133 for_each_online_cpu(cpu) {
1134 if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd))
1135 cpumask_set_cpu(cpu, mask);
1136 }
1137
1138 if (!cpumask_empty(mask))
1139 smp_call_function_many(mask, drop_other_mm_ref, mm, 1);
1140 free_cpumask_var(mask);
1141}
1142#else
1143static void xen_drop_mm_ref(struct mm_struct *mm)
1144{
1145 if (current->active_mm == mm)
1146 load_cr3(swapper_pg_dir);
1147}
1148#endif
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164static void xen_exit_mmap(struct mm_struct *mm)
1165{
1166 get_cpu();
1167 xen_drop_mm_ref(mm);
1168 put_cpu();
1169
1170 spin_lock(&mm->page_table_lock);
1171
1172
1173 if (xen_page_pinned(mm->pgd))
1174 xen_pgd_unpin(mm);
1175
1176 spin_unlock(&mm->page_table_lock);
1177}
1178
1179static void xen_post_allocator_init(void);
1180
1181static __init void xen_mapping_pagetable_reserve(u64 start, u64 end)
1182{
1183
1184 native_pagetable_reserve(start, end);
1185
1186
1187 printk(KERN_DEBUG "xen: setting RW the range %llx - %llx\n", end,
1188 PFN_PHYS(pgt_buf_top));
1189 while (end < PFN_PHYS(pgt_buf_top)) {
1190 make_lowmem_page_readwrite(__va(end));
1191 end += PAGE_SIZE;
1192 }
1193}
1194
1195#ifdef CONFIG_X86_64
1196static void __init xen_cleanhighmap(unsigned long vaddr,
1197 unsigned long vaddr_end)
1198{
1199 unsigned long kernel_end = roundup((unsigned long)_brk_end, PMD_SIZE) - 1;
1200 pmd_t *pmd = level2_kernel_pgt + pmd_index(vaddr);
1201
1202
1203
1204 for (; vaddr <= vaddr_end && (pmd < (level2_kernel_pgt + PAGE_SIZE));
1205 pmd++, vaddr += PMD_SIZE) {
1206 if (pmd_none(*pmd))
1207 continue;
1208 if (vaddr < (unsigned long) _text || vaddr > kernel_end)
1209 set_pmd(pmd, __pmd(0));
1210 }
1211
1212
1213 xen_mc_flush();
1214}
1215#endif
1216static void __init xen_pagetable_init(void)
1217{
1218#ifdef CONFIG_X86_64
1219 unsigned long size;
1220 unsigned long addr;
1221#endif
1222 paging_init();
1223 xen_setup_shared_info();
1224#ifdef CONFIG_X86_64
1225 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
1226 unsigned long new_mfn_list;
1227
1228 size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long));
1229
1230
1231 new_mfn_list = xen_revector_p2m_tree();
1232 if (new_mfn_list && new_mfn_list != xen_start_info->mfn_list) {
1233
1234 memset((void *)xen_start_info->mfn_list, 0xff, size);
1235
1236
1237 BUG_ON(xen_start_info->mfn_list < __START_KERNEL_map);
1238 addr = xen_start_info->mfn_list;
1239
1240
1241
1242
1243 size = roundup(size, PMD_SIZE);
1244 xen_cleanhighmap(addr, addr + size);
1245
1246 size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long));
1247 memblock_free(__pa(xen_start_info->mfn_list), size);
1248
1249 xen_start_info->mfn_list = new_mfn_list;
1250 } else
1251 goto skip;
1252 }
1253
1254
1255
1256
1257
1258
1259
1260
1261 addr = xen_start_info->pt_base;
1262 size = roundup(xen_start_info->nr_pt_frames * PAGE_SIZE, PMD_SIZE);
1263
1264 xen_cleanhighmap(addr, addr + size);
1265 xen_start_info->pt_base = (unsigned long)__va(__pa(xen_start_info->pt_base));
1266#ifdef DEBUG
1267
1268
1269
1270 xen_cleanhighmap(MODULES_VADDR, roundup(MODULES_VADDR, PUD_SIZE) - 1);
1271#endif
1272skip:
1273#endif
1274 xen_post_allocator_init();
1275}
1276static void xen_write_cr2(unsigned long cr2)
1277{
1278 this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
1279}
1280
1281static unsigned long xen_read_cr2(void)
1282{
1283 return this_cpu_read(xen_vcpu)->arch.cr2;
1284}
1285
1286unsigned long xen_read_cr2_direct(void)
1287{
1288 return this_cpu_read(xen_vcpu_info.arch.cr2);
1289}
1290
1291void xen_flush_tlb_all(void)
1292{
1293 struct mmuext_op *op;
1294 struct multicall_space mcs;
1295
1296 trace_xen_mmu_flush_tlb_all(0);
1297
1298 preempt_disable();
1299
1300 mcs = xen_mc_entry(sizeof(*op));
1301
1302 op = mcs.args;
1303 op->cmd = MMUEXT_TLB_FLUSH_ALL;
1304 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1305
1306 xen_mc_issue(PARAVIRT_LAZY_MMU);
1307
1308 preempt_enable();
1309}
1310static void xen_flush_tlb(void)
1311{
1312 struct mmuext_op *op;
1313 struct multicall_space mcs;
1314
1315 trace_xen_mmu_flush_tlb(0);
1316
1317 preempt_disable();
1318
1319 mcs = xen_mc_entry(sizeof(*op));
1320
1321 op = mcs.args;
1322 op->cmd = MMUEXT_TLB_FLUSH_LOCAL;
1323 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1324
1325 xen_mc_issue(PARAVIRT_LAZY_MMU);
1326
1327 preempt_enable();
1328}
1329
1330static void xen_flush_tlb_single(unsigned long addr)
1331{
1332 struct mmuext_op *op;
1333 struct multicall_space mcs;
1334
1335 trace_xen_mmu_flush_tlb_single(addr);
1336
1337 preempt_disable();
1338
1339 mcs = xen_mc_entry(sizeof(*op));
1340 op = mcs.args;
1341 op->cmd = MMUEXT_INVLPG_LOCAL;
1342 op->arg1.linear_addr = addr & PAGE_MASK;
1343 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1344
1345 xen_mc_issue(PARAVIRT_LAZY_MMU);
1346
1347 preempt_enable();
1348}
1349
1350static void xen_flush_tlb_others(const struct cpumask *cpus,
1351 struct mm_struct *mm, unsigned long start,
1352 unsigned long end)
1353{
1354 struct {
1355 struct mmuext_op op;
1356#ifdef CONFIG_SMP
1357 DECLARE_BITMAP(mask, num_processors);
1358#else
1359 DECLARE_BITMAP(mask, NR_CPUS);
1360#endif
1361 } *args;
1362 struct multicall_space mcs;
1363
1364 trace_xen_mmu_flush_tlb_others(cpus, mm, start, end);
1365
1366 if (cpumask_empty(cpus))
1367 return;
1368
1369 mcs = xen_mc_entry(sizeof(*args));
1370 args = mcs.args;
1371 args->op.arg2.vcpumask = to_cpumask(args->mask);
1372
1373
1374 cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
1375 cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
1376
1377 args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
1378 if (end != TLB_FLUSH_ALL && (end - start) <= PAGE_SIZE) {
1379 args->op.cmd = MMUEXT_INVLPG_MULTI;
1380 args->op.arg1.linear_addr = start;
1381 }
1382
1383 MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
1384
1385 xen_mc_issue(PARAVIRT_LAZY_MMU);
1386}
1387
1388static unsigned long xen_read_cr3(void)
1389{
1390 return this_cpu_read(xen_cr3);
1391}
1392
1393static void set_current_cr3(void *v)
1394{
1395 this_cpu_write(xen_current_cr3, (unsigned long)v);
1396}
1397
1398static void __xen_write_cr3(bool kernel, unsigned long cr3)
1399{
1400 struct mmuext_op op;
1401 unsigned long mfn;
1402
1403 trace_xen_mmu_write_cr3(kernel, cr3);
1404
1405 if (cr3)
1406 mfn = pfn_to_mfn(PFN_DOWN(cr3));
1407 else
1408 mfn = 0;
1409
1410 WARN_ON(mfn == 0 && kernel);
1411
1412 op.cmd = kernel ? MMUEXT_NEW_BASEPTR : MMUEXT_NEW_USER_BASEPTR;
1413 op.arg1.mfn = mfn;
1414
1415 xen_extend_mmuext_op(&op);
1416
1417 if (kernel) {
1418 this_cpu_write(xen_cr3, cr3);
1419
1420
1421
1422 xen_mc_callback(set_current_cr3, (void *)cr3);
1423 }
1424}
1425
1426static void xen_write_cr3(unsigned long cr3)
1427{
1428 BUG_ON(preemptible());
1429
1430 xen_mc_batch();
1431
1432
1433
1434 this_cpu_write(xen_cr3, cr3);
1435
1436 __xen_write_cr3(true, cr3);
1437
1438#ifdef CONFIG_X86_64
1439 {
1440 pgd_t *user_pgd = xen_get_user_pgd(__va(cr3));
1441 if (user_pgd)
1442 __xen_write_cr3(false, __pa(user_pgd));
1443 else
1444 __xen_write_cr3(false, 0);
1445 }
1446#endif
1447
1448 xen_mc_issue(PARAVIRT_LAZY_CPU);
1449}
1450
1451static int xen_pgd_alloc(struct mm_struct *mm)
1452{
1453 pgd_t *pgd = mm->pgd;
1454 int ret = 0;
1455
1456 BUG_ON(PagePinned(virt_to_page(pgd)));
1457
1458#ifdef CONFIG_X86_64
1459 {
1460 struct page *page = virt_to_page(pgd);
1461 pgd_t *user_pgd;
1462
1463 BUG_ON(page->private != 0);
1464
1465 ret = -ENOMEM;
1466
1467 user_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
1468 page->private = (unsigned long)user_pgd;
1469
1470 if (user_pgd != NULL) {
1471 user_pgd[pgd_index(VSYSCALL_START)] =
1472 __pgd(__pa(level3_user_vsyscall) | _PAGE_TABLE);
1473 ret = 0;
1474 }
1475
1476 BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd))));
1477 }
1478#endif
1479
1480 return ret;
1481}
1482
1483static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd)
1484{
1485#ifdef CONFIG_X86_64
1486 pgd_t *user_pgd = xen_get_user_pgd(pgd);
1487
1488 if (user_pgd)
1489 free_page((unsigned long)user_pgd);
1490#endif
1491}
1492
1493#ifdef CONFIG_X86_32
1494static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte)
1495{
1496
1497 if (pte_val_ma(*ptep) & _PAGE_PRESENT)
1498 pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) &
1499 pte_val_ma(pte));
1500
1501 return pte;
1502}
1503#else
1504static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte)
1505{
1506 unsigned long pfn = pte_pfn(pte);
1507
1508
1509
1510
1511
1512
1513
1514 if (((!is_early_ioremap_ptep(ptep) &&
1515 pfn >= pgt_buf_start && pfn < pgt_buf_top)) ||
1516 (is_early_ioremap_ptep(ptep) && pfn != (pgt_buf_end - 1)))
1517 pte = pte_wrprotect(pte);
1518
1519 return pte;
1520}
1521#endif
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537static void __init xen_set_pte_init(pte_t *ptep, pte_t pte)
1538{
1539 if (pte_mfn(pte) != INVALID_P2M_ENTRY)
1540 pte = mask_rw_pte(ptep, pte);
1541 else
1542 pte = __pte_ma(0);
1543
1544 native_set_pte(ptep, pte);
1545}
1546
1547static void pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
1548{
1549 struct mmuext_op op;
1550 op.cmd = cmd;
1551 op.arg1.mfn = pfn_to_mfn(pfn);
1552 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF))
1553 BUG();
1554}
1555
1556
1557
1558static void __init xen_alloc_pte_init(struct mm_struct *mm, unsigned long pfn)
1559{
1560#ifdef CONFIG_FLATMEM
1561 BUG_ON(mem_map);
1562#endif
1563 make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
1564 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
1565}
1566
1567
1568static void __init xen_alloc_pmd_init(struct mm_struct *mm, unsigned long pfn)
1569{
1570#ifdef CONFIG_FLATMEM
1571 BUG_ON(mem_map);
1572#endif
1573 make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
1574}
1575
1576
1577
1578static void __init xen_release_pte_init(unsigned long pfn)
1579{
1580 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
1581 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
1582}
1583
1584static void __init xen_release_pmd_init(unsigned long pfn)
1585{
1586 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
1587}
1588
1589static inline void __pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
1590{
1591 struct multicall_space mcs;
1592 struct mmuext_op *op;
1593
1594 mcs = __xen_mc_entry(sizeof(*op));
1595 op = mcs.args;
1596 op->cmd = cmd;
1597 op->arg1.mfn = pfn_to_mfn(pfn);
1598
1599 MULTI_mmuext_op(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
1600}
1601
1602static inline void __set_pfn_prot(unsigned long pfn, pgprot_t prot)
1603{
1604 struct multicall_space mcs;
1605 unsigned long addr = (unsigned long)__va(pfn << PAGE_SHIFT);
1606
1607 mcs = __xen_mc_entry(0);
1608 MULTI_update_va_mapping(mcs.mc, (unsigned long)addr,
1609 pfn_pte(pfn, prot), 0);
1610}
1611
1612
1613
1614static inline void xen_alloc_ptpage(struct mm_struct *mm, unsigned long pfn,
1615 unsigned level)
1616{
1617 bool pinned = PagePinned(virt_to_page(mm->pgd));
1618
1619 trace_xen_mmu_alloc_ptpage(mm, pfn, level, pinned);
1620
1621 if (pinned) {
1622 struct page *page = pfn_to_page(pfn);
1623
1624 SetPagePinned(page);
1625
1626 if (!PageHighMem(page)) {
1627 xen_mc_batch();
1628
1629 __set_pfn_prot(pfn, PAGE_KERNEL_RO);
1630
1631 if (level == PT_PTE && USE_SPLIT_PTLOCKS)
1632 __pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
1633
1634 xen_mc_issue(PARAVIRT_LAZY_MMU);
1635 } else {
1636
1637
1638 kmap_flush_unused();
1639 }
1640 }
1641}
1642
1643static void xen_alloc_pte(struct mm_struct *mm, unsigned long pfn)
1644{
1645 xen_alloc_ptpage(mm, pfn, PT_PTE);
1646}
1647
1648static void xen_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
1649{
1650 xen_alloc_ptpage(mm, pfn, PT_PMD);
1651}
1652
1653
1654static inline void xen_release_ptpage(unsigned long pfn, unsigned level)
1655{
1656 struct page *page = pfn_to_page(pfn);
1657 bool pinned = PagePinned(page);
1658
1659 trace_xen_mmu_release_ptpage(pfn, level, pinned);
1660
1661 if (pinned) {
1662 if (!PageHighMem(page)) {
1663 xen_mc_batch();
1664
1665 if (level == PT_PTE && USE_SPLIT_PTLOCKS)
1666 __pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
1667
1668 __set_pfn_prot(pfn, PAGE_KERNEL);
1669
1670 xen_mc_issue(PARAVIRT_LAZY_MMU);
1671 }
1672 ClearPagePinned(page);
1673 }
1674}
1675
1676static void xen_release_pte(unsigned long pfn)
1677{
1678 xen_release_ptpage(pfn, PT_PTE);
1679}
1680
1681static void xen_release_pmd(unsigned long pfn)
1682{
1683 xen_release_ptpage(pfn, PT_PMD);
1684}
1685
1686#if PAGETABLE_LEVELS == 4
1687static void xen_alloc_pud(struct mm_struct *mm, unsigned long pfn)
1688{
1689 xen_alloc_ptpage(mm, pfn, PT_PUD);
1690}
1691
1692static void xen_release_pud(unsigned long pfn)
1693{
1694 xen_release_ptpage(pfn, PT_PUD);
1695}
1696#endif
1697
1698void __init xen_reserve_top(void)
1699{
1700#ifdef CONFIG_X86_32
1701 unsigned long top = HYPERVISOR_VIRT_START;
1702 struct xen_platform_parameters pp;
1703
1704 if (HYPERVISOR_xen_version(XENVER_platform_parameters, &pp) == 0)
1705 top = pp.virt_start;
1706
1707 reserve_top_address(-top);
1708#endif
1709}
1710
1711
1712
1713
1714
1715static void *__ka(phys_addr_t paddr)
1716{
1717#ifdef CONFIG_X86_64
1718 return (void *)(paddr + __START_KERNEL_map);
1719#else
1720 return __va(paddr);
1721#endif
1722}
1723
1724
1725static unsigned long m2p(phys_addr_t maddr)
1726{
1727 phys_addr_t paddr;
1728
1729 maddr &= PTE_PFN_MASK;
1730 paddr = mfn_to_pfn(maddr >> PAGE_SHIFT) << PAGE_SHIFT;
1731
1732 return paddr;
1733}
1734
1735
1736static void *m2v(phys_addr_t maddr)
1737{
1738 return __ka(m2p(maddr));
1739}
1740
1741
1742static void set_page_prot(void *addr, pgprot_t prot)
1743{
1744 unsigned long pfn = __pa(addr) >> PAGE_SHIFT;
1745 pte_t pte = pfn_pte(pfn, prot);
1746
1747 if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, 0))
1748 BUG();
1749}
1750#ifdef CONFIG_X86_32
1751static void __init xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn)
1752{
1753 unsigned pmdidx, pteidx;
1754 unsigned ident_pte;
1755 unsigned long pfn;
1756
1757 level1_ident_pgt = extend_brk(sizeof(pte_t) * LEVEL1_IDENT_ENTRIES,
1758 PAGE_SIZE);
1759
1760 ident_pte = 0;
1761 pfn = 0;
1762 for (pmdidx = 0; pmdidx < PTRS_PER_PMD && pfn < max_pfn; pmdidx++) {
1763 pte_t *pte_page;
1764
1765
1766 if (pmd_present(pmd[pmdidx]))
1767 pte_page = m2v(pmd[pmdidx].pmd);
1768 else {
1769
1770 if (ident_pte == LEVEL1_IDENT_ENTRIES)
1771 break;
1772
1773 pte_page = &level1_ident_pgt[ident_pte];
1774 ident_pte += PTRS_PER_PTE;
1775
1776 pmd[pmdidx] = __pmd(__pa(pte_page) | _PAGE_TABLE);
1777 }
1778
1779
1780 for (pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) {
1781 pte_t pte;
1782
1783#ifdef CONFIG_X86_32
1784 if (pfn > max_pfn_mapped)
1785 max_pfn_mapped = pfn;
1786#endif
1787
1788 if (!pte_none(pte_page[pteidx]))
1789 continue;
1790
1791 pte = pfn_pte(pfn, PAGE_KERNEL_EXEC);
1792 pte_page[pteidx] = pte;
1793 }
1794 }
1795
1796 for (pteidx = 0; pteidx < ident_pte; pteidx += PTRS_PER_PTE)
1797 set_page_prot(&level1_ident_pgt[pteidx], PAGE_KERNEL_RO);
1798
1799 set_page_prot(pmd, PAGE_KERNEL_RO);
1800}
1801#endif
1802void __init xen_setup_machphys_mapping(void)
1803{
1804 struct xen_machphys_mapping mapping;
1805
1806 if (HYPERVISOR_memory_op(XENMEM_machphys_mapping, &mapping) == 0) {
1807 machine_to_phys_mapping = (unsigned long *)mapping.v_start;
1808 machine_to_phys_nr = mapping.max_mfn + 1;
1809 } else {
1810 machine_to_phys_nr = MACH2PHYS_NR_ENTRIES;
1811 }
1812#ifdef CONFIG_X86_32
1813 WARN_ON((machine_to_phys_mapping + (machine_to_phys_nr - 1))
1814 < machine_to_phys_mapping);
1815#endif
1816}
1817
1818#ifdef CONFIG_X86_64
1819static void convert_pfn_mfn(void *v)
1820{
1821 pte_t *pte = v;
1822 int i;
1823
1824
1825
1826 for (i = 0; i < PTRS_PER_PTE; i++)
1827 pte[i] = xen_make_pte(pte[i].pte);
1828}
1829static void __init check_pt_base(unsigned long *pt_base, unsigned long *pt_end,
1830 unsigned long addr)
1831{
1832 if (*pt_base == PFN_DOWN(__pa(addr))) {
1833 set_page_prot((void *)addr, PAGE_KERNEL);
1834 clear_page((void *)addr);
1835 (*pt_base)++;
1836 }
1837 if (*pt_end == PFN_DOWN(__pa(addr))) {
1838 set_page_prot((void *)addr, PAGE_KERNEL);
1839 clear_page((void *)addr);
1840 (*pt_end)--;
1841 }
1842}
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
1855{
1856 pud_t *l3;
1857 pmd_t *l2;
1858 unsigned long addr[3];
1859 unsigned long pt_base, pt_end;
1860 unsigned i;
1861
1862
1863
1864
1865
1866 max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->mfn_list));
1867
1868 pt_base = PFN_DOWN(__pa(xen_start_info->pt_base));
1869 pt_end = pt_base + xen_start_info->nr_pt_frames;
1870
1871
1872 init_level4_pgt[0] = __pgd(0);
1873
1874
1875
1876
1877 convert_pfn_mfn(init_level4_pgt);
1878
1879
1880 convert_pfn_mfn(level3_ident_pgt);
1881
1882
1883 convert_pfn_mfn(level3_kernel_pgt);
1884
1885
1886 l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd);
1887 l2 = m2v(l3[pud_index(__START_KERNEL_map)].pud);
1888
1889 addr[0] = (unsigned long)pgd;
1890 addr[1] = (unsigned long)l3;
1891 addr[2] = (unsigned long)l2;
1892
1893
1894
1895
1896
1897
1898 copy_page(level2_ident_pgt, l2);
1899
1900 copy_page(level2_kernel_pgt, l2);
1901
1902
1903 l3 = m2v(pgd[pgd_index(__START_KERNEL_map + PMD_SIZE)].pgd);
1904 l2 = m2v(l3[pud_index(__START_KERNEL_map + PMD_SIZE)].pud);
1905 copy_page(level2_fixmap_pgt, l2);
1906
1907
1908
1909
1910 set_page_prot(init_level4_pgt, PAGE_KERNEL_RO);
1911 set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO);
1912 set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO);
1913 set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO);
1914 set_page_prot(level2_ident_pgt, PAGE_KERNEL_RO);
1915 set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
1916 set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO);
1917
1918
1919 pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE,
1920 PFN_DOWN(__pa_symbol(init_level4_pgt)));
1921
1922
1923 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
1924
1925
1926
1927
1928
1929
1930 xen_mc_batch();
1931 __xen_write_cr3(true, __pa(init_level4_pgt));
1932 xen_mc_issue(PARAVIRT_LAZY_CPU);
1933
1934
1935
1936
1937
1938
1939
1940 for (i = 0; i < ARRAY_SIZE(addr); i++)
1941 check_pt_base(&pt_base, &pt_end, addr[i]);
1942
1943
1944 memblock_reserve(PFN_PHYS(pt_base), (pt_end - pt_base) * PAGE_SIZE);
1945
1946 xen_start_info = (struct start_info *)__va(__pa(xen_start_info));
1947}
1948#else
1949static RESERVE_BRK_ARRAY(pmd_t, initial_kernel_pmd, PTRS_PER_PMD);
1950static RESERVE_BRK_ARRAY(pmd_t, swapper_kernel_pmd, PTRS_PER_PMD);
1951
1952static void __init xen_write_cr3_init(unsigned long cr3)
1953{
1954 unsigned long pfn = PFN_DOWN(__pa(swapper_pg_dir));
1955
1956 BUG_ON(read_cr3() != __pa(initial_page_table));
1957 BUG_ON(cr3 != __pa(swapper_pg_dir));
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969 swapper_kernel_pmd =
1970 extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE);
1971 copy_page(swapper_kernel_pmd, initial_kernel_pmd);
1972 swapper_pg_dir[KERNEL_PGD_BOUNDARY] =
1973 __pgd(__pa(swapper_kernel_pmd) | _PAGE_PRESENT);
1974 set_page_prot(swapper_kernel_pmd, PAGE_KERNEL_RO);
1975
1976 set_page_prot(swapper_pg_dir, PAGE_KERNEL_RO);
1977 xen_write_cr3(cr3);
1978 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, pfn);
1979
1980 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE,
1981 PFN_DOWN(__pa(initial_page_table)));
1982 set_page_prot(initial_page_table, PAGE_KERNEL);
1983 set_page_prot(initial_kernel_pmd, PAGE_KERNEL);
1984
1985 pv_mmu_ops.write_cr3 = &xen_write_cr3;
1986}
1987
1988void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn)
1989{
1990 pmd_t *kernel_pmd;
1991
1992 initial_kernel_pmd =
1993 extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE);
1994
1995 max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->pt_base) +
1996 xen_start_info->nr_pt_frames * PAGE_SIZE +
1997 512*1024);
1998
1999 kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd);
2000 copy_page(initial_kernel_pmd, kernel_pmd);
2001
2002 xen_map_identity_early(initial_kernel_pmd, max_pfn);
2003
2004 copy_page(initial_page_table, pgd);
2005 initial_page_table[KERNEL_PGD_BOUNDARY] =
2006 __pgd(__pa(initial_kernel_pmd) | _PAGE_PRESENT);
2007
2008 set_page_prot(initial_kernel_pmd, PAGE_KERNEL_RO);
2009 set_page_prot(initial_page_table, PAGE_KERNEL_RO);
2010 set_page_prot(empty_zero_page, PAGE_KERNEL_RO);
2011
2012 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
2013
2014 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE,
2015 PFN_DOWN(__pa(initial_page_table)));
2016 xen_write_cr3(__pa(initial_page_table));
2017
2018 memblock_reserve(__pa(xen_start_info->pt_base),
2019 xen_start_info->nr_pt_frames * PAGE_SIZE);
2020}
2021#endif
2022
2023static unsigned char dummy_mapping[PAGE_SIZE] __page_aligned_bss;
2024
2025static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot)
2026{
2027 pte_t pte;
2028
2029 phys >>= PAGE_SHIFT;
2030
2031 switch (idx) {
2032 case FIX_BTMAP_END ... FIX_BTMAP_BEGIN:
2033#ifdef CONFIG_X86_F00F_BUG
2034 case FIX_F00F_IDT:
2035#endif
2036#ifdef CONFIG_X86_32
2037 case FIX_WP_TEST:
2038 case FIX_VDSO:
2039# ifdef CONFIG_HIGHMEM
2040 case FIX_KMAP_BEGIN ... FIX_KMAP_END:
2041# endif
2042#else
2043 case VSYSCALL_LAST_PAGE ... VSYSCALL_FIRST_PAGE:
2044 case VVAR_PAGE:
2045#endif
2046 case FIX_TEXT_POKE0:
2047 case FIX_TEXT_POKE1:
2048
2049 pte = pfn_pte(phys, prot);
2050 break;
2051
2052#ifdef CONFIG_X86_LOCAL_APIC
2053 case FIX_APIC_BASE:
2054 pte = pfn_pte(PFN_DOWN(__pa(dummy_mapping)), PAGE_KERNEL);
2055 break;
2056#endif
2057
2058#ifdef CONFIG_X86_IO_APIC
2059 case FIX_IO_APIC_BASE_0 ... FIX_IO_APIC_BASE_END:
2060
2061
2062
2063
2064 pte = pfn_pte(PFN_DOWN(__pa(dummy_mapping)), PAGE_KERNEL);
2065 break;
2066#endif
2067
2068 case FIX_PARAVIRT_BOOTMAP:
2069
2070
2071 pte = mfn_pte(phys, prot);
2072 break;
2073
2074 default:
2075
2076 pte = mfn_pte(phys, __pgprot(pgprot_val(prot) | _PAGE_IOMAP));
2077 break;
2078 }
2079
2080 __native_set_fixmap(idx, pte);
2081
2082#ifdef CONFIG_X86_64
2083
2084
2085 if ((idx >= VSYSCALL_LAST_PAGE && idx <= VSYSCALL_FIRST_PAGE) ||
2086 idx == VVAR_PAGE) {
2087 unsigned long vaddr = __fix_to_virt(idx);
2088 set_pte_vaddr_pud(level3_user_vsyscall, vaddr, pte);
2089 }
2090#endif
2091}
2092
2093static void __init xen_post_allocator_init(void)
2094{
2095 pv_mmu_ops.set_pte = xen_set_pte;
2096 pv_mmu_ops.set_pmd = xen_set_pmd;
2097 pv_mmu_ops.set_pud = xen_set_pud;
2098#if PAGETABLE_LEVELS == 4
2099 pv_mmu_ops.set_pgd = xen_set_pgd;
2100#endif
2101
2102
2103
2104 pv_mmu_ops.alloc_pte = xen_alloc_pte;
2105 pv_mmu_ops.alloc_pmd = xen_alloc_pmd;
2106 pv_mmu_ops.release_pte = xen_release_pte;
2107 pv_mmu_ops.release_pmd = xen_release_pmd;
2108#if PAGETABLE_LEVELS == 4
2109 pv_mmu_ops.alloc_pud = xen_alloc_pud;
2110 pv_mmu_ops.release_pud = xen_release_pud;
2111#endif
2112
2113#ifdef CONFIG_X86_64
2114 SetPagePinned(virt_to_page(level3_user_vsyscall));
2115#endif
2116 xen_mark_init_mm_pinned();
2117}
2118
2119static void xen_leave_lazy_mmu(void)
2120{
2121 preempt_disable();
2122 xen_mc_flush();
2123 paravirt_leave_lazy_mmu();
2124 preempt_enable();
2125}
2126
2127static const struct pv_mmu_ops xen_mmu_ops __initconst = {
2128 .read_cr2 = xen_read_cr2,
2129 .write_cr2 = xen_write_cr2,
2130
2131 .read_cr3 = xen_read_cr3,
2132#ifdef CONFIG_X86_32
2133 .write_cr3 = xen_write_cr3_init,
2134#else
2135 .write_cr3 = xen_write_cr3,
2136#endif
2137
2138 .flush_tlb_user = xen_flush_tlb,
2139 .flush_tlb_kernel = xen_flush_tlb,
2140 .flush_tlb_single = xen_flush_tlb_single,
2141 .flush_tlb_others = xen_flush_tlb_others,
2142
2143 .pte_update = paravirt_nop,
2144 .pte_update_defer = paravirt_nop,
2145
2146 .pgd_alloc = xen_pgd_alloc,
2147 .pgd_free = xen_pgd_free,
2148
2149 .alloc_pte = xen_alloc_pte_init,
2150 .release_pte = xen_release_pte_init,
2151 .alloc_pmd = xen_alloc_pmd_init,
2152 .release_pmd = xen_release_pmd_init,
2153
2154 .set_pte = xen_set_pte_init,
2155 .set_pte_at = xen_set_pte_at,
2156 .set_pmd = xen_set_pmd_hyper,
2157
2158 .ptep_modify_prot_start = __ptep_modify_prot_start,
2159 .ptep_modify_prot_commit = __ptep_modify_prot_commit,
2160
2161 .pte_val = PV_CALLEE_SAVE(xen_pte_val),
2162 .pgd_val = PV_CALLEE_SAVE(xen_pgd_val),
2163
2164 .make_pte = PV_CALLEE_SAVE(xen_make_pte),
2165 .make_pgd = PV_CALLEE_SAVE(xen_make_pgd),
2166
2167#ifdef CONFIG_X86_PAE
2168 .set_pte_atomic = xen_set_pte_atomic,
2169 .pte_clear = xen_pte_clear,
2170 .pmd_clear = xen_pmd_clear,
2171#endif
2172 .set_pud = xen_set_pud_hyper,
2173
2174 .make_pmd = PV_CALLEE_SAVE(xen_make_pmd),
2175 .pmd_val = PV_CALLEE_SAVE(xen_pmd_val),
2176
2177#if PAGETABLE_LEVELS == 4
2178 .pud_val = PV_CALLEE_SAVE(xen_pud_val),
2179 .make_pud = PV_CALLEE_SAVE(xen_make_pud),
2180 .set_pgd = xen_set_pgd_hyper,
2181
2182 .alloc_pud = xen_alloc_pmd_init,
2183 .release_pud = xen_release_pmd_init,
2184#endif
2185
2186 .activate_mm = xen_activate_mm,
2187 .dup_mmap = xen_dup_mmap,
2188 .exit_mmap = xen_exit_mmap,
2189
2190 .lazy_mode = {
2191 .enter = paravirt_enter_lazy_mmu,
2192 .leave = xen_leave_lazy_mmu,
2193 },
2194
2195 .set_fixmap = xen_set_fixmap,
2196};
2197
2198void __init xen_init_mmu_ops(void)
2199{
2200 x86_init.mapping.pagetable_reserve = xen_mapping_pagetable_reserve;
2201 x86_init.paging.pagetable_init = xen_pagetable_init;
2202 pv_mmu_ops = xen_mmu_ops;
2203
2204 memset(dummy_mapping, 0xff, PAGE_SIZE);
2205}
2206
2207
2208#define MAX_CONTIG_ORDER 9
2209static unsigned long discontig_frames[1<<MAX_CONTIG_ORDER];
2210
2211#define VOID_PTE (mfn_pte(0, __pgprot(0)))
2212static void xen_zap_pfn_range(unsigned long vaddr, unsigned int order,
2213 unsigned long *in_frames,
2214 unsigned long *out_frames)
2215{
2216 int i;
2217 struct multicall_space mcs;
2218
2219 xen_mc_batch();
2220 for (i = 0; i < (1UL<<order); i++, vaddr += PAGE_SIZE) {
2221 mcs = __xen_mc_entry(0);
2222
2223 if (in_frames)
2224 in_frames[i] = virt_to_mfn(vaddr);
2225
2226 MULTI_update_va_mapping(mcs.mc, vaddr, VOID_PTE, 0);
2227 __set_phys_to_machine(virt_to_pfn(vaddr), INVALID_P2M_ENTRY);
2228
2229 if (out_frames)
2230 out_frames[i] = virt_to_pfn(vaddr);
2231 }
2232 xen_mc_issue(0);
2233}
2234
2235
2236
2237
2238
2239
2240static void xen_remap_exchanged_ptes(unsigned long vaddr, int order,
2241 unsigned long *mfns,
2242 unsigned long first_mfn)
2243{
2244 unsigned i, limit;
2245 unsigned long mfn;
2246
2247 xen_mc_batch();
2248
2249 limit = 1u << order;
2250 for (i = 0; i < limit; i++, vaddr += PAGE_SIZE) {
2251 struct multicall_space mcs;
2252 unsigned flags;
2253
2254 mcs = __xen_mc_entry(0);
2255 if (mfns)
2256 mfn = mfns[i];
2257 else
2258 mfn = first_mfn + i;
2259
2260 if (i < (limit - 1))
2261 flags = 0;
2262 else {
2263 if (order == 0)
2264 flags = UVMF_INVLPG | UVMF_ALL;
2265 else
2266 flags = UVMF_TLB_FLUSH | UVMF_ALL;
2267 }
2268
2269 MULTI_update_va_mapping(mcs.mc, vaddr,
2270 mfn_pte(mfn, PAGE_KERNEL), flags);
2271
2272 set_phys_to_machine(virt_to_pfn(vaddr), mfn);
2273 }
2274
2275 xen_mc_issue(0);
2276}
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286static int xen_exchange_memory(unsigned long extents_in, unsigned int order_in,
2287 unsigned long *pfns_in,
2288 unsigned long extents_out,
2289 unsigned int order_out,
2290 unsigned long *mfns_out,
2291 unsigned int address_bits)
2292{
2293 long rc;
2294 int success;
2295
2296 struct xen_memory_exchange exchange = {
2297 .in = {
2298 .nr_extents = extents_in,
2299 .extent_order = order_in,
2300 .extent_start = pfns_in,
2301 .domid = DOMID_SELF
2302 },
2303 .out = {
2304 .nr_extents = extents_out,
2305 .extent_order = order_out,
2306 .extent_start = mfns_out,
2307 .address_bits = address_bits,
2308 .domid = DOMID_SELF
2309 }
2310 };
2311
2312 BUG_ON(extents_in << order_in != extents_out << order_out);
2313
2314 rc = HYPERVISOR_memory_op(XENMEM_exchange, &exchange);
2315 success = (exchange.nr_exchanged == extents_in);
2316
2317 BUG_ON(!success && ((exchange.nr_exchanged != 0) || (rc == 0)));
2318 BUG_ON(success && (rc != 0));
2319
2320 return success;
2321}
2322
2323int xen_create_contiguous_region(unsigned long vstart, unsigned int order,
2324 unsigned int address_bits)
2325{
2326 unsigned long *in_frames = discontig_frames, out_frame;
2327 unsigned long flags;
2328 int success;
2329
2330
2331
2332
2333
2334
2335
2336 if (xen_feature(XENFEAT_auto_translated_physmap))
2337 return 0;
2338
2339 if (unlikely(order > MAX_CONTIG_ORDER))
2340 return -ENOMEM;
2341
2342 memset((void *) vstart, 0, PAGE_SIZE << order);
2343
2344 spin_lock_irqsave(&xen_reservation_lock, flags);
2345
2346
2347 xen_zap_pfn_range(vstart, order, in_frames, NULL);
2348
2349
2350 out_frame = virt_to_pfn(vstart);
2351 success = xen_exchange_memory(1UL << order, 0, in_frames,
2352 1, order, &out_frame,
2353 address_bits);
2354
2355
2356 if (success)
2357 xen_remap_exchanged_ptes(vstart, order, NULL, out_frame);
2358 else
2359 xen_remap_exchanged_ptes(vstart, order, in_frames, 0);
2360
2361 spin_unlock_irqrestore(&xen_reservation_lock, flags);
2362
2363 return success ? 0 : -ENOMEM;
2364}
2365EXPORT_SYMBOL_GPL(xen_create_contiguous_region);
2366
2367void xen_destroy_contiguous_region(unsigned long vstart, unsigned int order)
2368{
2369 unsigned long *out_frames = discontig_frames, in_frame;
2370 unsigned long flags;
2371 int success;
2372
2373 if (xen_feature(XENFEAT_auto_translated_physmap))
2374 return;
2375
2376 if (unlikely(order > MAX_CONTIG_ORDER))
2377 return;
2378
2379 memset((void *) vstart, 0, PAGE_SIZE << order);
2380
2381 spin_lock_irqsave(&xen_reservation_lock, flags);
2382
2383
2384 in_frame = virt_to_mfn(vstart);
2385
2386
2387 xen_zap_pfn_range(vstart, order, NULL, out_frames);
2388
2389
2390 success = xen_exchange_memory(1, order, &in_frame, 1UL << order,
2391 0, out_frames, 0);
2392
2393
2394 if (success)
2395 xen_remap_exchanged_ptes(vstart, order, out_frames, 0);
2396 else
2397 xen_remap_exchanged_ptes(vstart, order, NULL, in_frame);
2398
2399 spin_unlock_irqrestore(&xen_reservation_lock, flags);
2400}
2401EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region);
2402
2403#ifdef CONFIG_XEN_PVHVM
2404#ifdef CONFIG_PROC_VMCORE
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415static int xen_oldmem_pfn_is_ram(unsigned long pfn)
2416{
2417 struct xen_hvm_get_mem_type a = {
2418 .domid = DOMID_SELF,
2419 .pfn = pfn,
2420 };
2421 int ram;
2422
2423 if (HYPERVISOR_hvm_op(HVMOP_get_mem_type, &a))
2424 return -ENXIO;
2425
2426 switch (a.mem_type) {
2427 case HVMMEM_mmio_dm:
2428 ram = 0;
2429 break;
2430 case HVMMEM_ram_rw:
2431 case HVMMEM_ram_ro:
2432 default:
2433 ram = 1;
2434 break;
2435 }
2436
2437 return ram;
2438}
2439#endif
2440
2441static void xen_hvm_exit_mmap(struct mm_struct *mm)
2442{
2443 struct xen_hvm_pagetable_dying a;
2444 int rc;
2445
2446 a.domid = DOMID_SELF;
2447 a.gpa = __pa(mm->pgd);
2448 rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a);
2449 WARN_ON_ONCE(rc < 0);
2450}
2451
2452static int is_pagetable_dying_supported(void)
2453{
2454 struct xen_hvm_pagetable_dying a;
2455 int rc = 0;
2456
2457 a.domid = DOMID_SELF;
2458 a.gpa = 0x00;
2459 rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a);
2460 if (rc < 0) {
2461 printk(KERN_DEBUG "HVMOP_pagetable_dying not supported\n");
2462 return 0;
2463 }
2464 return 1;
2465}
2466
2467void __init xen_hvm_init_mmu_ops(void)
2468{
2469 if (is_pagetable_dying_supported())
2470 pv_mmu_ops.exit_mmap = xen_hvm_exit_mmap;
2471#ifdef CONFIG_PROC_VMCORE
2472 register_oldmem_pfn_is_ram(&xen_oldmem_pfn_is_ram);
2473#endif
2474}
2475#endif
2476
2477#define REMAP_BATCH_SIZE 16
2478
2479struct remap_data {
2480 unsigned long mfn;
2481 pgprot_t prot;
2482 struct mmu_update *mmu_update;
2483};
2484
2485static int remap_area_mfn_pte_fn(pte_t *ptep, pgtable_t token,
2486 unsigned long addr, void *data)
2487{
2488 struct remap_data *rmd = data;
2489 pte_t pte = pte_mkspecial(pfn_pte(rmd->mfn++, rmd->prot));
2490
2491 rmd->mmu_update->ptr = virt_to_machine(ptep).maddr;
2492 rmd->mmu_update->val = pte_val_ma(pte);
2493 rmd->mmu_update++;
2494
2495 return 0;
2496}
2497
2498int xen_remap_domain_mfn_range(struct vm_area_struct *vma,
2499 unsigned long addr,
2500 xen_pfn_t mfn, int nr,
2501 pgprot_t prot, unsigned domid,
2502 struct page **pages)
2503
2504{
2505 struct remap_data rmd;
2506 struct mmu_update mmu_update[REMAP_BATCH_SIZE];
2507 int batch;
2508 unsigned long range;
2509 int err = 0;
2510
2511 if (xen_feature(XENFEAT_auto_translated_physmap))
2512 return -EINVAL;
2513
2514 prot = __pgprot(pgprot_val(prot) | _PAGE_IOMAP);
2515
2516 BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO)));
2517
2518 rmd.mfn = mfn;
2519 rmd.prot = prot;
2520
2521 while (nr) {
2522 batch = min(REMAP_BATCH_SIZE, nr);
2523 range = (unsigned long)batch << PAGE_SHIFT;
2524
2525 rmd.mmu_update = mmu_update;
2526 err = apply_to_page_range(vma->vm_mm, addr, range,
2527 remap_area_mfn_pte_fn, &rmd);
2528 if (err)
2529 goto out;
2530
2531 err = HYPERVISOR_mmu_update(mmu_update, batch, NULL, domid);
2532 if (err < 0)
2533 goto out;
2534
2535 nr -= batch;
2536 addr += range;
2537 }
2538
2539 err = 0;
2540out:
2541
2542 xen_flush_tlb_all();
2543
2544 return err;
2545}
2546EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_range);
2547
2548
2549int xen_unmap_domain_mfn_range(struct vm_area_struct *vma,
2550 int numpgs, struct page **pages)
2551{
2552 if (!pages || !xen_feature(XENFEAT_auto_translated_physmap))
2553 return 0;
2554
2555 return -EINVAL;
2556}
2557EXPORT_SYMBOL_GPL(xen_unmap_domain_mfn_range);
2558