1
2
3
4
5
6
7
8
9
10
11
12#include <linux/highmem.h>
13#include <linux/kvm_host.h>
14#include <linux/uaccess.h>
15#include <asm/mmu_context.h>
16#include <asm/pgalloc.h>
17
18
19
20
21
22#if defined(__PAGETABLE_PMD_FOLDED)
23#define KVM_MMU_CACHE_MIN_PAGES 1
24#else
25#define KVM_MMU_CACHE_MIN_PAGES 2
26#endif
27
28static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
29 int min, int max)
30{
31 void *page;
32
33 BUG_ON(max > KVM_NR_MEM_OBJS);
34 if (cache->nobjs >= min)
35 return 0;
36 while (cache->nobjs < max) {
37 page = (void *)__get_free_page(GFP_KERNEL);
38 if (!page)
39 return -ENOMEM;
40 cache->objects[cache->nobjs++] = page;
41 }
42 return 0;
43}
44
45static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
46{
47 while (mc->nobjs)
48 free_page((unsigned long)mc->objects[--mc->nobjs]);
49}
50
51static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
52{
53 void *p;
54
55 BUG_ON(!mc || !mc->nobjs);
56 p = mc->objects[--mc->nobjs];
57 return p;
58}
59
60void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu)
61{
62 mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
63}
64
65
66
67
68
69
70
71
72
73
74
75static void kvm_pgd_init(void *page)
76{
77 unsigned long *p, *end;
78 unsigned long entry;
79
80#ifdef __PAGETABLE_PMD_FOLDED
81 entry = (unsigned long)invalid_pte_table;
82#else
83 entry = (unsigned long)invalid_pmd_table;
84#endif
85
86 p = (unsigned long *)page;
87 end = p + PTRS_PER_PGD;
88
89 do {
90 p[0] = entry;
91 p[1] = entry;
92 p[2] = entry;
93 p[3] = entry;
94 p[4] = entry;
95 p += 8;
96 p[-3] = entry;
97 p[-2] = entry;
98 p[-1] = entry;
99 } while (p != end);
100}
101
102
103
104
105
106
107
108
109
110
111pgd_t *kvm_pgd_alloc(void)
112{
113 pgd_t *ret;
114
115 ret = (pgd_t *)__get_free_pages(GFP_KERNEL, PGD_ORDER);
116 if (ret)
117 kvm_pgd_init(ret);
118
119 return ret;
120}
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136static pte_t *kvm_mips_walk_pgd(pgd_t *pgd, struct kvm_mmu_memory_cache *cache,
137 unsigned long addr)
138{
139 pud_t *pud;
140 pmd_t *pmd;
141
142 pgd += pgd_index(addr);
143 if (pgd_none(*pgd)) {
144
145 BUG();
146 return NULL;
147 }
148 pud = pud_offset(pgd, addr);
149 if (pud_none(*pud)) {
150 pmd_t *new_pmd;
151
152 if (!cache)
153 return NULL;
154 new_pmd = mmu_memory_cache_alloc(cache);
155 pmd_init((unsigned long)new_pmd,
156 (unsigned long)invalid_pte_table);
157 pud_populate(NULL, pud, new_pmd);
158 }
159 pmd = pmd_offset(pud, addr);
160 if (pmd_none(*pmd)) {
161 pte_t *new_pte;
162
163 if (!cache)
164 return NULL;
165 new_pte = mmu_memory_cache_alloc(cache);
166 clear_page(new_pte);
167 pmd_populate_kernel(NULL, pmd, new_pte);
168 }
169 return pte_offset(pmd, addr);
170}
171
172
173static pte_t *kvm_mips_pte_for_gpa(struct kvm *kvm,
174 struct kvm_mmu_memory_cache *cache,
175 unsigned long addr)
176{
177 return kvm_mips_walk_pgd(kvm->arch.gpa_mm.pgd, cache, addr);
178}
179
180
181
182
183
184
185static bool kvm_mips_flush_gpa_pte(pte_t *pte, unsigned long start_gpa,
186 unsigned long end_gpa)
187{
188 int i_min = __pte_offset(start_gpa);
189 int i_max = __pte_offset(end_gpa);
190 bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PTE - 1);
191 int i;
192
193 for (i = i_min; i <= i_max; ++i) {
194 if (!pte_present(pte[i]))
195 continue;
196
197 set_pte(pte + i, __pte(0));
198 }
199 return safe_to_remove;
200}
201
202static bool kvm_mips_flush_gpa_pmd(pmd_t *pmd, unsigned long start_gpa,
203 unsigned long end_gpa)
204{
205 pte_t *pte;
206 unsigned long end = ~0ul;
207 int i_min = __pmd_offset(start_gpa);
208 int i_max = __pmd_offset(end_gpa);
209 bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PMD - 1);
210 int i;
211
212 for (i = i_min; i <= i_max; ++i, start_gpa = 0) {
213 if (!pmd_present(pmd[i]))
214 continue;
215
216 pte = pte_offset(pmd + i, 0);
217 if (i == i_max)
218 end = end_gpa;
219
220 if (kvm_mips_flush_gpa_pte(pte, start_gpa, end)) {
221 pmd_clear(pmd + i);
222 pte_free_kernel(NULL, pte);
223 } else {
224 safe_to_remove = false;
225 }
226 }
227 return safe_to_remove;
228}
229
230static bool kvm_mips_flush_gpa_pud(pud_t *pud, unsigned long start_gpa,
231 unsigned long end_gpa)
232{
233 pmd_t *pmd;
234 unsigned long end = ~0ul;
235 int i_min = __pud_offset(start_gpa);
236 int i_max = __pud_offset(end_gpa);
237 bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PUD - 1);
238 int i;
239
240 for (i = i_min; i <= i_max; ++i, start_gpa = 0) {
241 if (!pud_present(pud[i]))
242 continue;
243
244 pmd = pmd_offset(pud + i, 0);
245 if (i == i_max)
246 end = end_gpa;
247
248 if (kvm_mips_flush_gpa_pmd(pmd, start_gpa, end)) {
249 pud_clear(pud + i);
250 pmd_free(NULL, pmd);
251 } else {
252 safe_to_remove = false;
253 }
254 }
255 return safe_to_remove;
256}
257
258static bool kvm_mips_flush_gpa_pgd(pgd_t *pgd, unsigned long start_gpa,
259 unsigned long end_gpa)
260{
261 pud_t *pud;
262 unsigned long end = ~0ul;
263 int i_min = pgd_index(start_gpa);
264 int i_max = pgd_index(end_gpa);
265 bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PGD - 1);
266 int i;
267
268 for (i = i_min; i <= i_max; ++i, start_gpa = 0) {
269 if (!pgd_present(pgd[i]))
270 continue;
271
272 pud = pud_offset(pgd + i, 0);
273 if (i == i_max)
274 end = end_gpa;
275
276 if (kvm_mips_flush_gpa_pud(pud, start_gpa, end)) {
277 pgd_clear(pgd + i);
278 pud_free(NULL, pud);
279 } else {
280 safe_to_remove = false;
281 }
282 }
283 return safe_to_remove;
284}
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299bool kvm_mips_flush_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn)
300{
301 return kvm_mips_flush_gpa_pgd(kvm->arch.gpa_mm.pgd,
302 start_gfn << PAGE_SHIFT,
303 end_gfn << PAGE_SHIFT);
304}
305
306#define BUILD_PTE_RANGE_OP(name, op) \
307static int kvm_mips_##name##_pte(pte_t *pte, unsigned long start, \
308 unsigned long end) \
309{ \
310 int ret = 0; \
311 int i_min = __pte_offset(start); \
312 int i_max = __pte_offset(end); \
313 int i; \
314 pte_t old, new; \
315 \
316 for (i = i_min; i <= i_max; ++i) { \
317 if (!pte_present(pte[i])) \
318 continue; \
319 \
320 old = pte[i]; \
321 new = op(old); \
322 if (pte_val(new) == pte_val(old)) \
323 continue; \
324 set_pte(pte + i, new); \
325 ret = 1; \
326 } \
327 return ret; \
328} \
329 \
330 \
331static int kvm_mips_##name##_pmd(pmd_t *pmd, unsigned long start, \
332 unsigned long end) \
333{ \
334 int ret = 0; \
335 pte_t *pte; \
336 unsigned long cur_end = ~0ul; \
337 int i_min = __pmd_offset(start); \
338 int i_max = __pmd_offset(end); \
339 int i; \
340 \
341 for (i = i_min; i <= i_max; ++i, start = 0) { \
342 if (!pmd_present(pmd[i])) \
343 continue; \
344 \
345 pte = pte_offset(pmd + i, 0); \
346 if (i == i_max) \
347 cur_end = end; \
348 \
349 ret |= kvm_mips_##name##_pte(pte, start, cur_end); \
350 } \
351 return ret; \
352} \
353 \
354static int kvm_mips_##name##_pud(pud_t *pud, unsigned long start, \
355 unsigned long end) \
356{ \
357 int ret = 0; \
358 pmd_t *pmd; \
359 unsigned long cur_end = ~0ul; \
360 int i_min = __pud_offset(start); \
361 int i_max = __pud_offset(end); \
362 int i; \
363 \
364 for (i = i_min; i <= i_max; ++i, start = 0) { \
365 if (!pud_present(pud[i])) \
366 continue; \
367 \
368 pmd = pmd_offset(pud + i, 0); \
369 if (i == i_max) \
370 cur_end = end; \
371 \
372 ret |= kvm_mips_##name##_pmd(pmd, start, cur_end); \
373 } \
374 return ret; \
375} \
376 \
377static int kvm_mips_##name##_pgd(pgd_t *pgd, unsigned long start, \
378 unsigned long end) \
379{ \
380 int ret = 0; \
381 pud_t *pud; \
382 unsigned long cur_end = ~0ul; \
383 int i_min = pgd_index(start); \
384 int i_max = pgd_index(end); \
385 int i; \
386 \
387 for (i = i_min; i <= i_max; ++i, start = 0) { \
388 if (!pgd_present(pgd[i])) \
389 continue; \
390 \
391 pud = pud_offset(pgd + i, 0); \
392 if (i == i_max) \
393 cur_end = end; \
394 \
395 ret |= kvm_mips_##name##_pud(pud, start, cur_end); \
396 } \
397 return ret; \
398}
399
400
401
402
403
404
405
406BUILD_PTE_RANGE_OP(mkclean, pte_mkclean)
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423int kvm_mips_mkclean_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn)
424{
425 return kvm_mips_mkclean_pgd(kvm->arch.gpa_mm.pgd,
426 start_gfn << PAGE_SHIFT,
427 end_gfn << PAGE_SHIFT);
428}
429
430
431
432
433
434
435
436
437
438
439
440
441void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
442 struct kvm_memory_slot *slot,
443 gfn_t gfn_offset, unsigned long mask)
444{
445 gfn_t base_gfn = slot->base_gfn + gfn_offset;
446 gfn_t start = base_gfn + __ffs(mask);
447 gfn_t end = base_gfn + __fls(mask);
448
449 kvm_mips_mkclean_gpa_pt(kvm, start, end);
450}
451
452
453
454
455
456
457
458BUILD_PTE_RANGE_OP(mkold, pte_mkold)
459
460static int kvm_mips_mkold_gpa_pt(struct kvm *kvm, gfn_t start_gfn,
461 gfn_t end_gfn)
462{
463 return kvm_mips_mkold_pgd(kvm->arch.gpa_mm.pgd,
464 start_gfn << PAGE_SHIFT,
465 end_gfn << PAGE_SHIFT);
466}
467
468static int handle_hva_to_gpa(struct kvm *kvm,
469 unsigned long start,
470 unsigned long end,
471 int (*handler)(struct kvm *kvm, gfn_t gfn,
472 gpa_t gfn_end,
473 struct kvm_memory_slot *memslot,
474 void *data),
475 void *data)
476{
477 struct kvm_memslots *slots;
478 struct kvm_memory_slot *memslot;
479 int ret = 0;
480
481 slots = kvm_memslots(kvm);
482
483
484 kvm_for_each_memslot(memslot, slots) {
485 unsigned long hva_start, hva_end;
486 gfn_t gfn, gfn_end;
487
488 hva_start = max(start, memslot->userspace_addr);
489 hva_end = min(end, memslot->userspace_addr +
490 (memslot->npages << PAGE_SHIFT));
491 if (hva_start >= hva_end)
492 continue;
493
494
495
496
497
498 gfn = hva_to_gfn_memslot(hva_start, memslot);
499 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
500
501 ret |= handler(kvm, gfn, gfn_end, memslot, data);
502 }
503
504 return ret;
505}
506
507
508static int kvm_unmap_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end,
509 struct kvm_memory_slot *memslot, void *data)
510{
511 kvm_mips_flush_gpa_pt(kvm, gfn, gfn_end);
512 return 1;
513}
514
515int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
516{
517 handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL);
518
519 kvm_mips_callbacks->flush_shadow_all(kvm);
520 return 0;
521}
522
523static int kvm_set_spte_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end,
524 struct kvm_memory_slot *memslot, void *data)
525{
526 gpa_t gpa = gfn << PAGE_SHIFT;
527 pte_t hva_pte = *(pte_t *)data;
528 pte_t *gpa_pte = kvm_mips_pte_for_gpa(kvm, NULL, gpa);
529 pte_t old_pte;
530
531 if (!gpa_pte)
532 return 0;
533
534
535 old_pte = *gpa_pte;
536 if (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES && !pte_dirty(old_pte))
537 hva_pte = pte_mkclean(hva_pte);
538 else if (memslot->flags & KVM_MEM_READONLY)
539 hva_pte = pte_wrprotect(hva_pte);
540
541 set_pte(gpa_pte, hva_pte);
542
543
544 if (!pte_present(old_pte) || !pte_young(old_pte))
545 return 0;
546
547
548 return !pte_present(hva_pte) ||
549 !pte_young(hva_pte) ||
550 pte_pfn(old_pte) != pte_pfn(hva_pte) ||
551 (pte_dirty(old_pte) && !pte_dirty(hva_pte));
552}
553
554void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
555{
556 unsigned long end = hva + PAGE_SIZE;
557 int ret;
558
559 ret = handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &pte);
560 if (ret)
561 kvm_mips_callbacks->flush_shadow_all(kvm);
562}
563
564static int kvm_age_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end,
565 struct kvm_memory_slot *memslot, void *data)
566{
567 return kvm_mips_mkold_gpa_pt(kvm, gfn, gfn_end);
568}
569
570static int kvm_test_age_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end,
571 struct kvm_memory_slot *memslot, void *data)
572{
573 gpa_t gpa = gfn << PAGE_SHIFT;
574 pte_t *gpa_pte = kvm_mips_pte_for_gpa(kvm, NULL, gpa);
575
576 if (!gpa_pte)
577 return 0;
578 return pte_young(*gpa_pte);
579}
580
581int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
582{
583 return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL);
584}
585
586int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
587{
588 return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL);
589}
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609static int _kvm_mips_map_page_fast(struct kvm_vcpu *vcpu, unsigned long gpa,
610 bool write_fault,
611 pte_t *out_entry, pte_t *out_buddy)
612{
613 struct kvm *kvm = vcpu->kvm;
614 gfn_t gfn = gpa >> PAGE_SHIFT;
615 pte_t *ptep;
616 kvm_pfn_t pfn = 0;
617 bool pfn_valid = false;
618 int ret = 0;
619
620 spin_lock(&kvm->mmu_lock);
621
622
623 ptep = kvm_mips_pte_for_gpa(kvm, NULL, gpa);
624 if (!ptep || !pte_present(*ptep)) {
625 ret = -EFAULT;
626 goto out;
627 }
628
629
630 if (!pte_young(*ptep)) {
631 set_pte(ptep, pte_mkyoung(*ptep));
632 pfn = pte_pfn(*ptep);
633 pfn_valid = true;
634
635 }
636 if (write_fault && !pte_dirty(*ptep)) {
637 if (!pte_write(*ptep)) {
638 ret = -EFAULT;
639 goto out;
640 }
641
642
643 set_pte(ptep, pte_mkdirty(*ptep));
644 pfn = pte_pfn(*ptep);
645 mark_page_dirty(kvm, gfn);
646 kvm_set_pfn_dirty(pfn);
647 }
648
649 if (out_entry)
650 *out_entry = *ptep;
651 if (out_buddy)
652 *out_buddy = *ptep_buddy(ptep);
653
654out:
655 spin_unlock(&kvm->mmu_lock);
656 if (pfn_valid)
657 kvm_set_pfn_accessed(pfn);
658 return ret;
659}
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685static int kvm_mips_map_page(struct kvm_vcpu *vcpu, unsigned long gpa,
686 bool write_fault,
687 pte_t *out_entry, pte_t *out_buddy)
688{
689 struct kvm *kvm = vcpu->kvm;
690 struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
691 gfn_t gfn = gpa >> PAGE_SHIFT;
692 int srcu_idx, err;
693 kvm_pfn_t pfn;
694 pte_t *ptep, entry, old_pte;
695 bool writeable;
696 unsigned long prot_bits;
697 unsigned long mmu_seq;
698
699
700 srcu_idx = srcu_read_lock(&kvm->srcu);
701 err = _kvm_mips_map_page_fast(vcpu, gpa, write_fault, out_entry,
702 out_buddy);
703 if (!err)
704 goto out;
705
706
707 err = mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES,
708 KVM_NR_MEM_OBJS);
709 if (err)
710 goto out;
711
712retry:
713
714
715
716
717 mmu_seq = kvm->mmu_notifier_seq;
718
719
720
721
722
723
724
725
726
727
728
729 smp_rmb();
730
731
732 pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writeable);
733 if (is_error_noslot_pfn(pfn)) {
734 err = -EFAULT;
735 goto out;
736 }
737
738 spin_lock(&kvm->mmu_lock);
739
740 if (mmu_notifier_retry(kvm, mmu_seq)) {
741
742
743
744
745
746 spin_unlock(&kvm->mmu_lock);
747 kvm_release_pfn_clean(pfn);
748 goto retry;
749 }
750
751
752 ptep = kvm_mips_pte_for_gpa(kvm, memcache, gpa);
753
754
755 prot_bits = _PAGE_PRESENT | __READABLE | _page_cachable_default;
756 if (writeable) {
757 prot_bits |= _PAGE_WRITE;
758 if (write_fault) {
759 prot_bits |= __WRITEABLE;
760 mark_page_dirty(kvm, gfn);
761 kvm_set_pfn_dirty(pfn);
762 }
763 }
764 entry = pfn_pte(pfn, __pgprot(prot_bits));
765
766
767 old_pte = *ptep;
768 set_pte(ptep, entry);
769
770 err = 0;
771 if (out_entry)
772 *out_entry = *ptep;
773 if (out_buddy)
774 *out_buddy = *ptep_buddy(ptep);
775
776 spin_unlock(&kvm->mmu_lock);
777 kvm_release_pfn_clean(pfn);
778 kvm_set_pfn_accessed(pfn);
779out:
780 srcu_read_unlock(&kvm->srcu, srcu_idx);
781 return err;
782}
783
784static pte_t *kvm_trap_emul_pte_for_gva(struct kvm_vcpu *vcpu,
785 unsigned long addr)
786{
787 struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
788 pgd_t *pgdp;
789 int ret;
790
791
792 ret = mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES,
793 KVM_NR_MEM_OBJS);
794 if (ret)
795 return NULL;
796
797 if (KVM_GUEST_KERNEL_MODE(vcpu))
798 pgdp = vcpu->arch.guest_kernel_mm.pgd;
799 else
800 pgdp = vcpu->arch.guest_user_mm.pgd;
801
802 return kvm_mips_walk_pgd(pgdp, memcache, addr);
803}
804
805void kvm_trap_emul_invalidate_gva(struct kvm_vcpu *vcpu, unsigned long addr,
806 bool user)
807{
808 pgd_t *pgdp;
809 pte_t *ptep;
810
811 addr &= PAGE_MASK << 1;
812
813 pgdp = vcpu->arch.guest_kernel_mm.pgd;
814 ptep = kvm_mips_walk_pgd(pgdp, NULL, addr);
815 if (ptep) {
816 ptep[0] = pfn_pte(0, __pgprot(0));
817 ptep[1] = pfn_pte(0, __pgprot(0));
818 }
819
820 if (user) {
821 pgdp = vcpu->arch.guest_user_mm.pgd;
822 ptep = kvm_mips_walk_pgd(pgdp, NULL, addr);
823 if (ptep) {
824 ptep[0] = pfn_pte(0, __pgprot(0));
825 ptep[1] = pfn_pte(0, __pgprot(0));
826 }
827 }
828}
829
830
831
832
833
834
835static bool kvm_mips_flush_gva_pte(pte_t *pte, unsigned long start_gva,
836 unsigned long end_gva)
837{
838 int i_min = __pte_offset(start_gva);
839 int i_max = __pte_offset(end_gva);
840 bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PTE - 1);
841 int i;
842
843
844
845
846
847 if (safe_to_remove)
848 return true;
849
850 for (i = i_min; i <= i_max; ++i) {
851 if (!pte_present(pte[i]))
852 continue;
853
854 set_pte(pte + i, __pte(0));
855 }
856 return false;
857}
858
859static bool kvm_mips_flush_gva_pmd(pmd_t *pmd, unsigned long start_gva,
860 unsigned long end_gva)
861{
862 pte_t *pte;
863 unsigned long end = ~0ul;
864 int i_min = __pmd_offset(start_gva);
865 int i_max = __pmd_offset(end_gva);
866 bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PMD - 1);
867 int i;
868
869 for (i = i_min; i <= i_max; ++i, start_gva = 0) {
870 if (!pmd_present(pmd[i]))
871 continue;
872
873 pte = pte_offset(pmd + i, 0);
874 if (i == i_max)
875 end = end_gva;
876
877 if (kvm_mips_flush_gva_pte(pte, start_gva, end)) {
878 pmd_clear(pmd + i);
879 pte_free_kernel(NULL, pte);
880 } else {
881 safe_to_remove = false;
882 }
883 }
884 return safe_to_remove;
885}
886
887static bool kvm_mips_flush_gva_pud(pud_t *pud, unsigned long start_gva,
888 unsigned long end_gva)
889{
890 pmd_t *pmd;
891 unsigned long end = ~0ul;
892 int i_min = __pud_offset(start_gva);
893 int i_max = __pud_offset(end_gva);
894 bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PUD - 1);
895 int i;
896
897 for (i = i_min; i <= i_max; ++i, start_gva = 0) {
898 if (!pud_present(pud[i]))
899 continue;
900
901 pmd = pmd_offset(pud + i, 0);
902 if (i == i_max)
903 end = end_gva;
904
905 if (kvm_mips_flush_gva_pmd(pmd, start_gva, end)) {
906 pud_clear(pud + i);
907 pmd_free(NULL, pmd);
908 } else {
909 safe_to_remove = false;
910 }
911 }
912 return safe_to_remove;
913}
914
915static bool kvm_mips_flush_gva_pgd(pgd_t *pgd, unsigned long start_gva,
916 unsigned long end_gva)
917{
918 pud_t *pud;
919 unsigned long end = ~0ul;
920 int i_min = pgd_index(start_gva);
921 int i_max = pgd_index(end_gva);
922 bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PGD - 1);
923 int i;
924
925 for (i = i_min; i <= i_max; ++i, start_gva = 0) {
926 if (!pgd_present(pgd[i]))
927 continue;
928
929 pud = pud_offset(pgd + i, 0);
930 if (i == i_max)
931 end = end_gva;
932
933 if (kvm_mips_flush_gva_pud(pud, start_gva, end)) {
934 pgd_clear(pgd + i);
935 pud_free(NULL, pud);
936 } else {
937 safe_to_remove = false;
938 }
939 }
940 return safe_to_remove;
941}
942
943void kvm_mips_flush_gva_pt(pgd_t *pgd, enum kvm_mips_flush flags)
944{
945 if (flags & KMF_GPA) {
946
947 if (flags & KMF_KERN)
948
949 kvm_mips_flush_gva_pgd(pgd, 0, 0x7fffffff);
950 else
951
952 kvm_mips_flush_gva_pgd(pgd, 0, 0x3fffffff);
953 } else {
954
955 kvm_mips_flush_gva_pgd(pgd, 0, 0x3fffffff);
956
957
958 if (flags & KMF_KERN)
959 kvm_mips_flush_gva_pgd(pgd, 0x60000000, 0x7fffffff);
960 }
961}
962
963static pte_t kvm_mips_gpa_pte_to_gva_unmapped(pte_t pte)
964{
965
966
967
968
969
970 if (!pte_dirty(pte))
971 pte = pte_wrprotect(pte);
972
973 return pte;
974}
975
976static pte_t kvm_mips_gpa_pte_to_gva_mapped(pte_t pte, long entrylo)
977{
978
979 if (!(entrylo & ENTRYLO_D))
980 pte = pte_mkclean(pte);
981
982 return kvm_mips_gpa_pte_to_gva_unmapped(pte);
983}
984
985#ifdef CONFIG_KVM_MIPS_VZ
986int kvm_mips_handle_vz_root_tlb_fault(unsigned long badvaddr,
987 struct kvm_vcpu *vcpu,
988 bool write_fault)
989{
990 int ret;
991
992 ret = kvm_mips_map_page(vcpu, badvaddr, write_fault, NULL, NULL);
993 if (ret)
994 return ret;
995
996
997 return kvm_vz_host_tlb_inv(vcpu, badvaddr);
998}
999#endif
1000
1001
1002int kvm_mips_handle_kseg0_tlb_fault(unsigned long badvaddr,
1003 struct kvm_vcpu *vcpu,
1004 bool write_fault)
1005{
1006 unsigned long gpa;
1007 pte_t pte_gpa[2], *ptep_gva;
1008 int idx;
1009
1010 if (KVM_GUEST_KSEGX(badvaddr) != KVM_GUEST_KSEG0) {
1011 kvm_err("%s: Invalid BadVaddr: %#lx\n", __func__, badvaddr);
1012 kvm_mips_dump_host_tlbs();
1013 return -1;
1014 }
1015
1016
1017 gpa = KVM_GUEST_CPHYSADDR(badvaddr);
1018 idx = (badvaddr >> PAGE_SHIFT) & 1;
1019 if (kvm_mips_map_page(vcpu, gpa, write_fault, &pte_gpa[idx],
1020 &pte_gpa[!idx]) < 0)
1021 return -1;
1022
1023
1024 ptep_gva = kvm_trap_emul_pte_for_gva(vcpu, badvaddr & ~PAGE_SIZE);
1025 if (!ptep_gva) {
1026 kvm_err("No ptep for gva %lx\n", badvaddr);
1027 return -1;
1028 }
1029
1030
1031 ptep_gva[0] = kvm_mips_gpa_pte_to_gva_unmapped(pte_gpa[0]);
1032 ptep_gva[1] = kvm_mips_gpa_pte_to_gva_unmapped(pte_gpa[1]);
1033
1034
1035 kvm_mips_host_tlb_inv(vcpu, badvaddr, false, true);
1036 return 0;
1037}
1038
1039int kvm_mips_handle_mapped_seg_tlb_fault(struct kvm_vcpu *vcpu,
1040 struct kvm_mips_tlb *tlb,
1041 unsigned long gva,
1042 bool write_fault)
1043{
1044 struct kvm *kvm = vcpu->kvm;
1045 long tlb_lo[2];
1046 pte_t pte_gpa[2], *ptep_buddy, *ptep_gva;
1047 unsigned int idx = TLB_LO_IDX(*tlb, gva);
1048 bool kernel = KVM_GUEST_KERNEL_MODE(vcpu);
1049
1050 tlb_lo[0] = tlb->tlb_lo[0];
1051 tlb_lo[1] = tlb->tlb_lo[1];
1052
1053
1054
1055
1056
1057 if (!((gva ^ KVM_GUEST_COMMPAGE_ADDR) & VPN2_MASK & (PAGE_MASK << 1)))
1058 tlb_lo[TLB_LO_IDX(*tlb, KVM_GUEST_COMMPAGE_ADDR)] = 0;
1059
1060
1061 if (kvm_mips_map_page(vcpu, mips3_tlbpfn_to_paddr(tlb_lo[idx]),
1062 write_fault, &pte_gpa[idx], NULL) < 0)
1063 return -1;
1064
1065
1066 pte_gpa[!idx] = pfn_pte(0, __pgprot(0));
1067 if (tlb_lo[!idx] & ENTRYLO_V) {
1068 spin_lock(&kvm->mmu_lock);
1069 ptep_buddy = kvm_mips_pte_for_gpa(kvm, NULL,
1070 mips3_tlbpfn_to_paddr(tlb_lo[!idx]));
1071 if (ptep_buddy)
1072 pte_gpa[!idx] = *ptep_buddy;
1073 spin_unlock(&kvm->mmu_lock);
1074 }
1075
1076
1077 ptep_gva = kvm_trap_emul_pte_for_gva(vcpu, gva & ~PAGE_SIZE);
1078 if (!ptep_gva) {
1079 kvm_err("No ptep for gva %lx\n", gva);
1080 return -1;
1081 }
1082
1083
1084 ptep_gva[0] = kvm_mips_gpa_pte_to_gva_mapped(pte_gpa[0], tlb_lo[0]);
1085 ptep_gva[1] = kvm_mips_gpa_pte_to_gva_mapped(pte_gpa[1], tlb_lo[1]);
1086
1087
1088 kvm_mips_host_tlb_inv(vcpu, gva, !kernel, kernel);
1089
1090 kvm_debug("@ %#lx tlb_lo0: 0x%08lx tlb_lo1: 0x%08lx\n", vcpu->arch.pc,
1091 tlb->tlb_lo[0], tlb->tlb_lo[1]);
1092
1093 return 0;
1094}
1095
1096int kvm_mips_handle_commpage_tlb_fault(unsigned long badvaddr,
1097 struct kvm_vcpu *vcpu)
1098{
1099 kvm_pfn_t pfn;
1100 pte_t *ptep;
1101
1102 ptep = kvm_trap_emul_pte_for_gva(vcpu, badvaddr);
1103 if (!ptep) {
1104 kvm_err("No ptep for commpage %lx\n", badvaddr);
1105 return -1;
1106 }
1107
1108 pfn = PFN_DOWN(virt_to_phys(vcpu->arch.kseg0_commpage));
1109
1110 *ptep = pte_mkyoung(pte_mkdirty(pfn_pte(pfn, PAGE_SHARED)));
1111
1112
1113 kvm_mips_host_tlb_inv(vcpu, badvaddr, false, true);
1114 return 0;
1115}
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128static void kvm_mips_migrate_count(struct kvm_vcpu *vcpu)
1129{
1130 if (hrtimer_cancel(&vcpu->arch.comparecount_timer))
1131 hrtimer_restart(&vcpu->arch.comparecount_timer);
1132}
1133
1134
1135void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1136{
1137 unsigned long flags;
1138
1139 kvm_debug("%s: vcpu %p, cpu: %d\n", __func__, vcpu, cpu);
1140
1141 local_irq_save(flags);
1142
1143 vcpu->cpu = cpu;
1144 if (vcpu->arch.last_sched_cpu != cpu) {
1145 kvm_debug("[%d->%d]KVM VCPU[%d] switch\n",
1146 vcpu->arch.last_sched_cpu, cpu, vcpu->vcpu_id);
1147
1148
1149
1150
1151
1152 kvm_mips_migrate_count(vcpu);
1153 }
1154
1155
1156 kvm_mips_callbacks->vcpu_load(vcpu, cpu);
1157
1158 local_irq_restore(flags);
1159}
1160
1161
1162void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1163{
1164 unsigned long flags;
1165 int cpu;
1166
1167 local_irq_save(flags);
1168
1169 cpu = smp_processor_id();
1170 vcpu->arch.last_sched_cpu = cpu;
1171 vcpu->cpu = -1;
1172
1173
1174 kvm_mips_callbacks->vcpu_put(vcpu, cpu);
1175
1176 local_irq_restore(flags);
1177}
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195enum kvm_mips_fault_result kvm_trap_emul_gva_fault(struct kvm_vcpu *vcpu,
1196 unsigned long gva,
1197 bool write)
1198{
1199 struct mips_coproc *cop0 = vcpu->arch.cop0;
1200 struct kvm_mips_tlb *tlb;
1201 int index;
1202
1203 if (KVM_GUEST_KSEGX(gva) == KVM_GUEST_KSEG0) {
1204 if (kvm_mips_handle_kseg0_tlb_fault(gva, vcpu, write) < 0)
1205 return KVM_MIPS_GPA;
1206 } else if ((KVM_GUEST_KSEGX(gva) < KVM_GUEST_KSEG0) ||
1207 KVM_GUEST_KSEGX(gva) == KVM_GUEST_KSEG23) {
1208
1209 index = kvm_mips_guest_tlb_lookup(vcpu, (gva & VPN2_MASK) |
1210 (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID));
1211 if (index < 0)
1212 return KVM_MIPS_TLB;
1213 tlb = &vcpu->arch.guest_tlb[index];
1214
1215
1216 if (!TLB_IS_VALID(*tlb, gva))
1217 return KVM_MIPS_TLBINV;
1218 if (write && !TLB_IS_DIRTY(*tlb, gva))
1219 return KVM_MIPS_TLBMOD;
1220
1221 if (kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb, gva, write))
1222 return KVM_MIPS_GPA;
1223 } else {
1224 return KVM_MIPS_GVA;
1225 }
1226
1227 return KVM_MIPS_MAPPED;
1228}
1229
1230int kvm_get_inst(u32 *opc, struct kvm_vcpu *vcpu, u32 *out)
1231{
1232 int err;
1233
1234 if (WARN(IS_ENABLED(CONFIG_KVM_MIPS_VZ),
1235 "Expect BadInstr/BadInstrP registers to be used with VZ\n"))
1236 return -EINVAL;
1237
1238retry:
1239 kvm_trap_emul_gva_lockless_begin(vcpu);
1240 err = get_user(*out, opc);
1241 kvm_trap_emul_gva_lockless_end(vcpu);
1242
1243 if (unlikely(err)) {
1244
1245
1246
1247
1248 err = kvm_trap_emul_gva_fault(vcpu, (unsigned long)opc,
1249 false);
1250 if (unlikely(err)) {
1251 kvm_err("%s: illegal address: %p\n",
1252 __func__, opc);
1253 return -EFAULT;
1254 }
1255
1256
1257 goto retry;
1258 }
1259 return 0;
1260}
1261